247 lines
9.3 KiB
Go
247 lines
9.3 KiB
Go
package app
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
|
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
|
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
|
"go.maximumdirect.net/commandkit"
|
|
)
|
|
|
|
type pipelineCampaignConfig = config.LoadedPipelineCampaign
|
|
|
|
type commandConfig struct {
|
|
Config *config.Config
|
|
cleanup func() error
|
|
}
|
|
|
|
func (c *commandConfig) Close() error {
|
|
if c == nil || c.cleanup == nil {
|
|
return nil
|
|
}
|
|
cleanup := c.cleanup
|
|
c.cleanup = nil
|
|
return cleanup()
|
|
}
|
|
|
|
func retainedCommandConfig(cfg *config.Config) *commandConfig {
|
|
return &commandConfig{Config: cfg}
|
|
}
|
|
|
|
func loadCommandConfigWithDependencies(ctx context.Context, pipelineFlag, campaignFlag, campaignFileFlag, sessionFlag string, sessionOpts config.SessionLoadOptions, secretsDir *string, deps commandDependencies) (*commandConfig, error) {
|
|
selection := configurationSelection{}
|
|
for _, field := range []struct {
|
|
value string
|
|
target *commandkit.Supplied[string]
|
|
}{
|
|
{pipelineFlag, &selection.Config}, {campaignFlag, &selection.Campaign}, {campaignFileFlag, &selection.CampaignFile},
|
|
{sessionFlag, &selection.SessionFile}, {sessionOpts.SessionID, &selection.Session},
|
|
} {
|
|
if field.value != "" {
|
|
*field.target = commandkit.Supplied[string]{Value: field.value, Set: true}
|
|
}
|
|
}
|
|
if secretsDir != nil {
|
|
selection.SecretsDir = commandkit.Supplied[string]{Value: *secretsDir, Set: true}
|
|
}
|
|
if sessionOpts.Overlay != nil {
|
|
selection.Overlay = commandkit.Supplied[string]{Value: *sessionOpts.Overlay, Set: true}
|
|
}
|
|
acquisition := newConfigurationAcquisition(selection, deps)
|
|
acquisition.previousSessionID = sessionOpts.PreviousSessionID
|
|
cfg, err := acquisition.Domain(ctx)
|
|
if err != nil {
|
|
return nil, errors.Join(err, acquisition.Close())
|
|
}
|
|
return &commandConfig{Config: cfg, cleanup: acquisition.Close}, nil
|
|
}
|
|
|
|
type acquiredSession struct {
|
|
Config *config.SessionConfig
|
|
Path string
|
|
Source config.SessionSource
|
|
cleanup func() error
|
|
}
|
|
|
|
func loadRemoteSessionConfig(ctx context.Context, base *config.LoadedPipelineCampaign, sessionOpts config.SessionLoadOptions, deps commandDependencies) (loaded *acquiredSession, err error) {
|
|
var cleanup func() error
|
|
defer func() {
|
|
if err != nil && cleanup != nil {
|
|
err = errors.Join(err, cleanup())
|
|
}
|
|
}()
|
|
sessionOpts.Campaign = base.Campaign
|
|
sessionOpts.CampaignPath = base.CampaignPath
|
|
sessionID := strings.TrimSpace(sessionOpts.SessionID)
|
|
if sessionID == "" {
|
|
return nil, missingSessionConfigError(config.DefaultSessionConfigSearchPaths, "remote session loading requires a session_id")
|
|
}
|
|
|
|
rootPrefix := ""
|
|
if base.Pipeline.Storage.S3 != nil {
|
|
rootPrefix = base.Pipeline.Storage.S3.RootPrefix
|
|
}
|
|
sessionPrefix := artifacts.S3SessionPrefix(rootPrefix, config.CampaignID(base.Campaign), sessionID)
|
|
remoteKey := artifacts.S3SessionConfigKey(sessionPrefix)
|
|
partialCfg := &config.Config{Pipeline: base.Pipeline, PipelinePath: base.PipelinePath, Campaign: base.Campaign, CampaignPath: base.CampaignPath}
|
|
store, err := newCommandObjectStoreWithDependencies(ctx, partialCfg, nil, deps)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("remote session %q unavailable: %w", remoteKey, err)
|
|
}
|
|
|
|
sessionInfo, checksum, body, err := findRemoteSessionConfig(ctx, store, sessionPrefix, remoteKey, config.CampaignID(base.Campaign), sessionID)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("discover remote session: %w", err)
|
|
}
|
|
defer func() { err = errors.Join(err, body.Close()) }()
|
|
remoteKey = sessionInfo.Key
|
|
temporary, err := os.CreateTemp("", "narratio-session-*.yml")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("remote session %q download failed: %w", remoteKey, err)
|
|
}
|
|
sessionTempPath := temporary.Name()
|
|
cleanup = func() error {
|
|
if err := fileops.RemoveAllUnderRoot(filepath.Dir(sessionTempPath), sessionTempPath); err != nil {
|
|
return fmt.Errorf("remove downloaded remote session config: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
_, copyErr := io.Copy(temporary, body)
|
|
closeErr := temporary.Close()
|
|
if err := errors.Join(copyErr, closeErr); err != nil {
|
|
return nil, fmt.Errorf("download remote session %q: %w", remoteKey, err)
|
|
}
|
|
sessionBytes, err := os.ReadFile(sessionTempPath)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("read downloaded remote session config: %w", err)
|
|
}
|
|
if checksum != "" {
|
|
digest := sha256.Sum256(sessionBytes)
|
|
if !strings.EqualFold(hex.EncodeToString(digest[:]), checksum) || int64(len(sessionBytes)) != sessionInfo.Size {
|
|
return nil, fmt.Errorf("remote session %q does not match its committed checksum or size", remoteKey)
|
|
}
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
sessionCfg, err := config.LoadSessionBytesWithOptions("s3://"+s3BucketName(base.Pipeline)+"/"+remoteKey, sessionBytes, sessionOpts)
|
|
err = errors.Join(err, ctx.Err())
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
source := "session_config.s3"
|
|
if checksum != "" {
|
|
source = "session_config.s3.committed_snapshot"
|
|
}
|
|
loaded = &acquiredSession{Config: sessionCfg, Path: sessionTempPath, cleanup: cleanup,
|
|
Source: config.SessionSource{Source: source, LocalPath: sessionTempPath, S3Bucket: s3BucketName(base.Pipeline), S3Key: remoteKey, S3Size: sessionInfo.Size, S3ETag: sessionInfo.ETag}}
|
|
cleanup = nil
|
|
return loaded, nil
|
|
}
|
|
|
|
func loadPipelineCampaignConfigWithDependencies(pipelineFlag, campaignFlag, campaignFileFlag string, pipelineOpts config.PipelineLoadOptions, secretsDir *string, deps commandDependencies) (*pipelineCampaignConfig, error) {
|
|
deps = normalizeCommandDependencies(deps)
|
|
loadedPipelinePath, pipelineCfg, err := loadPipelineConfigWithDependencies(pipelineFlag, pipelineOpts, secretsDir, deps)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
resolvedCampaignPath, err := resolveCampaignConfigPath(pipelineCfg, campaignFlag, campaignFileFlag)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
campaignCfg, err := config.LoadCampaign(resolvedCampaignPath)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if selectedID := strings.TrimSpace(campaignFlag); selectedID != "" {
|
|
if got := config.CampaignID(campaignCfg); got != selectedID {
|
|
return nil, &config.SessionIdentityError{Source: resolvedCampaignPath, Field: "campaign_id", Expected: selectedID, Actual: got}
|
|
}
|
|
}
|
|
loaded, err := config.LoadPipelineCampaign(loadedPipelinePath, pipelineCfg, resolvedCampaignPath, campaignCfg)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &loaded, nil
|
|
}
|
|
|
|
func loadPipelineConfigWithDependencies(pipelineFlag string, opts config.PipelineLoadOptions, secretsDir *string, deps commandDependencies) (string, *config.PipelineConfig, error) {
|
|
deps = normalizeCommandDependencies(deps)
|
|
if !deps.secretsBootstrap.selectionSet {
|
|
deps.secretsBootstrap.selection = secretsDir
|
|
deps.secretsBootstrap.selectionSet = true
|
|
}
|
|
if err := deps.secretsBootstrap.load(deps.loadSecretsDir, deps.diagnostics); err != nil {
|
|
return "", nil, err
|
|
}
|
|
opts.Search = deps.pipelineSearch
|
|
pipelineCfg, err := deps.loadPipelineConfig(strings.TrimSpace(pipelineFlag), opts)
|
|
if err != nil {
|
|
return "", nil, err
|
|
}
|
|
resolvedPipelinePath := config.EffectivePipelineRootPath(pipelineCfg)
|
|
if resolvedPipelinePath == "" {
|
|
resolvedPipelinePath = strings.TrimSpace(pipelineFlag)
|
|
}
|
|
return resolvedPipelinePath, pipelineCfg, nil
|
|
}
|
|
|
|
func findRemoteSessionConfig(ctx context.Context, store storage.ObjectStore, sessionPrefix, remoteKey, campaign, sessionID string) (storage.ObjectInfo, string, io.ReadCloser, error) {
|
|
authored, err := store.Exists(ctx, remoteKey)
|
|
if err != nil {
|
|
return storage.ObjectInfo{}, "", nil, fmt.Errorf("check authored session %q: %w", remoteKey, err)
|
|
}
|
|
var selected *artifacts.RemoteArtifact
|
|
if !authored {
|
|
current, err := artifacts.LoadCurrentState(ctx, store, sessionPrefix, artifacts.CurrentStateValidation{ExpectedCampaign: campaign, ExpectedSessionID: sessionID, ValidateRunID: true})
|
|
if err == nil {
|
|
for _, item := range current.Commit.Artifacts {
|
|
if item.Type == artifacts.RemoteArtifactTypeRunFile && item.Source == "run:prepare/inputs/session.yml" {
|
|
identity, err := artifacts.ParseImmutableRunObjectKey(sessionPrefix, item.DestinationKey)
|
|
if err != nil || "run:"+identity.RelativePath != item.Source {
|
|
return storage.ObjectInfo{}, "", nil, fmt.Errorf("committed session source does not match its object key")
|
|
}
|
|
copy := item
|
|
selected = ©
|
|
remoteKey = item.DestinationKey
|
|
break
|
|
}
|
|
}
|
|
} else if !errors.Is(err, artifacts.ErrCurrentCommitPointerMissing) {
|
|
return storage.ObjectInfo{}, "", nil, fmt.Errorf("load remote session selection: %w", err)
|
|
}
|
|
}
|
|
info, body, err := store.Read(ctx, remoteKey)
|
|
if err != nil {
|
|
if errors.Is(err, os.ErrNotExist) {
|
|
return storage.ObjectInfo{}, "", nil, fmt.Errorf("remote session %q not found: %w", remoteKey, err)
|
|
}
|
|
return storage.ObjectInfo{}, "", nil, fmt.Errorf("read remote session %q: %w", remoteKey, err)
|
|
}
|
|
if selected != nil {
|
|
if info.Size != selected.Size || strings.TrimSpace(info.ETag) != selected.Generation {
|
|
return storage.ObjectInfo{}, "", nil, errors.Join(fmt.Errorf("remote session %q differs from committed object identity", remoteKey), body.Close())
|
|
}
|
|
return info, selected.SHA256, body, nil
|
|
}
|
|
return info, "", body, nil
|
|
}
|
|
|
|
func s3BucketName(cfg *config.PipelineConfig) string {
|
|
if cfg == nil || cfg.Storage.S3 == nil {
|
|
return ""
|
|
}
|
|
return strings.TrimSpace(cfg.Storage.S3.Bucket)
|
|
}
|