package app import ( "context" "crypto/sha256" "encoding/hex" "errors" "fmt" "io" "os" "path/filepath" "strings" "gitea.maximumdirect.net/eric/narratio/internal/adapters/storage" "gitea.maximumdirect.net/eric/narratio/internal/artifacts" "gitea.maximumdirect.net/eric/narratio/internal/config" "gitea.maximumdirect.net/eric/narratio/internal/fileops" "go.maximumdirect.net/commandkit" ) type pipelineCampaignConfig = config.LoadedPipelineCampaign type commandConfig struct { Config *config.Config cleanup func() error } func (c *commandConfig) Close() error { if c == nil || c.cleanup == nil { return nil } cleanup := c.cleanup c.cleanup = nil return cleanup() } func retainedCommandConfig(cfg *config.Config) *commandConfig { return &commandConfig{Config: cfg} } func loadCommandConfigWithDependencies(ctx context.Context, pipelineFlag, campaignFlag, campaignFileFlag, sessionFlag string, sessionOpts config.SessionLoadOptions, secretsDir *string, deps commandDependencies) (*commandConfig, error) { selection := configurationSelection{} for _, field := range []struct { value string target *commandkit.Supplied[string] }{ {pipelineFlag, &selection.Config}, {campaignFlag, &selection.Campaign}, {campaignFileFlag, &selection.CampaignFile}, {sessionFlag, &selection.SessionFile}, {sessionOpts.SessionID, &selection.Session}, } { if field.value != "" { *field.target = commandkit.Supplied[string]{Value: field.value, Set: true} } } if secretsDir != nil { selection.SecretsDir = commandkit.Supplied[string]{Value: *secretsDir, Set: true} } if sessionOpts.Overlay != nil { selection.Overlay = commandkit.Supplied[string]{Value: *sessionOpts.Overlay, Set: true} } acquisition := newConfigurationAcquisition(selection, deps) acquisition.previousSessionID = sessionOpts.PreviousSessionID cfg, err := acquisition.Domain(ctx) if err != nil { return nil, errors.Join(err, acquisition.Close()) } return &commandConfig{Config: cfg, cleanup: acquisition.Close}, nil } type acquiredSession struct { Config *config.SessionConfig Path string Source config.SessionSource cleanup func() error } func loadRemoteSessionConfig(ctx context.Context, base *config.LoadedPipelineCampaign, sessionOpts config.SessionLoadOptions, deps commandDependencies) (loaded *acquiredSession, err error) { var cleanup func() error defer func() { if err != nil && cleanup != nil { err = errors.Join(err, cleanup()) } }() sessionOpts.Campaign = base.Campaign sessionOpts.CampaignPath = base.CampaignPath sessionID := strings.TrimSpace(sessionOpts.SessionID) if sessionID == "" { return nil, missingSessionConfigError(config.DefaultSessionConfigSearchPaths, "remote session loading requires a session_id") } rootPrefix := "" if base.Pipeline.Storage.S3 != nil { rootPrefix = base.Pipeline.Storage.S3.RootPrefix } sessionPrefix := artifacts.S3SessionPrefix(rootPrefix, config.CampaignID(base.Campaign), sessionID) remoteKey := artifacts.S3SessionConfigKey(sessionPrefix) partialCfg := &config.Config{Pipeline: base.Pipeline, PipelinePath: base.PipelinePath, Campaign: base.Campaign, CampaignPath: base.CampaignPath} store, err := newCommandObjectStoreWithDependencies(ctx, partialCfg, nil, deps) if err != nil { return nil, fmt.Errorf("remote session %q unavailable: %w", remoteKey, err) } sessionInfo, checksum, body, err := findRemoteSessionConfig(ctx, store, sessionPrefix, remoteKey, config.CampaignID(base.Campaign), sessionID) if err != nil { return nil, fmt.Errorf("discover remote session: %w", err) } defer func() { err = errors.Join(err, body.Close()) }() remoteKey = sessionInfo.Key temporary, err := os.CreateTemp("", "narratio-session-*.yml") if err != nil { return nil, fmt.Errorf("remote session %q download failed: %w", remoteKey, err) } sessionTempPath := temporary.Name() cleanup = func() error { if err := fileops.RemoveAllUnderRoot(filepath.Dir(sessionTempPath), sessionTempPath); err != nil { return fmt.Errorf("remove downloaded remote session config: %w", err) } return nil } _, copyErr := io.Copy(temporary, body) closeErr := temporary.Close() if err := errors.Join(copyErr, closeErr); err != nil { return nil, fmt.Errorf("download remote session %q: %w", remoteKey, err) } sessionBytes, err := os.ReadFile(sessionTempPath) if err != nil { return nil, fmt.Errorf("read downloaded remote session config: %w", err) } if checksum != "" { digest := sha256.Sum256(sessionBytes) if !strings.EqualFold(hex.EncodeToString(digest[:]), checksum) || int64(len(sessionBytes)) != sessionInfo.Size { return nil, fmt.Errorf("remote session %q does not match its committed checksum or size", remoteKey) } } if err := ctx.Err(); err != nil { return nil, err } sessionCfg, err := config.LoadSessionBytesWithOptions("s3://"+s3BucketName(base.Pipeline)+"/"+remoteKey, sessionBytes, sessionOpts) err = errors.Join(err, ctx.Err()) if err != nil { return nil, err } source := "session_config.s3" if checksum != "" { source = "session_config.s3.committed_snapshot" } loaded = &acquiredSession{Config: sessionCfg, Path: sessionTempPath, cleanup: cleanup, Source: config.SessionSource{Source: source, LocalPath: sessionTempPath, S3Bucket: s3BucketName(base.Pipeline), S3Key: remoteKey, S3Size: sessionInfo.Size, S3ETag: sessionInfo.ETag}} cleanup = nil return loaded, nil } func loadPipelineCampaignConfigWithDependencies(pipelineFlag, campaignFlag, campaignFileFlag string, pipelineOpts config.PipelineLoadOptions, secretsDir *string, deps commandDependencies) (*pipelineCampaignConfig, error) { deps = normalizeCommandDependencies(deps) loadedPipelinePath, pipelineCfg, err := loadPipelineConfigWithDependencies(pipelineFlag, pipelineOpts, secretsDir, deps) if err != nil { return nil, err } resolvedCampaignPath, err := resolveCampaignConfigPath(pipelineCfg, campaignFlag, campaignFileFlag) if err != nil { return nil, err } campaignCfg, err := config.LoadCampaign(resolvedCampaignPath) if err != nil { return nil, err } if selectedID := strings.TrimSpace(campaignFlag); selectedID != "" { if got := config.CampaignID(campaignCfg); got != selectedID { return nil, &config.SessionIdentityError{Source: resolvedCampaignPath, Field: "campaign_id", Expected: selectedID, Actual: got} } } loaded, err := config.LoadPipelineCampaign(loadedPipelinePath, pipelineCfg, resolvedCampaignPath, campaignCfg) if err != nil { return nil, err } return &loaded, nil } func loadPipelineConfigWithDependencies(pipelineFlag string, opts config.PipelineLoadOptions, secretsDir *string, deps commandDependencies) (string, *config.PipelineConfig, error) { deps = normalizeCommandDependencies(deps) if !deps.secretsBootstrap.selectionSet { deps.secretsBootstrap.selection = secretsDir deps.secretsBootstrap.selectionSet = true } if err := deps.secretsBootstrap.load(deps.loadSecretsDir, deps.diagnostics); err != nil { return "", nil, err } opts.Search = deps.pipelineSearch pipelineCfg, err := deps.loadPipelineConfig(strings.TrimSpace(pipelineFlag), opts) if err != nil { return "", nil, err } resolvedPipelinePath := config.EffectivePipelineRootPath(pipelineCfg) if resolvedPipelinePath == "" { resolvedPipelinePath = strings.TrimSpace(pipelineFlag) } return resolvedPipelinePath, pipelineCfg, nil } func findRemoteSessionConfig(ctx context.Context, store storage.ObjectStore, sessionPrefix, remoteKey, campaign, sessionID string) (storage.ObjectInfo, string, io.ReadCloser, error) { authored, err := store.Exists(ctx, remoteKey) if err != nil { return storage.ObjectInfo{}, "", nil, fmt.Errorf("check authored session %q: %w", remoteKey, err) } var selected *artifacts.RemoteArtifact if !authored { current, err := artifacts.LoadCurrentState(ctx, store, sessionPrefix, artifacts.CurrentStateValidation{ExpectedCampaign: campaign, ExpectedSessionID: sessionID, ValidateRunID: true}) if err == nil { for _, item := range current.Commit.Artifacts { if item.Type == artifacts.RemoteArtifactTypeRunFile && item.Source == "run:prepare/inputs/session.yml" { identity, err := artifacts.ParseImmutableRunObjectKey(sessionPrefix, item.DestinationKey) if err != nil || "run:"+identity.RelativePath != item.Source { return storage.ObjectInfo{}, "", nil, fmt.Errorf("committed session source does not match its object key") } copy := item selected = © remoteKey = item.DestinationKey break } } } else if !errors.Is(err, artifacts.ErrCurrentCommitPointerMissing) { return storage.ObjectInfo{}, "", nil, fmt.Errorf("load remote session selection: %w", err) } } info, body, err := store.Read(ctx, remoteKey) if err != nil { if errors.Is(err, os.ErrNotExist) { return storage.ObjectInfo{}, "", nil, fmt.Errorf("remote session %q not found: %w", remoteKey, err) } return storage.ObjectInfo{}, "", nil, fmt.Errorf("read remote session %q: %w", remoteKey, err) } if selected != nil { if info.Size != selected.Size || strings.TrimSpace(info.ETag) != selected.Generation { return storage.ObjectInfo{}, "", nil, errors.Join(fmt.Errorf("remote session %q differs from committed object identity", remoteKey), body.Close()) } return info, selected.SHA256, body, nil } return info, "", body, nil } func s3BucketName(cfg *config.PipelineConfig) string { if cfg == nil || cfg.Storage.S3 == nil { return "" } return strings.TrimSpace(cfg.Storage.S3.Bucket) }