From ed7e72207711a24571c5375dea54d50cef9bc9ae Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Tue, 17 Nov 2026 22:27:24 +0000 Subject: [PATCH] Publish and restore prepared session documents --- docs/roadmap/implementation.md | 2 +- internal/app/config_loader.go | 43 +++++- .../app/prepared_session_roundtrip_test.go | 130 ++++++++++++++++++ internal/app/restore_plan.go | 10 ++ internal/stage/publish.go | 19 ++- internal/stage/publish_sources.go | 41 ++++++ 6 files changed, 238 insertions(+), 7 deletions(-) create mode 100644 internal/app/prepared_session_roundtrip_test.go diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index a62aa53..2a3a872 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -256,7 +256,7 @@ Test standalone prepared documents, imported changes, equivalent layouts, and remote-origin metadata. Canonical-byte format changes can cause a normal prepare rerun; no session migration or new manifest schema is needed for serialization. -## Stage 7: Close publication, restore, and remote-session round trips +## Stage 7: Close publication, restore, and remote-session round trips [x] Read publish input selection, remote-session loading, restore contracts, and existing in-memory storage integration tests. diff --git a/internal/app/config_loader.go b/internal/app/config_loader.go index c509b62..a6bd653 100644 --- a/internal/app/config_loader.go +++ b/internal/app/config_loader.go @@ -2,6 +2,8 @@ package app import ( "context" + "crypto/sha256" + "encoding/hex" "errors" "fmt" "os" @@ -91,10 +93,11 @@ func loadCommandConfigWithDependencies(ctx context.Context, pipelineFlag, campai return nil, missingSessionConfigError(config.DefaultSessionConfigSearchPaths, fmt.Sprintf("remote session %q unavailable: %v", remoteKey, err)) } - sessionInfo, err := findRemoteSessionConfig(ctx, store, sessionPrefix, remoteKey) + sessionInfo, checksum, err := findRemoteSessionConfig(ctx, store, sessionPrefix, remoteKey, config.CampaignID(base.Campaign), sessionID) if err != nil { return nil, missingSessionConfigError(config.DefaultSessionConfigSearchPaths, err.Error()) } + remoteKey = sessionInfo.Key sessionTempPath, err := deps.downloadObjectToTemp(ctx, store, remoteKey, "narratio-session-*.yml") if err != nil { return nil, missingSessionConfigError(config.DefaultSessionConfigSearchPaths, fmt.Sprintf("remote session %q download failed: %v", remoteKey, err)) @@ -109,6 +112,12 @@ func loadCommandConfigWithDependencies(ctx context.Context, pipelineFlag, campai if err != nil { return nil, fmt.Errorf("read downloaded remote session config: %w", err) } + if checksum != "" { + digest := sha256.Sum256(sessionBytes) + if !strings.EqualFold(hex.EncodeToString(digest[:]), checksum) || int64(len(sessionBytes)) != sessionInfo.Size { + return nil, fmt.Errorf("remote session %q does not match its committed checksum or size", remoteKey) + } + } sessionCfg, err := config.LoadSessionBytesWithOptions("s3://"+s3BucketName(base.Pipeline)+"/"+remoteKey, sessionBytes, sessionOpts) if err != nil { return nil, err @@ -182,17 +191,41 @@ func loadPipelineConfigWithDependencies(pipelineFlag string, opts config.Pipelin return resolvedPipelinePath, pipelineCfg, nil } -func findRemoteSessionConfig(ctx context.Context, store storage.ObjectStore, sessionPrefix, remoteKey string) (storage.ObjectInfo, error) { +func findRemoteSessionConfig(ctx context.Context, store storage.ObjectStore, sessionPrefix, remoteKey, campaign, sessionID string) (storage.ObjectInfo, string, error) { + current, err := artifacts.LoadCurrentState(ctx, store, sessionPrefix, artifacts.CurrentStateValidation{ExpectedCampaign: campaign, ExpectedSessionID: sessionID, ValidateRunID: true}) + var selected *artifacts.RemoteArtifact + if err == nil { + for _, item := range current.Commit.Artifacts { + if item.Type == artifacts.RemoteArtifactTypeRunFile && item.Source == "run:prepare/inputs/session.yml" { + identity, err := artifacts.ParseImmutableRunObjectKey(sessionPrefix, item.DestinationKey) + if err != nil || "run:"+identity.RelativePath != item.Source { + return storage.ObjectInfo{}, "", fmt.Errorf("committed session source does not match its object key") + } + copy := item + selected = © + remoteKey = item.DestinationKey + break + } + } + } else if !errors.Is(err, artifacts.ErrCurrentCommitPointerMissing) { + return storage.ObjectInfo{}, "", fmt.Errorf("load remote session selection: %w", err) + } objects, err := store.List(ctx, sessionPrefix) if err != nil { - return storage.ObjectInfo{}, fmt.Errorf("remote session %q list failed: %w", remoteKey, err) + return storage.ObjectInfo{}, "", fmt.Errorf("remote session %q list failed: %w", remoteKey, err) } for _, obj := range objects { if obj.Key == remoteKey { - return obj, nil + if selected != nil { + if obj.Size != selected.Size || strings.TrimSpace(obj.ETag) != selected.Generation { + return storage.ObjectInfo{}, "", fmt.Errorf("remote session %q differs from committed object identity", remoteKey) + } + return obj, selected.SHA256, nil + } + return obj, "", nil } } - return storage.ObjectInfo{}, fmt.Errorf("remote session %q not found", remoteKey) + return storage.ObjectInfo{}, "", fmt.Errorf("remote session %q not found", remoteKey) } func s3BucketName(cfg *config.PipelineConfig) string { diff --git a/internal/app/prepared_session_roundtrip_test.go b/internal/app/prepared_session_roundtrip_test.go new file mode 100644 index 0000000..be2c4f8 --- /dev/null +++ b/internal/app/prepared_session_roundtrip_test.go @@ -0,0 +1,130 @@ +package app + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "gitea.maximumdirect.net/eric/narratio/internal/adapters/storage" + "gitea.maximumdirect.net/eric/narratio/internal/artifacts" + "gitea.maximumdirect.net/eric/narratio/internal/config" + "gitea.maximumdirect.net/eric/narratio/internal/manifest" + "gitea.maximumdirect.net/eric/narratio/internal/stage" +) + +func TestPreparedSessionPublishAndRemoteLoad(t *testing.T) { + ctx := context.Background() + pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, t.TempDir()) + fragment := filepath.Join(filepath.Dir(sessionPath), "fragment.yml") + mustWriteTestFile(t, sessionPath, "config:\n imports:\n - path: fragment.yml\nsession_id: 2026-05-03\n") + mustWriteTestFile(t, fragment, "title: Published title\ninputs: {audio_dir: audio}\n") + cfg, err := config.LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, config.SessionLoadOptions{}) + if err != nil { + t.Fatal(err) + } + enabled := true + cfg.Pipeline.Publish.Enabled = &enabled + cfg.Pipeline.Publish.UploadRun = &enabled + cfg.Pipeline.Publish.Outputs = nil + fake := &storage.FakeBackend{} + store := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root) + env := &stage.Env{Config: cfg, ArtifactStore: store, ObjectStore: fake} + m := manifest.New("2026-05-03", time.Now().UTC()) + m.Campaign = "sample-campaign" + m.RunID = "20260519T010203Z-a1b2c3d4" + runRoot := artifacts.SessionRunRootForCampaign(cfg.Pipeline.Workspace.Root, m.Campaign, m.SessionID, m.RunID) + m.LocalWorkDir = runRoot + if err := os.MkdirAll(runRoot, 0o755); err != nil { + t.Fatal(err) + } + prepare, err := stage.Select("prepare") + if err != nil { + t.Fatal(err) + } + if _, err := prepare.Run(ctx, env, m); err != nil { + t.Fatal(err) + } + for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze"} { + m.MarkStageSucceeded(name, time.Now().UTC(), nil) + } + if err := os.Remove(sessionPath); err != nil { + t.Fatal(err) + } + if err := os.Remove(fragment); err != nil { + t.Fatal(err) + } + publish, err := stage.Select("publish") + if err != nil { + t.Fatal(err) + } + if _, err := publish.Run(ctx, env, m); err != nil { + t.Fatal(err) + } + prefix := artifacts.S3SessionPrefix("dnd", m.Campaign, m.SessionID) + key := artifacts.S3RunRelativeDestinationKey(artifacts.S3RunPrefix(prefix, m.RunID), "prepare/inputs/session.yml") + if got := string(fake.Objects[key].Data); !strings.Contains(got, "Published title") || strings.Contains(got, "imports:") || strings.Contains(got, "speakers_file:") { + t.Fatalf("published session = %q", got) + } + for _, name := range []string{"campaign.yml", "party.yml"} { + key := artifacts.S3RunRelativeDestinationKey(artifacts.S3RunPrefix(prefix, m.RunID), "prepare/inputs/"+name) + if data := fake.Objects[key].Data; len(data) == 0 || strings.Contains(string(data), "imports:") { + t.Fatalf("published %s = %q", name, data) + } + } + setSessionConfigSearchPaths(t, []string{sessionPath}) + deps, _ := remoteSessionLoaderDependencies(fake) + loaded, err := loadCommandConfigWithDependencies(ctx, pipelinePath, "", campaignPath, "", config.SessionLoadOptions{SessionID: m.SessionID}, nil, deps) + if err != nil { + t.Fatal(err) + } + defer func() { _ = loaded.Close() }() + if loaded.Config.Session.Title != "Published title" || loaded.Config.SessionSource.S3Key != key { + t.Fatalf("remote session = %#v, source=%#v", loaded.Config.Session, loaded.Config.SessionSource) + } + if got := loaded.Config.StableInputs.SpeakersFile; got.Source != "campaign_config" || got.Path != filepath.Join(filepath.Dir(campaignPath), "speakers.yml") { + t.Fatalf("receiving campaign speakers = %#v", got) + } + current, err := discoverRemoteCurrentState(ctx, loaded.Config, fake) + if err != nil { + t.Fatal(err) + } + plan, err := buildRestorePlan(ctx, loaded.Config, current, fake, RestorePlanOptions{Force: true}) + if err != nil { + t.Fatal(err) + } + want := map[string]bool{"inputs/session.yml": false, "inputs/campaign.yml": false, "inputs/party.yml": false} + for _, action := range plan.Actions { + if _, ok := want[action.LocalRelativePath]; ok { + want[action.LocalRelativePath] = true + } + } + for path, found := range want { + if !found { + t.Fatalf("restore omitted %q", path) + } + } + paths := store.SessionPathsFor(m.Campaign, m.SessionID) + for _, name := range []string{"session.yml", "campaign.yml", "party.yml"} { + if err := os.Remove(filepath.Join(paths.InputsDir, name)); err != nil { + t.Fatal(err) + } + } + var output strings.Builder + if err := restoreWithDependencies(ctx, []string{m.SessionID, "--config", pipelinePath, "--campaign-file", campaignPath, "--force"}, &output, deps); err != nil { + t.Fatalf("restore: %v", err) + } + restored, err := os.ReadFile(filepath.Join(paths.InputsDir, "session.yml")) + if err != nil || string(restored) != string(fake.Objects[key].Data) { + t.Fatalf("restored session = %q, %v", restored, err) + } + object := fake.Objects[key] + modified := append([]byte(nil), object.Data...) + modified[0] ^= 1 + fake.SeedObject(storage.FakeObject{Key: key, Data: modified, ETag: object.ETag}) + if _, err := loadCommandConfigWithDependencies(ctx, pipelinePath, "", campaignPath, "", config.SessionLoadOptions{SessionID: m.SessionID}, nil, deps); err == nil || !strings.Contains(err.Error(), "committed checksum") { + t.Fatalf("changed remote session error = %v", err) + } +} diff --git a/internal/app/restore_plan.go b/internal/app/restore_plan.go index 1ccc5bb..f93fc54 100644 --- a/internal/app/restore_plan.go +++ b/internal/app/restore_plan.go @@ -179,6 +179,16 @@ func restoreLocalRelativePathForCommittedArtifact(sessionPrefix string, artifact if artifact.Type == artifacts.RemoteArtifactTypeSessionManifest { return config.PathManifestFile, true, nil } + if artifact.Type == artifacts.RemoteArtifactTypeRunFile { + switch artifact.Source { + case "run:prepare/inputs/session.yml", "run:prepare/inputs/campaign.yml", "run:prepare/inputs/party.yml": + identity, err := artifacts.ParseImmutableRunObjectKey(sessionPrefix, artifact.DestinationKey) + if err != nil || "run:"+identity.RelativePath != artifact.Source { + return "", false, fmt.Errorf("prepared input does not match its run object key") + } + return strings.TrimPrefix(identity.RelativePath, "prepare/"), true, nil + } + } if artifact.Type != artifacts.RemoteArtifactTypePublishedOutput { return "", false, nil } diff --git a/internal/stage/publish.go b/internal/stage/publish.go index 2ca0688..8080d9d 100644 --- a/internal/stage/publish.go +++ b/internal/stage/publish.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path/filepath" + "sort" "strings" "gitea.maximumdirect.net/eric/narratio/internal/adapters/storage" @@ -82,10 +83,27 @@ func (publishStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S return nil, fmt.Errorf("publish: run id is required") } + sessionPaths := publishSessionPaths(env, m) runFiles, err := collectPublishRunFiles(runRoot, m) if err != nil { return nil, fmt.Errorf("publish: collect run files: %w", err) } + preparedFiles, err := collectPublishPreparedInputs(env.ArtifactStore, sessionPaths, runRoot, m.Inputs) + if err != nil { + return nil, fmt.Errorf("publish: collect prepared inputs: %w", err) + } + byPath := make(map[string]publishUploadFile, len(runFiles)+len(preparedFiles)) + for _, file := range runFiles { + byPath[file.RelativePath] = file + } + for _, file := range preparedFiles { + byPath[file.RelativePath] = file + } + runFiles = runFiles[:0] + for _, file := range byPath { + runFiles = append(runFiles, file) + } + sort.Slice(runFiles, func(i, j int) bool { return runFiles[i].RelativePath < runFiles[j].RelativePath }) runSources, err := openPublishRunFiles(runRoot, runFiles) if err != nil { return nil, fmt.Errorf("publish: verify run files: %w", err) @@ -95,7 +113,6 @@ func (publishStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S if !ok { return nil, fmt.Errorf("publish: object store does not support verified source uploads") } - sessionPaths := publishSessionPaths(env, m) previousFiles, err := collectPublishPreviousFiles(ctx, sessionPaths.PreviousDir) if err != nil { return nil, fmt.Errorf("publish: collect previous files: %w", err) diff --git a/internal/stage/publish_sources.go b/internal/stage/publish_sources.go index 50e5c65..07c4825 100644 --- a/internal/stage/publish_sources.go +++ b/internal/stage/publish_sources.go @@ -12,6 +12,7 @@ import ( "sort" "strings" + "gitea.maximumdirect.net/eric/narratio/internal/artifacts" "gitea.maximumdirect.net/eric/narratio/internal/config" "gitea.maximumdirect.net/eric/narratio/internal/manifest" "gitea.maximumdirect.net/eric/narratio/internal/pathsafe" @@ -23,6 +24,46 @@ type publishUploadFile struct { Checksum string } +// collectPublishPreparedInputs copies the prepared documents into this run's +// archive, retaining their manifest checksums for the ordinary upload path. +func collectPublishPreparedInputs(store artifacts.Store, paths artifacts.SessionPaths, runRoot string, records []manifest.InputRecord) ([]publishUploadFile, error) { + filenames := map[string]string{"session_config": "session.yml", "campaign_config": "campaign.yml", "party": "party.yml"} + var result []publishUploadFile + for _, record := range records { + name, ok := filenames[record.Kind] + if !ok { + continue + } + rel, err := pathsafe.SlashRelativeFromRoot(paths.Root, record.Path) + if err != nil { + return nil, err + } + source, err := openPublishSourceUnderRoot(paths.Root, rel) + if err != nil { + return nil, err + } + if err := verifyPublishSourceChecksum(source, rel, record.Checksum); err != nil { + _ = source.Close() + return nil, err + } + data, err := io.ReadAll(source) + closeErr := source.Close() + if err != nil { + return nil, err + } + if closeErr != nil { + return nil, closeErr + } + archive := filepath.ToSlash(filepath.Join("prepare", "inputs", name)) + destination := filepath.Join(runRoot, filepath.FromSlash(archive)) + if _, err := writeBytesIfChanged(store, destination, data); err != nil { + return nil, err + } + result = append(result, publishUploadFile{RelativePath: archive, SourceRelativePath: archive, Checksum: record.Checksum}) + } + return result, nil +} + func collectPublishRunFiles(runRoot string, sessionManifest *manifest.Manifest) ([]publishUploadFile, error) { if sessionManifest == nil { return nil, fmt.Errorf("session manifest is required")