diff --git a/cmd/narratio/main.go b/cmd/narratio/main.go index 42c6fff..a1894f2 100644 --- a/cmd/narratio/main.go +++ b/cmd/narratio/main.go @@ -3,6 +3,8 @@ package main import ( "context" "os" + "os/signal" + "syscall" "go.maximumdirect.net/commandkit" @@ -10,6 +12,11 @@ import ( ) func main() { - outcome := app.Execute(context.Background(), os.Args[1:], commandkit.Streams{Out: os.Stdout, Err: os.Stderr}) - os.Exit(outcome.Status) + os.Exit(runProcess(os.Args[1:], commandkit.Streams{Out: os.Stdout, Err: os.Stderr}, app.Execute)) +} + +func runProcess(args []string, streams commandkit.Streams, execute func(context.Context, []string, commandkit.Streams) commandkit.Outcome) int { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + return execute(ctx, args, streams).Status } diff --git a/cmd/narratio/main_test.go b/cmd/narratio/main_test.go new file mode 100644 index 0000000..f464961 --- /dev/null +++ b/cmd/narratio/main_test.go @@ -0,0 +1,56 @@ +package main + +import ( + "bufio" + "context" + "fmt" + "os" + "os/exec" + "syscall" + "testing" + "time" + + "go.maximumdirect.net/commandkit" +) + +func TestProcessSignalsCancelInvocation(t *testing.T) { + if os.Getenv("NARRATIO_TEST_SIGNAL_CHILD") == "1" { + os.Exit(runProcess(nil, commandkit.Streams{Out: os.Stdout, Err: os.Stderr}, func(ctx context.Context, _ []string, streams commandkit.Streams) commandkit.Outcome { + fmt.Fprintln(streams.Out, "ready") + <-ctx.Done() + return commandkit.Failure("interrupted").WithCause(ctx.Err()) + })) + } + for _, sig := range []os.Signal{os.Interrupt, syscall.SIGTERM} { + t.Run(sig.String(), func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + child := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestProcessSignalsCancelInvocation$") + child.Env = append(os.Environ(), "NARRATIO_TEST_SIGNAL_CHILD=1") + out, err := child.StdoutPipe() + if err != nil { + t.Fatal(err) + } + child.Stderr = os.Stderr + if err := child.Start(); err != nil { + t.Fatal(err) + } + scanner := bufio.NewScanner(out) + if !scanner.Scan() || scanner.Text() != "ready" { + _ = child.Process.Kill() + _ = child.Wait() + t.Fatal("child failed to register its signal context") + } + if err := child.Process.Signal(sig); err != nil { + _ = child.Process.Kill() + _ = child.Wait() + t.Fatal(err) + } + err = child.Wait() + exit, ok := err.(*exec.ExitError) + if !ok || exit.ExitCode() != 1 { + t.Fatalf("child result=%v, want operational exit 1", err) + } + }) + } +} diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 3a3ef31..79ed6cc 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -644,6 +644,18 @@ save and lock-release failures and verify underlying causes remain inspectable. ## Stage 16: Propagate process cancellation and finalize resumable state +**Completed.** Interrupt/termination signals now cancel the native invocation. +Acquisition checks synchronous native loads, planning and scheduling check the +operation context, and running adapters/storage retain it. Transcription settles +workers and preserves returned shutdown/checkpoint causes alongside cancellation. +Terminal local manifest persistence and evidence readback detach cancellation +without detaching active work; acquired locks still release and completed speaker +and artifact checkpoints remain resumable. Restore preserves its incomplete marker +on cancellation, and initialization/cleanup stop before further local actions. +Deterministic process-signal, mid-transcription/analysis, failed-terminal-save, +retry, acquisition, runner/restore, adapter cancellation, focused race, package, +build, and diff checks pass with Go 1.27.4. + Read: process entry, runner lifecycle, HTTP/subprocess adapter cancellation, restore finalization, and CommandKit's invocation-context/outcome contracts. diff --git a/internal/app/clean_command.go b/internal/app/clean_command.go index e47eb8f..8ad645e 100644 --- a/internal/app/clean_command.go +++ b/internal/app/clean_command.go @@ -92,6 +92,9 @@ func cleanSelectedSession(ctx context.Context, input *cleanCommandInput, deps co return report, err } appendTarget := func(root, target, policy string, directory bool) error { + if err := ctx.Err(); err != nil { + return err + } action, err := cleanScopedTarget(root, target, policy, directory, input.DryRun) report.Actions = append(report.Actions, action) return err @@ -100,7 +103,7 @@ func cleanSelectedSession(ctx context.Context, input *cleanCommandInput, deps co if err := appendTarget(pipeline.Workspace.Root, filepath.Join(pipeline.Workspace.Root, config.PathWorkDirSegment), "clean.workspace.all", true); err != nil { return report, err } - actions, err := cleanRootChildren(pipeline.Spool.Root, "clean.spool.all", input.DryRun) + actions, err := cleanRootChildren(ctx, pipeline.Spool.Root, "clean.spool.all", input.DryRun) report.Actions = append(report.Actions, actions...) if err != nil { return report, err @@ -173,7 +176,7 @@ func cleanScopedTarget(root, target, policy string, directory, dryRun bool) (cle return action, nil } -func cleanRootChildren(root, policy string, dryRun bool) ([]cleanAction, error) { +func cleanRootChildren(ctx context.Context, root, policy string, dryRun bool) ([]cleanAction, error) { rootAbs, entries, err := cleanableRootChildren(root, policy) if err != nil { return []cleanAction{{Path: root, Status: "failed", Error: err.Error()}}, err @@ -183,6 +186,9 @@ func cleanRootChildren(root, policy string, dryRun bool) ([]cleanAction, error) } actions := make([]cleanAction, 0, len(entries)) for _, entry := range entries { + if err := ctx.Err(); err != nil { + return actions, err + } action := cleanAction{Path: entry, Status: "would_delete"} if !dryRun { if err := fileops.RemoveAllUnderRoot(rootAbs, entry); err != nil { @@ -213,6 +219,9 @@ func cleanSessionCache(ctx context.Context, cfg *config.Config, dryRun bool, dep } actions := []cleanAction{} for _, object := range objects { + if err := ctx.Err(); err != nil { + return actions, err + } key := strings.TrimSpace(object.Key) if key == "" || strings.HasSuffix(key, "/") || !cleanIsFlac(key) { continue diff --git a/internal/app/config_loader.go b/internal/app/config_loader.go index 6dabd9f..9ee1877 100644 --- a/internal/app/config_loader.go +++ b/internal/app/config_loader.go @@ -131,7 +131,11 @@ func loadRemoteSessionConfig(ctx context.Context, base *config.LoadedPipelineCam return nil, fmt.Errorf("remote session %q does not match its committed checksum or size", remoteKey) } } + if err := ctx.Err(); err != nil { + return nil, err + } sessionCfg, err := config.LoadSessionBytesWithOptions("s3://"+s3BucketName(base.Pipeline)+"/"+remoteKey, sessionBytes, sessionOpts) + err = errors.Join(err, ctx.Err()) if err != nil { return nil, err } diff --git a/internal/app/configuration_acquisition.go b/internal/app/configuration_acquisition.go index 5230010..b851b66 100644 --- a/internal/app/configuration_acquisition.go +++ b/internal/app/configuration_acquisition.go @@ -69,7 +69,7 @@ func (a *configurationAcquisition) bootstrap(ctx context.Context) error { a.deps.secretsBootstrap.selection = selectedString(a.selection.SecretsDir) a.deps.secretsBootstrap.selectionSet = true } - return a.deps.secretsBootstrap.load(a.deps.loadSecretsDir, a.deps.diagnostics) + return errors.Join(a.deps.secretsBootstrap.load(a.deps.loadSecretsDir, a.deps.diagnostics), ctx.Err()) } func (a *configurationAcquisition) Pipeline(ctx context.Context) (*config.PipelineConfig, error) { @@ -82,12 +82,10 @@ func (a *configurationAcquisition) Pipeline(ctx context.Context) (*config.Pipeli path, cfg, err := loadPipelineConfigWithDependencies(a.selection.Config.Value, config.PipelineLoadOptions{Overlay: selectedString(a.selection.Overlay), Overrides: a.pipelineOverrides}, selectedString(a.selection.SecretsDir), a.deps) + err = errors.Join(err, ctx.Err()) if err != nil { return nil, err } - if err := ctx.Err(); err != nil { - return nil, err - } a.pipeline, a.pipelinePath = cfg, path return cfg, nil } @@ -115,12 +113,10 @@ func (a *configurationAcquisition) Campaign(ctx context.Context) (*config.Campai return nil, err } cfg, err := config.LoadCampaign(path) + err = errors.Join(err, ctx.Err()) if err != nil { return nil, err } - if err := ctx.Err(); err != nil { - return nil, err - } if a.selection.Campaign.Set && config.CampaignID(cfg) != a.selection.Campaign.Value { return nil, &config.SessionIdentityError{Source: path, Field: "campaign_id", Expected: a.selection.Campaign.Value, Actual: config.CampaignID(cfg)} } @@ -144,12 +140,10 @@ func (a *configurationAcquisition) Party(ctx context.Context) (*config.ResolvedP } } party, err := config.ResolveParty(a.campaignPath, campaign, selectedString(a.selection.PartyFile)) + err = errors.Join(err, ctx.Err()) if err != nil { return nil, err } - if err := ctx.Err(); err != nil { - return nil, err - } a.party = &party return a.party, nil } @@ -175,16 +169,15 @@ func (a *configurationAcquisition) Session(ctx context.Context) (*config.Session return nil, err } session, err := config.LoadSessionWithOptions(path, opts) + err = errors.Join(err, ctx.Err()) if err != nil { return nil, err } - if err := ctx.Err(); err != nil { - return nil, err - } a.session = &acquiredSession{Config: session, Path: path, Source: config.SessionSource{Source: "session_config", LocalPath: path}} return session, nil } session, path, err := config.DiscoverSession(config.DefaultSessionConfigSearchPaths, opts) + err = errors.Join(err, ctx.Err()) if err != nil { return nil, err } @@ -204,6 +197,7 @@ func (a *configurationAcquisition) Session(ctx context.Context) (*config.Session } base.Pipeline, base.PipelinePath = pipeline, a.pipelinePath loaded, err := loadRemoteSessionConfig(ctx, base, opts, a.deps) + err = errors.Join(err, ctx.Err()) if err != nil { if loaded != nil && loaded.cleanup != nil { err = errors.Join(err, loaded.cleanup()) @@ -246,6 +240,7 @@ func (a *configurationAcquisition) Domain(ctx context.Context) (*config.Config, } base := config.LoadedPipelineCampaign{PipelinePath: a.pipelinePath, Pipeline: pipeline, CampaignPath: a.campaignPath, Campaign: campaign, Party: *party, ArtifactCatalog: catalog} cfg, err := config.ResolveLoadedPipelineCampaign(base, a.session.Path, session, a.session.Source) + err = errors.Join(err, ctx.Err()) if err != nil { return nil, err } @@ -279,6 +274,7 @@ func (a *configurationAcquisition) Catalog(ctx context.Context) (*config.Artifac party = *selected } catalog, err := config.DeriveArtifactCatalog(pipeline, party) + err = errors.Join(err, ctx.Err()) if err != nil { return nil, err } diff --git a/internal/app/configuration_acquisition_test.go b/internal/app/configuration_acquisition_test.go index 1ee0d44..dcc991f 100644 --- a/internal/app/configuration_acquisition_test.go +++ b/internal/app/configuration_acquisition_test.go @@ -196,3 +196,20 @@ func TestSessionAcquisitionPreservesRemoteFailureCause(t *testing.T) { t.Fatal(err) } } + +func TestConfigurationAcquisitionStopsAfterSynchronousCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + loadErr := errors.New("native configuration load failed") + acquired := newConfigurationAcquisition(configurationSelection{Config: suppliedSelection("synthetic.yaml")}, commandDependencies{loadPipelineConfig: func(string, config.PipelineLoadOptions) (*config.PipelineConfig, error) { + cancel() + return nil, loadErr + }}) + _, err := acquired.Pipeline(ctx) + if !errors.Is(err, context.Canceled) || !errors.Is(err, loadErr) || operationOutcome(err).Status != 1 { + t.Fatalf("lost canceled native load causes: %v", err) + } + if acquired.pipeline != nil { + t.Fatal("cached a canceled native load") + } +} diff --git a/internal/app/execution_command.go b/internal/app/execution_command.go index e28f1ab..02378d9 100644 --- a/internal/app/execution_command.go +++ b/internal/app/execution_command.go @@ -132,6 +132,9 @@ func executeSelectedPipeline(ctx context.Context, input *executionInput, deps co if err != nil { return nil, &inputError{message: err.Error(), cause: err} } + if err := ctx.Err(); err != nil { + return nil, err + } summary, err := executor(ctx, cfg, input.plan, RunOptions{Force: input.Force.Value, SelectedArtifacts: selected, EffectiveArtifacts: effective}) if summary == nil { if err == nil { diff --git a/internal/app/plan_command.go b/internal/app/plan_command.go index feb781b..9c309f4 100644 --- a/internal/app/plan_command.go +++ b/internal/app/plan_command.go @@ -93,6 +93,9 @@ func planSelectedPipeline(ctx context.Context, input *executionInput, deps comma } func buildSessionPlan(ctx context.Context, cfg *config.Config, input *executionInput, selectedArtifacts []string, effective artifacts.EffectiveArtifactSet) (*sessionPlanReport, error) { + if err := ctx.Err(); err != nil { + return nil, err + } m, err := loadManifestIfPresent(ctx, cfg) if err != nil { return nil, fmt.Errorf("plan: %w", err) @@ -114,6 +117,9 @@ func buildSessionPlan(ctx context.Context, cfg *config.Config, input *executionI report := &sessionPlanReport{Campaign: config.CampaignID(cfg.Campaign), SessionID: cfg.Session.SessionID, Stages: []plannedStage{}, Publication: []plannedPublication{}} for _, selectedStage := range stages { + if err := ctx.Err(); err != nil { + return report, err + } semanticConfig, err := currentStageSemanticConfig(selectedStage, stageEnv) if err != nil { return nil, fmt.Errorf("plan: fingerprint semantic configuration for stage %q: %w", selectedStage.Name(), err) @@ -176,5 +182,5 @@ func buildSessionPlan(ctx context.Context, cfg *config.Config, input *executionI report.Publication = append(report.Publication, plannedPublication{Source: action.Source, Destination: action.Destination, Required: action.Required, Selected: action.Selected}) } } - return report, nil + return report, ctx.Err() } diff --git a/internal/app/restore_command.go b/internal/app/restore_command.go index 29d5d21..a696092 100644 --- a/internal/app/restore_command.go +++ b/internal/app/restore_command.go @@ -129,6 +129,7 @@ func restoreSessionState(ctx context.Context, cfg *config.Config, opts RestorePl } result, err := deps.executeRestorePlan(ctx, cfg, current, plan, report, objectStore) + err = errors.Join(err, ctx.Err()) if err != nil { report.setFailed(err) if _, reportErr := persistRestoreReport(artifactStore, cfg, report); reportErr != nil { diff --git a/internal/app/restore_execute.go b/internal/app/restore_execute.go index 3403f26..8453d0b 100644 --- a/internal/app/restore_execute.go +++ b/internal/app/restore_execute.go @@ -33,6 +33,9 @@ func executeRestorePlan( report *RestoreReport, store storage.ObjectStore, ) (*RestoreExecutionResult, error) { + if err := ctx.Err(); err != nil { + return nil, err + } if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil { return nil, fmt.Errorf("resolved config with pipeline/session is required") } @@ -100,6 +103,9 @@ func executeRestoreDownloadAction( action RestoreAction, store storage.ObjectStore, ) error { + if err := ctx.Err(); err != nil { + return err + } safeLocalPath, err := joinWithinSessionRoot(sessionRoot, action.LocalRelativePath) if err != nil { return fmt.Errorf("resolve safe local path: %w", err) @@ -165,6 +171,9 @@ func executeRestoreDownloadAction( if err := validateRestoreDestinationLeaf(parent, leaf); err != nil { return err } + if err := ctx.Err(); err != nil { + return err + } if err := fileops.InstallSiblingTempFile(temporaryPath, safeLocalPath, fileops.WorkspaceFileMode); err != nil { return fmt.Errorf("install file atomically: %w", err) } diff --git a/internal/app/restore_plan.go b/internal/app/restore_plan.go index 8e47760..8176538 100644 --- a/internal/app/restore_plan.go +++ b/internal/app/restore_plan.go @@ -72,6 +72,9 @@ type RestorePlanOptions struct { } func buildRestorePlan(ctx context.Context, cfg *config.Config, current *RemoteCurrentState, store storage.ObjectStore, opts RestorePlanOptions) (*RestorePlan, error) { + if err := ctx.Err(); err != nil { + return nil, err + } if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil { return nil, fmt.Errorf("resolved config with pipeline/session is required") } diff --git a/internal/app/runner.go b/internal/app/runner.go index b894008..5571427 100644 --- a/internal/app/runner.go +++ b/internal/app/runner.go @@ -88,6 +88,9 @@ func executePlanWithDependencies(ctx context.Context, cfg *config.Config, plan B } for index, selected := range runtime.stages { + if err := ctx.Err(); err != nil { + return nil, err + } outcome, err := executeSelectedStage(runtime, selected) result := &runtime.stageResults[index] result.Attempted = outcome.executed @@ -112,7 +115,7 @@ func executePlanWithDependencies(ctx context.Context, cfg *config.Config, plan B return nil, err } } - return nil, nil + return nil, ctx.Err() } func stagesContainAny(stages []stage.Stage, names ...string) bool { @@ -147,7 +150,9 @@ func persistSessionTerminalFailure( sessionManifest.RecordFailure(at, operationErr.Error()) } - sessionErr := sessionStore.Save(ctx, sessionManifestPath, sessionManifest) + // Only terminal local persistence is detached; adapters and checkpoints keep + // the canceled operation context. + sessionErr := sessionStore.Save(context.WithoutCancel(ctx), sessionManifestPath, sessionManifest) return errors.Join( operationErr, wrapTerminalPersistenceError("save terminal session manifest", sessionErr), diff --git a/internal/app/runner_cancellation_test.go b/internal/app/runner_cancellation_test.go new file mode 100644 index 0000000..d4c25e3 --- /dev/null +++ b/internal/app/runner_cancellation_test.go @@ -0,0 +1,152 @@ +package app + +import ( + "context" + "errors" + "slices" + "testing" + + "gitea.maximumdirect.net/eric/narratio/internal/adapters/scriptorium" + "gitea.maximumdirect.net/eric/narratio/internal/adapters/whisperx" + "gitea.maximumdirect.net/eric/narratio/internal/artifacts" + "gitea.maximumdirect.net/eric/narratio/internal/config" + "gitea.maximumdirect.net/eric/narratio/internal/manifest" + "gitea.maximumdirect.net/eric/narratio/internal/stage" +) + +type cancellationTerminalStore struct { + manifest.Store + terminalCalls int + terminalErr error + value any +} + +func (s *cancellationTerminalStore) Save(ctx context.Context, path string, m *manifest.Manifest) error { + if m.LastError != nil { + s.terminalCalls++ + if ctx.Err() != nil || ctx.Value(cancellationContextKey{}) != s.value { + return errors.New("terminal save lost its detached invocation context") + } + if s.terminalErr != nil { + return s.terminalErr + } + } + return s.Store.Save(ctx, path, m) +} + +type cancellationContextKey struct{} + +func TestCancellationFinalizesCheckpointsAndStopsLaterWork(t *testing.T) { + for _, saveFails := range []bool{false, true} { + t.Run(map[bool]string{false: "saved", true: "save failure"}[saveFails], func(t *testing.T) { + cfg, base := setupTranscribeRecoveryFixture(t, []string{"a", "b", "c"}) + concurrency := 1 + cfg.Pipeline.WhisperX.Concurrency = &concurrency + ctx, cancel := context.WithCancel(context.WithValue(context.Background(), cancellationContextKey{}, "invocation-value")) + defer cancel() + saveErr := errors.New("terminal save unavailable") + shutdownErr := errors.New("adapter shutdown failed") + store := &cancellationTerminalStore{Store: base, value: "invocation-value"} + if saveFails { + store.terminalErr = saveErr + } + local := &releaseObservedStore{Store: artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root)} + fake := &whisperx.FakeClient{TranscribeFn: func(adapterCtx context.Context, request whisperx.TranscribeRequest) (whisperx.TranscribeResult, error) { + switch request.SpeakerID { + case "a": + return writeTranscribeRecoveryResult(request) + case "b": + cancel() + <-adapterCtx.Done() + return whisperx.TranscribeResult{}, errors.Join(adapterCtx.Err(), shutdownErr) + default: + t.Fatal("dispatched a speaker after cancellation") + return whisperx.TranscribeResult{}, nil + } + }} + later := 0 + summary, err := executeStages(ctx, cfg, []stage.Stage{mustSelectStage(t, "transcribe"), countingStage{name: "merge", runs: &later}, countingStage{name: "cleanup", runs: &later}}, RunOptions{Env: &Env{ManifestStore: store, ArtifactStore: local, WhisperX: fake}}) + if !errors.Is(err, context.Canceled) || !errors.Is(err, shutdownErr) || (saveFails && !errors.Is(err, saveErr)) { + t.Fatalf("lost cancellation/finalization cause: %v", err) + } + if operationOutcome(err).Status != 1 || summary == nil || summary.Complete || !local.released || store.terminalCalls != 1 || later != 0 { + t.Fatalf("summary=%#v released=%v saves=%d later=%d err=%v", summary, local.released, store.terminalCalls, later, err) + } + if summary.StageResults[0].Outcome != "failed" || summary.StageResults[1].Outcome != "not_attempted" || summary.StageResults[2].Outcome != "not_attempted" { + t.Fatalf("outcomes=%#v", summary.StageResults) + } + checkpointed := 0 + for _, item := range summary.AvailableArtifacts { + if item.Evidence == "speaker_checkpoint" { + checkpointed++ + } + } + if checkpointed != 1 { + t.Fatalf("durable checkpoint report=%#v", summary.AvailableArtifacts) + } + saved := loadTranscribeRecoveryManifest(t, base, cfg) + if !slices.Equal(transcribeSpeakerKeys(saved), []string{"a"}) { + t.Fatal("successful speaker checkpoint lost") + } + if !saveFails && (saved.Stages["transcribe"].Status != manifest.StatusFailed || saved.LastError == nil) { + t.Fatal("terminal manifest was not saved with cancelled context") + } + retry := &whisperx.FakeClient{} + resumed, err := executeStages(context.Background(), cfg, []stage.Stage{mustSelectStage(t, "transcribe")}, RunOptions{Env: &Env{ManifestStore: base, WhisperX: retry}}) + if err != nil || !resumed.Complete || !slices.Equal(requestedTranscribeSpeakers(retry), []string{"b", "c"}) { + t.Fatalf("retry=%#v requests=%v err=%v", resumed, requestedTranscribeSpeakers(retry), err) + } + }) + } +} + +type cancelingAnalysisRunner struct { + analyzeCheckpointRunner + cancel context.CancelFunc +} + +func (r *cancelingAnalysisRunner) RunArtifact(ctx context.Context, req scriptorium.RunArtifactRequest) (scriptorium.ArtifactResult, error) { + if len(r.Calls) == 1 { + r.cancel() + return scriptorium.ArtifactResult{}, ctx.Err() + } + return r.analyzeCheckpointRunner.RunArtifact(ctx, req) +} + +func TestCancellationRetainsArtifactCheckpointForResume(t *testing.T) { + cfg := testConfig(t) + definitions := map[string]config.ScriptoriumArtifactConfig{ + "alpha": {Enabled: true, PromptID: "alpha", OutputPath: "artifacts/alpha.md"}, + "beta": {Enabled: true, PromptID: "beta", OutputPath: "artifacts/beta.md"}, + "gamma": {Enabled: true, PromptID: "gamma", OutputPath: "artifacts/gamma.md"}, + } + cfg.Pipeline.Scriptorium = &config.ScriptoriumConfig{Artifacts: definitions} + cfg.ArtifactCatalog = &config.ArtifactCatalog{Artifacts: definitions} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + runner := &cancelingAnalysisRunner{cancel: cancel} + later := 0 + summary, err := executeStages(ctx, cfg, []stage.Stage{mustSelectStage(t, "analyze"), countingStage{name: "publish", runs: &later}, countingStage{name: "cleanup", runs: &later}}, RunOptions{Env: &Env{Scriptorium: runner}}) + if !errors.Is(err, context.Canceled) || summary == nil || summary.Complete || later != 0 { + t.Fatalf("summary=%#v err=%v later=%d", summary, err, later) + } + store := &manifest.LocalStore{} + durable := loadAnalyzeRecoveryManifest(t, store, cfg) + if durable.Stages["analyze"].Status != manifest.StatusFailed || durable.Stages["analyze"].AnalyzeArtifacts["alpha"].Status != manifest.AnalyzeArtifactCurrent { + t.Fatalf("checkpoint lost: %#v", durable.Stages["analyze"]) + } + checkpointed := 0 + for _, artifact := range summary.AvailableArtifacts { + if artifact.Evidence == "artifact_checkpoint" && artifact.Source == "narratio.artifact.alpha" { + checkpointed++ + } + } + if checkpointed != 1 { + t.Fatalf("partial artifacts=%#v", summary.AvailableArtifacts) + } + retry := &analyzeCheckpointRunner{} + resumed, err := executeStages(context.Background(), cfg, []stage.Stage{mustSelectStage(t, "analyze")}, RunOptions{Env: &Env{Scriptorium: retry}}) + if err != nil || !resumed.Complete || !slices.Equal(retry.Calls, []string{"beta", "gamma"}) { + t.Fatalf("resume=%#v calls=%v err=%v", resumed, retry.Calls, err) + } +} diff --git a/internal/app/runner_composition.go b/internal/app/runner_composition.go index 81c1c20..b4d1b7b 100644 --- a/internal/app/runner_composition.go +++ b/internal/app/runner_composition.go @@ -11,6 +11,9 @@ func composeInvocation(runtime *invocationRuntime) error { stages := runtime.stages dependencies := runtime.dependencies + if err := runtime.ctx.Err(); err != nil { + return err + } if env.WhisperX == nil && stagesContainAny(stages, "transcribe") { client, err := dependencies.newWhisperX(env.Config) if err != nil { @@ -18,6 +21,9 @@ func composeInvocation(runtime *invocationRuntime) error { } env.WhisperX = client } + if err := runtime.ctx.Err(); err != nil { + return err + } if env.Seriatim == nil && stagesContainAny(stages, "merge", "normalize", "trim", "render") { runner, err := dependencies.newSeriatim(env.Config) if err != nil { @@ -25,6 +31,9 @@ func composeInvocation(runtime *invocationRuntime) error { } env.Seriatim = runner } + if err := runtime.ctx.Err(); err != nil { + return err + } if env.Audita == nil && stagesContainAny(stages, "polish") { runner, err := dependencies.newAudita(env.Config) if err != nil { @@ -32,12 +41,21 @@ func composeInvocation(runtime *invocationRuntime) error { } env.Audita = runner } + if err := runtime.ctx.Err(); err != nil { + return err + } if env.Notarius == nil && needsNotariusForRun(env.Config, stages) { env.Notarius = dependencies.newNotarius() } + if err := runtime.ctx.Err(); err != nil { + return err + } if env.Scriptorium == nil && stagesContainAny(stages, "trim", "analyze") { env.Scriptorium = dependencies.newScriptorium() } + if err := runtime.ctx.Err(); err != nil { + return err + } if env.ObjectStore == nil && needsObjectStoreForRun(env.Config, stages, runtime.effectiveArtifacts) { objectStore, err := dependencies.newObjectStore(runtime.ctx, env.Config) if err != nil { @@ -45,8 +63,11 @@ func composeInvocation(runtime *invocationRuntime) error { } env.ObjectStore = objectStore } + if err := runtime.ctx.Err(); err != nil { + return err + } if env.Notifier == nil && stagesContainAny(stages, "notify") { env.Notifier = ¬ify.NoopSender{} } - return nil + return runtime.ctx.Err() } diff --git a/internal/app/runner_finalization.go b/internal/app/runner_finalization.go index fea528c..6dcfc75 100644 --- a/internal/app/runner_finalization.go +++ b/internal/app/runner_finalization.go @@ -1,6 +1,7 @@ package app import ( + "context" "errors" "fmt" "sort" @@ -34,7 +35,7 @@ func finalizeInvocation(runtime *invocationRuntime, operationErr error) (*RunSum // Read durable authority after a failure: checkpoint saves can succeed // even when terminal persistence fails. var err error - evidence, err = runtime.manifestStore.Load(runtime.ctx, runtime.manifestPath) + evidence, err = runtime.manifestStore.Load(context.WithoutCancel(runtime.ctx), runtime.manifestPath) if err != nil { resultErr = errors.Join(resultErr, fmt.Errorf("read durable run evidence: %w", err)) evidence = nil diff --git a/internal/app/runner_runtime.go b/internal/app/runner_runtime.go index 44ea364..495cbb2 100644 --- a/internal/app/runner_runtime.go +++ b/internal/app/runner_runtime.go @@ -37,6 +37,9 @@ type invocationRuntime struct { } func prepareInvocation(ctx context.Context, cfg *config.Config, plan BoundedPlan, opts RunOptions, dependencies invocationDependencies) (*invocationRuntime, error) { + if err := ctx.Err(); err != nil { + return nil, err + } var prerequisiteStore manifest.Store if opts.Env != nil { prerequisiteStore = opts.Env.ManifestStore @@ -125,6 +128,9 @@ func prepareInvocation(ctx context.Context, cfg *config.Config, plan BoundedPlan } func prepareInvocationLayout(runtime *invocationRuntime) error { + if err := runtime.ctx.Err(); err != nil { + return err + } paths, err := runtime.artifactStore.EnsureLayoutFor(runtime.identity.Campaign, runtime.identity.SessionID) if err != nil { return fmt.Errorf("prepare workdir: %w", err) diff --git a/internal/app/runner_stage_execution.go b/internal/app/runner_stage_execution.go index e128055..9a6e8de 100644 --- a/internal/app/runner_stage_execution.go +++ b/internal/app/runner_stage_execution.go @@ -26,6 +26,9 @@ func executeSelectedStage(runtime *invocationRuntime, selected stage.Stage) (out resultErr = errors.Join(resultErr, fmt.Errorf("invalidate dependents after stage %q failure: %w", selected.Name(), invalidationErr)) } }() + if err := runtime.ctx.Err(); err != nil { + return outcome, err + } env := runtime.env m := runtime.manifest env.Force = runtime.opts.Force @@ -54,6 +57,9 @@ func executeSelectedStage(runtime *invocationRuntime, selected stage.Stage) (out } } + if err := runtime.ctx.Err(); err != nil { + return outcome, err + } if decision.action == stageActionSkip { env.Logger.Info("skipping stage", "stage", selected.Name(), "reason", "already_succeeded", "force", runtime.opts.Force) return stageExecutionOutcome{skipped: true, reused: true, reason: "already_succeeded"}, nil @@ -75,7 +81,11 @@ func executeSelectedStage(runtime *invocationRuntime, selected stage.Stage) (out } env.Logger.Debug("manifest saved", "stage", selected.Name(), "transition", "running", "path", runtime.manifestPath) + if err := runtime.ctx.Err(); err != nil { + return outcome, err + } result, err := selected.Run(runtime.ctx, env, m) + err = errors.Join(err, runtime.ctx.Err()) if err == nil { err = validateStageResult(result) } diff --git a/internal/app/session_init_command.go b/internal/app/session_init_command.go index 9eb6814..516e062 100644 --- a/internal/app/session_init_command.go +++ b/internal/app/session_init_command.go @@ -149,6 +149,7 @@ func initializeSession(ctx context.Context, request *sessionInitializationInput, SessionID: sessionID, PreviousSessionID: previousSessionID, }) + err = errors.Join(err, ctx.Err()) if err != nil { return report, fmt.Errorf("session init: %w", err) } @@ -164,6 +165,9 @@ func initializeSession(ctx context.Context, request *sessionInitializationInput, return report, fmt.Errorf("session init: render effective session: %w", err) } + if err := ctx.Err(); err != nil { + return report, err + } if !remote { if err := writeLocalFile(output, data, force); err != nil { return report, fmt.Errorf("session init: %w", err) diff --git a/internal/stage/analyze.go b/internal/stage/analyze.go index 068e0a2..2c894cb 100644 --- a/internal/stage/analyze.go +++ b/internal/stage/analyze.go @@ -46,6 +46,9 @@ type analyzeExecutionContext struct { } func (analyzeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*StageResult, error) { + if err := ctx.Err(); err != nil { + return nil, err + } if env == nil || env.Config == nil { return nil, fmt.Errorf("analyze: stage environment config is required") } @@ -138,6 +141,9 @@ func (analyzeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S } for _, item := range workPlan.ExecutionOrder { + if err := ctx.Err(); err != nil { + return nil, err + } artifactCfg := env.Config.ConcreteArtifacts()[item.Key] fingerprint, _, _, err := computeAnalyzeArtifactFingerprint( item.Key, diff --git a/internal/stage/transcribe.go b/internal/stage/transcribe.go index a6bc549..d42c44f 100644 --- a/internal/stage/transcribe.go +++ b/internal/stage/transcribe.go @@ -3,6 +3,7 @@ package stage import ( "context" "encoding/json" + "errors" "fmt" "os" "path/filepath" @@ -86,6 +87,9 @@ func (transcribeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) group, groupCtx := errgroup.WithContext(ctx) group.SetLimit(concurrency) for _, job := range jobs { + if groupCtx.Err() != nil { + break + } job := job group.Go(func() error { if err := groupCtx.Err(); err != nil { @@ -116,13 +120,10 @@ func (transcribeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) return nil }) } - groupErr := group.Wait() - if err := state.checkpointFailure(); err != nil { - return nil, err - } + groupErr := errors.Join(group.Wait(), state.checkpointFailure()) completedCount := state.completedCount() if err := ctx.Err(); err != nil { - return nil, fmt.Errorf("transcribe: canceled after completing %d of %d audio jobs: %w", completedCount, len(jobs), err) + return nil, errors.Join(fmt.Errorf("transcribe: canceled after completing %d of %d audio jobs: %w", completedCount, len(jobs), err), groupErr) } if groupErr != nil { return nil, groupErr