diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index d06e354..3a3ef31 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -604,6 +604,18 @@ actions accurately instead of losing all result data. ## Stage 15: Preserve partial run results and terminal failures +**Completed.** Runner summaries retain the full selected plan, attempted/failed, +completed, reused/self-skipped, and unattempted outcomes, plus durable output and +speaker/artifact checkpoint evidence. One finalizer owns terminal persistence +and summary construction; failed completion/self-skip saves remain failures, +earlier successes survive, and original, persistence, and release causes stay +inspectable. Native reports follow resource release, honor quiet visibility, +and cannot undo completed work when their writer fails. Runner, checkpoint, +retry, composition, native-report, focused race, configuration/stage/artifact, +build, and diff checks pass with Go 1.27.4. Updated an existing publication test +fixture to supply its separately derived publication rules; remaining legacy +CLI conversions stay assigned to final cleanup. + Read: `runner.go`, `runner_stage_execution.go`, `runner_runtime.go`, `runner_finalization.go`, `run_control.go`, and manifest checkpoint contracts. diff --git a/internal/app/execution_command.go b/internal/app/execution_command.go index 3e90c21..e28f1ab 100644 --- a/internal/app/execution_command.go +++ b/internal/app/execution_command.go @@ -20,14 +20,16 @@ type executionInput struct { } type runReport struct { - Campaign string `json:"campaign"` - SessionID string `json:"session_id"` - RunID string `json:"run_id"` - Stages []string `json:"stages"` - Executed []string `json:"executed"` - Skipped []string `json:"skipped"` - ArtifactTargets []string `json:"artifact_targets"` - Complete bool `json:"complete"` + Campaign string `json:"campaign"` + SessionID string `json:"session_id"` + RunID string `json:"run_id"` + Stages []string `json:"stages"` + Executed []string `json:"executed"` + Skipped []string `json:"skipped"` + ArtifactTargets []string `json:"artifact_targets"` + StageResults []RunStageResult `json:"stage_results"` + AvailableArtifacts []RunArtifact `json:"available_artifacts"` + Complete bool `json:"complete"` } func checkExecutionInput(input *executionInput) error { @@ -137,5 +139,5 @@ func executeSelectedPipeline(ctx context.Context, input *executionInput, deps co } return nil, err } - return &runReport{Campaign: config.CampaignID(cfg.Campaign), SessionID: summary.SessionID, RunID: summary.RunID, Stages: input.plan.Names(), Executed: summary.Executed, Skipped: summary.Skipped, ArtifactTargets: effective.Keys(), Complete: err == nil}, err + return &runReport{Campaign: config.CampaignID(cfg.Campaign), SessionID: summary.SessionID, RunID: summary.RunID, Stages: input.plan.Names(), Executed: summary.Executed, Skipped: summary.Skipped, ArtifactTargets: effective.Keys(), StageResults: summary.StageResults, AvailableArtifacts: summary.AvailableArtifacts, Complete: err == nil}, err } diff --git a/internal/app/execution_command_test.go b/internal/app/execution_command_test.go index 013875f..f0ac174 100644 --- a/internal/app/execution_command_test.go +++ b/internal/app/execution_command_test.go @@ -4,7 +4,12 @@ import ( "bytes" "context" "encoding/json" + "errors" + "gitea.maximumdirect.net/eric/narratio/internal/artifacts" + "gitea.maximumdirect.net/eric/narratio/internal/manifest" + "gitea.maximumdirect.net/eric/narratio/internal/stage" "reflect" + "strings" "testing" "gitea.maximumdirect.net/eric/narratio/internal/config" @@ -101,3 +106,102 @@ func TestExecutionCommandsRejectInvalidSelectionBeforeBootstrap(t *testing.T) { } } } + +type releaseObservedStore struct { + artifacts.Store + released bool +} + +func (s *releaseObservedStore) ReleaseSessionLock(lock *artifacts.LockHandle) error { + err := s.Store.ReleaseSessionLock(lock) + s.released = true + return err +} + +type failingRunReportWriter struct { + t *testing.T + store *releaseObservedStore + err error +} + +func (w failingRunReportWriter) Write([]byte) (int, error) { + if !w.store.released { + w.t.Fatal("report written before lock release") + } + return 0, w.err +} + +func TestExecutionCommandReportFailurePreservesCompletedWork(t *testing.T) { + pipeline, campaign, session := writeValidConfigFiles(t, t.TempDir()) + writeErr := errors.New("report destination unavailable") + var saved *RunSummary + var local *releaseObservedStore + app, err := newApplication(commandDependencies{}, func(ctx context.Context, cfg *config.Config, _ BoundedPlan, opts RunOptions) (*RunSummary, error) { + local = &releaseObservedStore{Store: artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root)} + opts.Env = &Env{ArtifactStore: local} + var err error + saved, err = executeStages(ctx, cfg, []stage.Stage{resultStage{name: "prepare"}}, opts) + return saved, err + }) + if err != nil { + t.Fatal(err) + } + // Defer resolving the store until Report invokes Write. + writer := runReportWriteFunc(func(p []byte) (int, error) { + return (failingRunReportWriter{t: t, store: local, err: writeErr}).Write(p) + }) + var diagnostic bytes.Buffer + outcome := app.Execute(context.Background(), []string{"run", "2026-05-03", "--stage", "prepare", "--config", pipeline, "--campaign-file", campaign, "--session-file", session, "--format", "json"}, commandkit.Streams{Out: writer, Err: &diagnostic}) + if outcome.Status != 1 || !errors.Is(outcome.Cause, writeErr) { + t.Fatalf("outcome=%#v diagnostics=%s", outcome, diagnostic.String()) + } + if saved == nil || !saved.Complete { + t.Fatalf("completed domain result=%#v", saved) + } + durable, err := (&manifest.LocalStore{}).Load(context.Background(), saved.ManifestPath) + if err != nil { + t.Fatal(err) + } + if durable.Stages["prepare"].Status != manifest.StatusSucceeded || durable.LastError != nil { + t.Fatalf("report failure changed domain state: %#v", durable) + } +} + +type runReportWriteFunc func([]byte) (int, error) + +func (f runReportWriteFunc) Write(p []byte) (int, error) { return f(p) } + +func TestExecutionCommandPresentsPartialFailureThroughNativeVisibility(t *testing.T) { + pipeline, campaign, session := writeValidConfigFiles(t, t.TempDir()) + cause := errors.New("synthetic terminal operation failure") + app, err := newApplication(commandDependencies{}, func(_ context.Context, cfg *config.Config, _ BoundedPlan, _ RunOptions) (*RunSummary, error) { + return &RunSummary{SessionID: cfg.Session.SessionID, RunID: "synthetic-run", Executed: []string{"prepare", "transcribe"}, StageResults: []RunStageResult{{Name: "prepare", Attempted: true, Outcome: "completed"}, {Name: "transcribe", Attempted: true, Outcome: "failed"}}}, cause + }) + if err != nil { + t.Fatal(err) + } + for _, quiet := range []bool{false, true} { + args := []string{"run", "2026-05-03", "--config", pipeline, "--campaign-file", campaign, "--session-file", session, "--format", "json"} + if quiet { + args = append(args, "--quiet") + } + var out, diagnostic bytes.Buffer + outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &out, Err: &diagnostic}) + if outcome.Status != 1 || !errors.Is(outcome.Cause, cause) || strings.Count(diagnostic.String(), cause.Error()) != 1 { + t.Fatalf("outcome=%#v diagnostics=%s", outcome, diagnostic.String()) + } + if quiet { + if out.Len() != 0 { + t.Fatal("quiet emitted a report") + } + continue + } + var report runReport + if err := json.Unmarshal(out.Bytes(), &report); err != nil { + t.Fatal(err) + } + if report.Complete || len(report.StageResults) != 2 || report.StageResults[1].Outcome != "failed" { + t.Fatalf("report=%#v", report) + } + } +} diff --git a/internal/app/runner.go b/internal/app/runner.go index 90e11a7..b894008 100644 --- a/internal/app/runner.go +++ b/internal/app/runner.go @@ -26,12 +26,15 @@ type RunOptions struct { } type RunSummary struct { - SessionID string - RunID string - ManifestPath string - StageNames []string - Executed []string - Skipped []string + SessionID string + RunID string + ManifestPath string + StageNames []string + Executed []string + Skipped []string + StageResults []RunStageResult + AvailableArtifacts []RunArtifact + Complete bool } type pipelineExecutor func(context.Context, *config.Config, BoundedPlan, RunOptions) (*RunSummary, error) @@ -62,7 +65,20 @@ func executePlanWithDependencies(ctx context.Context, cfg *config.Config, plan B resultErr = errors.Join(resultErr, fmt.Errorf("release session lock: %w", releaseErr)) } } + if summary != nil { + summary.Complete = resultErr == nil + } }() + defer func() { + if runtime.started { + summary, resultErr = finalizeInvocation(runtime, resultErr) + } + }() + + for _, selected := range runtime.stages { + runtime.stageNames = append(runtime.stageNames, selected.Name()) + runtime.stageResults = append(runtime.stageResults, RunStageResult{Name: selected.Name(), Outcome: "not_attempted"}) + } if err := initializeLockedInvocation(runtime); err != nil { return nil, err @@ -71,20 +87,32 @@ func executePlanWithDependencies(ctx context.Context, cfg *config.Config, plan B return nil, err } - for _, selected := range runtime.stages { - runtime.stageNames = append(runtime.stageNames, selected.Name()) + for index, selected := range runtime.stages { outcome, err := executeSelectedStage(runtime, selected) - if err != nil { - return nil, err + result := &runtime.stageResults[index] + result.Attempted = outcome.executed + result.Reason = outcome.reason + switch { + case err != nil: + result.Outcome = "failed" + case outcome.reused: + result.Outcome = "reused" + case outcome.skipped: + result.Outcome = "skipped" + default: + result.Outcome = "completed" } if outcome.executed { runtime.executed = append(runtime.executed, selected.Name()) } - if outcome.skipped { + if outcome.skipped && err == nil { runtime.skipped = append(runtime.skipped, selected.Name()) } + if err != nil { + return nil, err + } } - return finalizeInvocation(runtime) + return nil, nil } func stagesContainAny(stages []stage.Stage, names ...string) bool { diff --git a/internal/app/runner_composition.go b/internal/app/runner_composition.go index 627e64b..81c1c20 100644 --- a/internal/app/runner_composition.go +++ b/internal/app/runner_composition.go @@ -14,21 +14,21 @@ func composeInvocation(runtime *invocationRuntime) error { if env.WhisperX == nil && stagesContainAny(stages, "transcribe") { client, err := dependencies.newWhisperX(env.Config) if err != nil { - return runtime.persistTerminalFailure(fmt.Errorf("initialize whisperx client: %w", err)) + return fmt.Errorf("initialize whisperx client: %w", err) } env.WhisperX = client } if env.Seriatim == nil && stagesContainAny(stages, "merge", "normalize", "trim", "render") { runner, err := dependencies.newSeriatim(env.Config) if err != nil { - return runtime.persistTerminalFailure(fmt.Errorf("initialize seriatim runner: %w", err)) + return fmt.Errorf("initialize seriatim runner: %w", err) } env.Seriatim = runner } if env.Audita == nil && stagesContainAny(stages, "polish") { runner, err := dependencies.newAudita(env.Config) if err != nil { - return runtime.persistTerminalFailure(fmt.Errorf("initialize audita runner: %w", err)) + return fmt.Errorf("initialize audita runner: %w", err) } env.Audita = runner } @@ -41,7 +41,7 @@ func composeInvocation(runtime *invocationRuntime) error { if env.ObjectStore == nil && needsObjectStoreForRun(env.Config, stages, runtime.effectiveArtifacts) { objectStore, err := dependencies.newObjectStore(runtime.ctx, env.Config) if err != nil { - return runtime.persistTerminalFailure(err) + return err } env.ObjectStore = objectStore } diff --git a/internal/app/runner_finalization.go b/internal/app/runner_finalization.go index 41e8216..fea528c 100644 --- a/internal/app/runner_finalization.go +++ b/internal/app/runner_finalization.go @@ -1,12 +1,106 @@ package app -func finalizeInvocation(runtime *invocationRuntime) (*RunSummary, error) { - return &RunSummary{ - SessionID: runtime.identity.SessionID, - RunID: runtime.identity.RunID, - ManifestPath: runtime.manifestPath, - StageNames: runtime.stageNames, - Executed: runtime.executed, - Skipped: runtime.skipped, - }, nil +import ( + "errors" + "fmt" + "sort" + + "gitea.maximumdirect.net/eric/narratio/internal/manifest" +) + +// RunStageResult describes this invocation's work, independently of prior state. +type RunStageResult struct { + Name string `json:"name"` + Attempted bool `json:"attempted"` + Outcome string `json:"outcome"` + Reason string `json:"reason,omitempty"` +} + +// RunArtifact identifies durable output evidence without exposing manifest internals. +type RunArtifact struct { + Stage string `json:"stage"` + Source string `json:"source,omitempty"` + Kind string `json:"kind"` + Path string `json:"path"` + ProducerRunID string `json:"producer_run_id,omitempty"` + Evidence string `json:"evidence"` +} + +func finalizeInvocation(runtime *invocationRuntime, operationErr error) (*RunSummary, error) { + evidence := runtime.manifest + resultErr := operationErr + if operationErr != nil { + resultErr = persistSessionTerminalFailure(runtime.ctx, runtime.manifestStore, runtime.manifestPath, runtime.manifest, operationErr, runtime.dependencies.now()) + // Read durable authority after a failure: checkpoint saves can succeed + // even when terminal persistence fails. + var err error + evidence, err = runtime.manifestStore.Load(runtime.ctx, runtime.manifestPath) + if err != nil { + resultErr = errors.Join(resultErr, fmt.Errorf("read durable run evidence: %w", err)) + evidence = nil + } + } + return &RunSummary{ + SessionID: runtime.identity.SessionID, RunID: runtime.identity.RunID, + ManifestPath: runtime.manifestPath, StageNames: runtime.stageNames, + Executed: runtime.executed, Skipped: runtime.skipped, + StageResults: runtime.stageResults, AvailableArtifacts: durableRunArtifacts(evidence), + Complete: resultErr == nil, + }, resultErr +} + +func durableRunArtifacts(m *manifest.Manifest) []RunArtifact { + out := []RunArtifact{} + if m == nil { + return out + } + names := make([]string, 0, len(m.Stages)) + for name := range m.Stages { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + record := m.Stages[name] + if record == nil { + continue + } + seen := map[string]bool{} + add := func(value manifest.ArtifactRecord, evidence string) { + key := value.SourceID + "\x00" + value.LocalPath + if seen[key] { + return + } + seen[key] = true + out = append(out, RunArtifact{Stage: name, Source: value.SourceID, Kind: value.Kind, Path: value.LocalPath, ProducerRunID: value.ProducerRunID, Evidence: evidence}) + } + if record.Status == manifest.StatusSucceeded { + for _, value := range record.Outputs { + add(value, "stage_result") + } + } + if record.HasVersionedTranscribeState() { + keys := make([]string, 0, len(record.TranscribeSpeakers)) + for key := range record.TranscribeSpeakers { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + add(record.TranscribeSpeakers[key].Output, "speaker_checkpoint") + } + } + if record.HasVersionedAnalyzeState() { + keys := make([]string, 0, len(record.AnalyzeArtifacts)) + for key := range record.AnalyzeArtifacts { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + value := record.AnalyzeArtifacts[key] + if value.Status == manifest.AnalyzeArtifactCurrent && value.Output != nil { + add(*value.Output, "artifact_checkpoint") + } + } + } + } + return out } diff --git a/internal/app/runner_partial_result_test.go b/internal/app/runner_partial_result_test.go new file mode 100644 index 0000000..225686c --- /dev/null +++ b/internal/app/runner_partial_result_test.go @@ -0,0 +1,94 @@ +package app + +import ( + "context" + "errors" + "testing" + + "gitea.maximumdirect.net/eric/narratio/internal/adapters/whisperx" + "gitea.maximumdirect.net/eric/narratio/internal/artifacts" + "gitea.maximumdirect.net/eric/narratio/internal/config" + "gitea.maximumdirect.net/eric/narratio/internal/manifest" + "gitea.maximumdirect.net/eric/narratio/internal/stage" +) + +type terminalCountingStore struct { + manifest.Store + terminalSaves int + terminalErr error +} + +func (s *terminalCountingStore) Save(ctx context.Context, path string, m *manifest.Manifest) error { + if m.LastError != nil { + s.terminalSaves++ + if s.terminalErr != nil { + return s.terminalErr + } + } + return s.Store.Save(ctx, path, m) +} + +func TestPartialRunRetainsAllFailureCausesAndDurableSuccess(t *testing.T) { + for _, compositionFailure := range []bool{false, true} { + for _, saveFails := range []bool{false, true} { + t.Run(map[bool]string{false: "stage", true: "composition"}[compositionFailure]+map[bool]string{false: " saved", true: " save failed"}[saveFails], func(t *testing.T) { + cfg := testConfig(t) + cause := errors.New("operation unavailable") + saveErr := errors.New("terminal persistence unavailable") + releaseErr := errors.New("lock release failed") + store := &terminalCountingStore{Store: &manifest.LocalStore{}} + if saveFails { + store.terminalErr = saveErr + } + local := &releaseErrorArtifactStore{Store: artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root), err: releaseErr} + firstRuns := 0 + stages := []stage.Stage{countingStage{name: "prepare", runs: &firstRuns}, failingStage{name: "transcribe", err: cause}, countingStage{name: "merge", runs: new(int)}} + deps := invocationDependencies{newWhisperX: func(*config.Config) (whisperx.Client, error) { + if compositionFailure { + return nil, cause + } + return &whisperx.NoopClient{}, nil + }} + summary, err := executePlanWithDependencies(context.Background(), cfg, BoundedPlan{stages: stages}, RunOptions{Env: &Env{ManifestStore: store, ArtifactStore: local}}, deps) + if !errors.Is(err, cause) || !errors.Is(err, releaseErr) || (saveFails && !errors.Is(err, saveErr)) { + t.Fatalf("lost causes: %v", err) + } + if store.terminalSaves != 1 { + t.Fatalf("terminal saves=%d", store.terminalSaves) + } + if summary == nil || summary.Complete || len(summary.StageResults) != 3 { + t.Fatalf("summary=%#v", summary) + } + if compositionFailure { + for _, result := range summary.StageResults { + if result.Attempted || result.Outcome != "not_attempted" { + t.Fatalf("composition scheduled work: %#v", summary) + } + } + return + } + if firstRuns != 1 || summary.StageResults[0].Outcome != "completed" || summary.StageResults[1].Outcome != "failed" || !summary.StageResults[1].Attempted || summary.StageResults[2].Outcome != "not_attempted" { + t.Fatalf("partial results=%#v", summary) + } + durable, err := store.Load(context.Background(), summary.ManifestPath) + if err != nil { + t.Fatal(err) + } + if durable.Stages["prepare"].Status != manifest.StatusSucceeded { + t.Fatal("earlier success lost") + } + if saveFails { + if durable.Stages["transcribe"].Status != manifest.StatusRunning { + t.Fatal("failed terminal save advertised durable failure") + } + } else if durable.Stages["transcribe"].Status != manifest.StatusFailed { + t.Fatal("failed attempt not persisted") + } + retry, err := executeStages(context.Background(), cfg, []stage.Stage{countingStage{name: "prepare", runs: &firstRuns}, countingStage{name: "transcribe", runs: new(int)}}, RunOptions{}) + if err != nil || firstRuns != 1 || retry.StageResults[0].Outcome != "reused" || !retry.Complete { + t.Fatalf("retry=%#v runs=%d err=%v", retry, firstRuns, err) + } + }) + } + } +} diff --git a/internal/app/runner_runtime.go b/internal/app/runner_runtime.go index 915cfe0..44ea364 100644 --- a/internal/app/runner_runtime.go +++ b/internal/app/runner_runtime.go @@ -28,6 +28,8 @@ type invocationRuntime struct { artifactStore artifacts.Store manifestPath string manifest *manifest.Manifest + started bool + stageResults []RunStageResult stageNames []string executed []string skipped []string @@ -159,6 +161,7 @@ func initializeLockedInvocation(runtime *invocationRuntime) error { if err := validateBoundedPrerequisites(runtime.plan, m); err != nil { return fmt.Errorf("validate bounded run prerequisites under session lock: %w", err) } + runtime.started = true runtime.identity.applyToSessionManifest(m) applyEffectiveConfigProvenance(m, runtime.cfg) if err := runtime.manifestStore.Save(runtime.ctx, runtime.manifestPath, m); err != nil { @@ -176,14 +179,3 @@ func (runtime *invocationRuntime) ensureRunWorkspaceFor(selected stage.Stage) er } return nil } - -func (runtime *invocationRuntime) persistTerminalFailure(err error) error { - return persistSessionTerminalFailure( - runtime.ctx, - runtime.manifestStore, - runtime.manifestPath, - runtime.manifest, - err, - runtime.dependencies.now(), - ) -} diff --git a/internal/app/runner_stage_execution.go b/internal/app/runner_stage_execution.go index d126704..e128055 100644 --- a/internal/app/runner_stage_execution.go +++ b/internal/app/runner_stage_execution.go @@ -10,40 +10,45 @@ import ( type stageExecutionOutcome struct { executed bool skipped bool + reused bool + reason string } -func executeSelectedStage(runtime *invocationRuntime, selected stage.Stage) (stageExecutionOutcome, error) { +func executeSelectedStage(runtime *invocationRuntime, selected stage.Stage) (outcome stageExecutionOutcome, resultErr error) { + defer func() { + if resultErr == nil || !outcome.executed { + return + } + failedAt := runtime.dependencies.now() + runtime.manifest.MarkStageFailed(selected.Name(), failedAt, resultErr.Error()) + _, invalidationErr := invalidateDependentSucceededStagesWithReason(runtime.manifest, selected.Name(), failedAt, staleReasonFailure) + if invalidationErr != nil { + resultErr = errors.Join(resultErr, fmt.Errorf("invalidate dependents after stage %q failure: %w", selected.Name(), invalidationErr)) + } + }() env := runtime.env m := runtime.manifest env.Force = runtime.opts.Force semanticConfig, err := currentStageSemanticConfig(selected, env) if err != nil { - return stageExecutionOutcome{}, runtime.persistTerminalFailure( - fmt.Errorf("fingerprint semantic configuration for stage %q: %w", selected.Name(), err), - ) + return outcome, fmt.Errorf("fingerprint semantic configuration for stage %q: %w", selected.Name(), err) } decision := decideStageExecution(recordedStageStatus(m, selected.Name()), runtime.opts.Force, nil) if decision.action == stageActionSkip { validation, err := evaluateStageResume(runtime.ctx, selected, env, m, semanticConfig) if err != nil { - return stageExecutionOutcome{}, runtime.persistTerminalFailure( - fmt.Errorf("validate resume for stage %q: %w", selected.Name(), err), - ) + return outcome, fmt.Errorf("validate resume for stage %q: %w", selected.Name(), err) } decision = decideStageExecution(recordedStageStatus(m, selected.Name()), runtime.opts.Force, validation) if decision.markSelectedStale { staleAt := runtime.dependencies.now() if _, err := applyPreExecutionDecision(m, selected.Name(), decision, staleAt); err != nil { - return stageExecutionOutcome{}, runtime.persistTerminalFailure( - fmt.Errorf("invalidate dependents after resume validation for stage %q: %w", selected.Name(), err), - ) + return outcome, fmt.Errorf("invalidate dependents after resume validation for stage %q: %w", selected.Name(), err) } if err := runtime.manifestStore.Save(runtime.ctx, runtime.manifestPath, m); err != nil { - return stageExecutionOutcome{}, runtime.persistTerminalFailure( - fmt.Errorf("save manifest after resume validation for stage %q: %w", selected.Name(), err), - ) + return outcome, fmt.Errorf("save manifest after resume validation for stage %q: %w", selected.Name(), err) } env.Logger.Info("stage result is not resumable", "stage", selected.Name(), "reason", decision.selectedStaleReason) } @@ -51,28 +56,22 @@ func executeSelectedStage(runtime *invocationRuntime, selected stage.Stage) (sta if decision.action == stageActionSkip { env.Logger.Info("skipping stage", "stage", selected.Name(), "reason", "already_succeeded", "force", runtime.opts.Force) - return stageExecutionOutcome{skipped: true}, nil + return stageExecutionOutcome{skipped: true, reused: true, reason: "already_succeeded"}, nil } if err := runtime.ensureRunWorkspaceFor(selected); err != nil { - return stageExecutionOutcome{}, runtime.persistTerminalFailure(err) + return outcome, err } priorOutcome := capturePriorStageOutcome(m, selected.Name()) - attempted := stageExecutionOutcome{executed: true} + outcome.executed = true startedAt := runtime.dependencies.now() if _, err := beginStageAttempt(m, selected.Name(), decision, startedAt); err != nil { - return stageExecutionOutcome{}, runtime.persistTerminalFailure( - fmt.Errorf("invalidate dependents before forced stage %q: %w", selected.Name(), err), - ) + return outcome, fmt.Errorf("invalidate dependents before forced stage %q: %w", selected.Name(), err) } env.Logger.Info("starting stage", "stage", selected.Name()) if err := runtime.manifestStore.Save(runtime.ctx, runtime.manifestPath, m); err != nil { operationErr := fmt.Errorf("save manifest before stage %q: %w", selected.Name(), err) - m.MarkStageFailed(selected.Name(), runtime.dependencies.now(), operationErr.Error()) - if _, invalidationErr := invalidateDependentSucceededStagesWithReason(m, selected.Name(), runtime.dependencies.now(), staleReasonFailure); invalidationErr != nil { - operationErr = errors.Join(operationErr, fmt.Errorf("invalidate dependents after stage %q persistence failure: %w", selected.Name(), invalidationErr)) - } - return stageExecutionOutcome{}, runtime.persistTerminalFailure(operationErr) + return outcome, operationErr } env.Logger.Debug("manifest saved", "stage", selected.Name(), "transition", "running", "path", runtime.manifestPath) @@ -81,49 +80,33 @@ func executeSelectedStage(runtime *invocationRuntime, selected stage.Stage) (sta err = validateStageResult(result) } if err != nil { - failedAt := runtime.dependencies.now() - m.MarkStageFailed(selected.Name(), failedAt, err.Error()) - if _, invalidationErr := invalidateDependentSucceededStagesWithReason(m, selected.Name(), failedAt, staleReasonFailure); invalidationErr != nil { - err = errors.Join(err, fmt.Errorf("invalidate dependents after stage %q failure: %w", selected.Name(), invalidationErr)) - } - env.Logger.Info("stage failed", "stage", selected.Name(), "error", err) - return stageExecutionOutcome{}, runtime.persistTerminalFailure(fmt.Errorf("stage %q failed: %w", selected.Name(), err)) + return outcome, fmt.Errorf("stage %q failed: %w", selected.Name(), err) } if result != nil && result.Disposition == stage.StageDispositionSkipped { skippedAt := runtime.dependencies.now() if _, err := applyCompletedStageOutcome(m, selected.Name(), result, nil, semanticConfig, priorOutcome, skippedAt); err != nil { - return stageExecutionOutcome{}, runtime.persistTerminalFailure( - fmt.Errorf("invalidate dependents after stage %q self-skip: %w", selected.Name(), err), - ) + return outcome, fmt.Errorf("invalidate dependents after stage %q self-skip: %w", selected.Name(), err) } if err := runtime.manifestStore.Save(runtime.ctx, runtime.manifestPath, m); err != nil { - return stageExecutionOutcome{}, runtime.persistTerminalFailure( - fmt.Errorf("save manifest after self-skip %q: %w", selected.Name(), err), - ) + return outcome, fmt.Errorf("save manifest after self-skip %q: %w", selected.Name(), err) } env.Logger.Debug("manifest saved", "stage", selected.Name(), "transition", "skipped", "path", runtime.manifestPath) env.Logger.Info("stage skipped", "stage", selected.Name(), "reason", result.SkipReason) - attempted.skipped = true - return attempted, nil + outcome.skipped = true + outcome.reason = result.SkipReason + return outcome, nil } outputs := mapResultOutputs(selected.Name(), result, runtime.identity.RunID) succeededAt := runtime.dependencies.now() if _, err := applyCompletedStageOutcome(m, selected.Name(), result, outputs, semanticConfig, priorOutcome, succeededAt); err != nil { - return stageExecutionOutcome{}, runtime.persistTerminalFailure( - fmt.Errorf("invalidate dependents after changed stage %q result: %w", selected.Name(), err), - ) + return outcome, fmt.Errorf("invalidate dependents after changed stage %q result: %w", selected.Name(), err) } if err := runtime.manifestStore.Save(runtime.ctx, runtime.manifestPath, m); err != nil { operationErr := fmt.Errorf("save manifest after stage %q: %w", selected.Name(), err) - failedAt := runtime.dependencies.now() - m.MarkStageFailed(selected.Name(), failedAt, operationErr.Error()) - if _, invalidationErr := invalidateDependentSucceededStagesWithReason(m, selected.Name(), failedAt, staleReasonFailure); invalidationErr != nil { - operationErr = errors.Join(operationErr, fmt.Errorf("invalidate dependents after stage %q persistence failure: %w", selected.Name(), invalidationErr)) - } - return stageExecutionOutcome{}, runtime.persistTerminalFailure(operationErr) + return outcome, operationErr } env.Logger.Debug("manifest saved", "stage", selected.Name(), "transition", "succeeded", "path", runtime.manifestPath) env.Logger.Info("stage succeeded", "stage", selected.Name()) - return attempted, nil + return outcome, nil } diff --git a/internal/app/runner_terminal_failure_test.go b/internal/app/runner_terminal_failure_test.go index 8635eb5..8b9cf57 100644 --- a/internal/app/runner_terminal_failure_test.go +++ b/internal/app/runner_terminal_failure_test.go @@ -18,10 +18,11 @@ type terminalSessionStore struct { } type successfulStageSaveFailingStore struct { - delegate manifest.Store - stageName string - dependent string - failed bool + delegate manifest.Store + stageName string + failStatus manifest.StageStatus + dependent string + failed bool } func (s *successfulStageSaveFailingStore) Create(ctx context.Context, sessionID string) (*manifest.Manifest, error) { @@ -34,7 +35,11 @@ func (s *successfulStageSaveFailingStore) Load(ctx context.Context, path string) func (s *successfulStageSaveFailingStore) Save(ctx context.Context, path string, m *manifest.Manifest) error { record := m.Stages[s.stageName] - if !s.failed && record != nil && record.Status == manifest.StatusSucceeded { + status := s.failStatus + if status == "" { + status = manifest.StatusSucceeded + } + if !s.failed && record != nil && record.Status == status { s.failed = true m.MarkStageSucceeded(s.dependent, time.Now().UTC(), nil) return errors.New("injected successful-stage save failure") @@ -166,3 +171,19 @@ func TestExecuteStagesSuccessfulSaveFailureMarksStageFailedAndInvalidatesDepende t.Fatalf("normalize stage = %#v, want stale", got) } } + +func TestExecuteStagesSelfSkipSaveFailureIsNotReportedComplete(t *testing.T) { + cfg := testConfig(t) + store := &successfulStageSaveFailingStore{delegate: &manifest.LocalStore{}, stageName: "polish", dependent: "normalize", failStatus: manifest.StatusSkipped} + summary, err := executeStages(context.Background(), cfg, []stage.Stage{resultStage{name: "polish", result: &stage.StageResult{Disposition: stage.StageDispositionSkipped, SkipReason: "disabled"}}}, RunOptions{Env: &Env{ManifestStore: store}}) + if err == nil || summary == nil || summary.Complete || len(summary.Skipped) != 0 || summary.StageResults[0].Outcome != "failed" { + t.Fatalf("summary=%#v error=%v", summary, err) + } + saved, err := store.Load(context.Background(), summary.ManifestPath) + if err != nil { + t.Fatal(err) + } + if saved.Stages["polish"].Status != manifest.StatusFailed || saved.Stages["normalize"].Status != manifest.StatusStale { + t.Fatalf("unpersisted skip preserved as completion: %#v", saved.Stages) + } +} diff --git a/internal/app/runner_test.go b/internal/app/runner_test.go index fcd3714..4c815a2 100644 --- a/internal/app/runner_test.go +++ b/internal/app/runner_test.go @@ -499,7 +499,7 @@ func TestExecuteStagesPublishSkipsRequiredUnselectedConfiguredOutput(t *testing. }, }, } - cfg.ArtifactCatalog = &config.ArtifactCatalog{Artifacts: cfg.Pipeline.Scriptorium.Artifacts} + cfg.ArtifactCatalog = &config.ArtifactCatalog{Artifacts: cfg.Pipeline.Scriptorium.Artifacts, PublishOutputs: cfg.Pipeline.Publish.Outputs} cfg.ArtifactCatalog.Artifacts["player_handout"] = config.ScriptoriumArtifactConfig{PromptID: "handout", OutputPath: "artifacts/handout.md"} store := &manifest.LocalStore{} @@ -860,8 +860,8 @@ func TestExecuteStagesFailureUpdatesManifest(t *testing.T) { if err == nil { t.Fatal("expected error, got nil") } - if summary != nil { - t.Fatalf("summary = %#v, want nil on failure", summary) + if summary == nil || summary.Complete || len(summary.StageResults) != 3 || summary.StageResults[0].Outcome != "completed" || summary.StageResults[1].Outcome != "failed" || !summary.StageResults[1].Attempted || summary.StageResults[2].Outcome != "not_attempted" { + t.Fatalf("incorrect partial summary: %#v", summary) } if !strings.Contains(err.Error(), "stage \"transcribe\" failed") { t.Fatalf("error = %q, want stage failure", err.Error()) @@ -1191,8 +1191,8 @@ func TestExecuteStagesRejectsSkippedResultWithOutputs(t *testing.T) { if err == nil { t.Fatal("executeStages() error = nil, want invalid skipped result failure") } - if summary != nil { - t.Fatalf("summary = %#v, want nil", summary) + if summary == nil || summary.Complete || len(summary.StageResults) != 1 || summary.StageResults[0].Outcome != "failed" { + t.Fatalf("incorrect partial summary: %#v", summary) } if !strings.Contains(err.Error(), "skipped result contains 1 output") { t.Fatalf("error = %q, want skipped-output validation", err) diff --git a/internal/app/transcribe_checkpoint_recovery_test.go b/internal/app/transcribe_checkpoint_recovery_test.go index 9c1b0e8..452ff18 100644 --- a/internal/app/transcribe_checkpoint_recovery_test.go +++ b/internal/app/transcribe_checkpoint_recovery_test.go @@ -88,11 +88,24 @@ func TestExecuteStagesRetriesOnlyUncommittedTranscribeSpeakers(t *testing.T) { }} transcribe := mustSelectStage(t, "transcribe") - if _, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{ + summary, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{ Env: &Env{ManifestStore: baseStore, WhisperX: failing}, - }); !errors.Is(err, cause) { + }) + if !errors.Is(err, cause) { t.Fatalf("failed executeStages() error = %v, want %v", err, cause) } + if summary == nil || summary.Complete || summary.StageResults[0].Outcome != "failed" { + t.Fatalf("partial report = %#v", summary) + } + checkpoints := 0 + for _, artifact := range summary.AvailableArtifacts { + if artifact.Evidence == "speaker_checkpoint" { + checkpoints++ + } + } + if checkpoints != 2 { + t.Fatalf("reported speaker checkpoints = %d, want 2", checkpoints) + } failed := loadTranscribeRecoveryManifest(t, baseStore, cfg) if failed.Stages["transcribe"].Status != manifest.StatusFailed { t.Fatalf("transcribe status = %q, want failed", failed.Stages["transcribe"].Status) diff --git a/internal/app/whisperx_wiring_test.go b/internal/app/whisperx_wiring_test.go index ccb10e3..2681b04 100644 --- a/internal/app/whisperx_wiring_test.go +++ b/internal/app/whisperx_wiring_test.go @@ -104,8 +104,8 @@ func TestExecuteStagesDefaultWiringWhisperXFailureMarksManifestFailed(t *testing if err == nil { t.Fatal("expected error, got nil") } - if summary != nil { - t.Fatalf("summary = %#v, want nil on failure", summary) + if summary == nil || summary.Complete || len(summary.StageResults) != 2 || summary.StageResults[0].Outcome != "completed" || summary.StageResults[1].Outcome != "failed" { + t.Fatalf("incorrect partial summary: %#v", summary) } if calls.Load() != 2 { t.Fatalf("whisperx calls = %d, want 2 (retry + initial)", calls.Load())