Preserve partial run results and terminal failure causes

This commit is contained in:
2026-11-20 05:47:41 +00:00
parent 85df36a770
commit a6c6f1e4ab
13 changed files with 451 additions and 108 deletions

View File

@@ -604,6 +604,18 @@ actions accurately instead of losing all result data.
## Stage 15: Preserve partial run results and terminal failures
**Completed.** Runner summaries retain the full selected plan, attempted/failed,
completed, reused/self-skipped, and unattempted outcomes, plus durable output and
speaker/artifact checkpoint evidence. One finalizer owns terminal persistence
and summary construction; failed completion/self-skip saves remain failures,
earlier successes survive, and original, persistence, and release causes stay
inspectable. Native reports follow resource release, honor quiet visibility,
and cannot undo completed work when their writer fails. Runner, checkpoint,
retry, composition, native-report, focused race, configuration/stage/artifact,
build, and diff checks pass with Go 1.27.4. Updated an existing publication test
fixture to supply its separately derived publication rules; remaining legacy
CLI conversions stay assigned to final cleanup.
Read: `runner.go`, `runner_stage_execution.go`, `runner_runtime.go`,
`runner_finalization.go`, `run_control.go`, and manifest checkpoint contracts.

View File

@@ -20,14 +20,16 @@ type executionInput struct {
}
type runReport struct {
Campaign string `json:"campaign"`
SessionID string `json:"session_id"`
RunID string `json:"run_id"`
Stages []string `json:"stages"`
Executed []string `json:"executed"`
Skipped []string `json:"skipped"`
ArtifactTargets []string `json:"artifact_targets"`
Complete bool `json:"complete"`
Campaign string `json:"campaign"`
SessionID string `json:"session_id"`
RunID string `json:"run_id"`
Stages []string `json:"stages"`
Executed []string `json:"executed"`
Skipped []string `json:"skipped"`
ArtifactTargets []string `json:"artifact_targets"`
StageResults []RunStageResult `json:"stage_results"`
AvailableArtifacts []RunArtifact `json:"available_artifacts"`
Complete bool `json:"complete"`
}
func checkExecutionInput(input *executionInput) error {
@@ -137,5 +139,5 @@ func executeSelectedPipeline(ctx context.Context, input *executionInput, deps co
}
return nil, err
}
return &runReport{Campaign: config.CampaignID(cfg.Campaign), SessionID: summary.SessionID, RunID: summary.RunID, Stages: input.plan.Names(), Executed: summary.Executed, Skipped: summary.Skipped, ArtifactTargets: effective.Keys(), Complete: err == nil}, err
return &runReport{Campaign: config.CampaignID(cfg.Campaign), SessionID: summary.SessionID, RunID: summary.RunID, Stages: input.plan.Names(), Executed: summary.Executed, Skipped: summary.Skipped, ArtifactTargets: effective.Keys(), StageResults: summary.StageResults, AvailableArtifacts: summary.AvailableArtifacts, Complete: err == nil}, err
}

View File

@@ -4,7 +4,12 @@ import (
"bytes"
"context"
"encoding/json"
"errors"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
"gitea.maximumdirect.net/eric/narratio/internal/stage"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/narratio/internal/config"
@@ -101,3 +106,102 @@ func TestExecutionCommandsRejectInvalidSelectionBeforeBootstrap(t *testing.T) {
}
}
}
type releaseObservedStore struct {
artifacts.Store
released bool
}
func (s *releaseObservedStore) ReleaseSessionLock(lock *artifacts.LockHandle) error {
err := s.Store.ReleaseSessionLock(lock)
s.released = true
return err
}
type failingRunReportWriter struct {
t *testing.T
store *releaseObservedStore
err error
}
func (w failingRunReportWriter) Write([]byte) (int, error) {
if !w.store.released {
w.t.Fatal("report written before lock release")
}
return 0, w.err
}
func TestExecutionCommandReportFailurePreservesCompletedWork(t *testing.T) {
pipeline, campaign, session := writeValidConfigFiles(t, t.TempDir())
writeErr := errors.New("report destination unavailable")
var saved *RunSummary
var local *releaseObservedStore
app, err := newApplication(commandDependencies{}, func(ctx context.Context, cfg *config.Config, _ BoundedPlan, opts RunOptions) (*RunSummary, error) {
local = &releaseObservedStore{Store: artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root)}
opts.Env = &Env{ArtifactStore: local}
var err error
saved, err = executeStages(ctx, cfg, []stage.Stage{resultStage{name: "prepare"}}, opts)
return saved, err
})
if err != nil {
t.Fatal(err)
}
// Defer resolving the store until Report invokes Write.
writer := runReportWriteFunc(func(p []byte) (int, error) {
return (failingRunReportWriter{t: t, store: local, err: writeErr}).Write(p)
})
var diagnostic bytes.Buffer
outcome := app.Execute(context.Background(), []string{"run", "2026-05-03", "--stage", "prepare", "--config", pipeline, "--campaign-file", campaign, "--session-file", session, "--format", "json"}, commandkit.Streams{Out: writer, Err: &diagnostic})
if outcome.Status != 1 || !errors.Is(outcome.Cause, writeErr) {
t.Fatalf("outcome=%#v diagnostics=%s", outcome, diagnostic.String())
}
if saved == nil || !saved.Complete {
t.Fatalf("completed domain result=%#v", saved)
}
durable, err := (&manifest.LocalStore{}).Load(context.Background(), saved.ManifestPath)
if err != nil {
t.Fatal(err)
}
if durable.Stages["prepare"].Status != manifest.StatusSucceeded || durable.LastError != nil {
t.Fatalf("report failure changed domain state: %#v", durable)
}
}
type runReportWriteFunc func([]byte) (int, error)
func (f runReportWriteFunc) Write(p []byte) (int, error) { return f(p) }
func TestExecutionCommandPresentsPartialFailureThroughNativeVisibility(t *testing.T) {
pipeline, campaign, session := writeValidConfigFiles(t, t.TempDir())
cause := errors.New("synthetic terminal operation failure")
app, err := newApplication(commandDependencies{}, func(_ context.Context, cfg *config.Config, _ BoundedPlan, _ RunOptions) (*RunSummary, error) {
return &RunSummary{SessionID: cfg.Session.SessionID, RunID: "synthetic-run", Executed: []string{"prepare", "transcribe"}, StageResults: []RunStageResult{{Name: "prepare", Attempted: true, Outcome: "completed"}, {Name: "transcribe", Attempted: true, Outcome: "failed"}}}, cause
})
if err != nil {
t.Fatal(err)
}
for _, quiet := range []bool{false, true} {
args := []string{"run", "2026-05-03", "--config", pipeline, "--campaign-file", campaign, "--session-file", session, "--format", "json"}
if quiet {
args = append(args, "--quiet")
}
var out, diagnostic bytes.Buffer
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &out, Err: &diagnostic})
if outcome.Status != 1 || !errors.Is(outcome.Cause, cause) || strings.Count(diagnostic.String(), cause.Error()) != 1 {
t.Fatalf("outcome=%#v diagnostics=%s", outcome, diagnostic.String())
}
if quiet {
if out.Len() != 0 {
t.Fatal("quiet emitted a report")
}
continue
}
var report runReport
if err := json.Unmarshal(out.Bytes(), &report); err != nil {
t.Fatal(err)
}
if report.Complete || len(report.StageResults) != 2 || report.StageResults[1].Outcome != "failed" {
t.Fatalf("report=%#v", report)
}
}
}

View File

@@ -26,12 +26,15 @@ type RunOptions struct {
}
type RunSummary struct {
SessionID string
RunID string
ManifestPath string
StageNames []string
Executed []string
Skipped []string
SessionID string
RunID string
ManifestPath string
StageNames []string
Executed []string
Skipped []string
StageResults []RunStageResult
AvailableArtifacts []RunArtifact
Complete bool
}
type pipelineExecutor func(context.Context, *config.Config, BoundedPlan, RunOptions) (*RunSummary, error)
@@ -62,7 +65,20 @@ func executePlanWithDependencies(ctx context.Context, cfg *config.Config, plan B
resultErr = errors.Join(resultErr, fmt.Errorf("release session lock: %w", releaseErr))
}
}
if summary != nil {
summary.Complete = resultErr == nil
}
}()
defer func() {
if runtime.started {
summary, resultErr = finalizeInvocation(runtime, resultErr)
}
}()
for _, selected := range runtime.stages {
runtime.stageNames = append(runtime.stageNames, selected.Name())
runtime.stageResults = append(runtime.stageResults, RunStageResult{Name: selected.Name(), Outcome: "not_attempted"})
}
if err := initializeLockedInvocation(runtime); err != nil {
return nil, err
@@ -71,20 +87,32 @@ func executePlanWithDependencies(ctx context.Context, cfg *config.Config, plan B
return nil, err
}
for _, selected := range runtime.stages {
runtime.stageNames = append(runtime.stageNames, selected.Name())
for index, selected := range runtime.stages {
outcome, err := executeSelectedStage(runtime, selected)
if err != nil {
return nil, err
result := &runtime.stageResults[index]
result.Attempted = outcome.executed
result.Reason = outcome.reason
switch {
case err != nil:
result.Outcome = "failed"
case outcome.reused:
result.Outcome = "reused"
case outcome.skipped:
result.Outcome = "skipped"
default:
result.Outcome = "completed"
}
if outcome.executed {
runtime.executed = append(runtime.executed, selected.Name())
}
if outcome.skipped {
if outcome.skipped && err == nil {
runtime.skipped = append(runtime.skipped, selected.Name())
}
if err != nil {
return nil, err
}
}
return finalizeInvocation(runtime)
return nil, nil
}
func stagesContainAny(stages []stage.Stage, names ...string) bool {

View File

@@ -14,21 +14,21 @@ func composeInvocation(runtime *invocationRuntime) error {
if env.WhisperX == nil && stagesContainAny(stages, "transcribe") {
client, err := dependencies.newWhisperX(env.Config)
if err != nil {
return runtime.persistTerminalFailure(fmt.Errorf("initialize whisperx client: %w", err))
return fmt.Errorf("initialize whisperx client: %w", err)
}
env.WhisperX = client
}
if env.Seriatim == nil && stagesContainAny(stages, "merge", "normalize", "trim", "render") {
runner, err := dependencies.newSeriatim(env.Config)
if err != nil {
return runtime.persistTerminalFailure(fmt.Errorf("initialize seriatim runner: %w", err))
return fmt.Errorf("initialize seriatim runner: %w", err)
}
env.Seriatim = runner
}
if env.Audita == nil && stagesContainAny(stages, "polish") {
runner, err := dependencies.newAudita(env.Config)
if err != nil {
return runtime.persistTerminalFailure(fmt.Errorf("initialize audita runner: %w", err))
return fmt.Errorf("initialize audita runner: %w", err)
}
env.Audita = runner
}
@@ -41,7 +41,7 @@ func composeInvocation(runtime *invocationRuntime) error {
if env.ObjectStore == nil && needsObjectStoreForRun(env.Config, stages, runtime.effectiveArtifacts) {
objectStore, err := dependencies.newObjectStore(runtime.ctx, env.Config)
if err != nil {
return runtime.persistTerminalFailure(err)
return err
}
env.ObjectStore = objectStore
}

View File

@@ -1,12 +1,106 @@
package app
func finalizeInvocation(runtime *invocationRuntime) (*RunSummary, error) {
return &RunSummary{
SessionID: runtime.identity.SessionID,
RunID: runtime.identity.RunID,
ManifestPath: runtime.manifestPath,
StageNames: runtime.stageNames,
Executed: runtime.executed,
Skipped: runtime.skipped,
}, nil
import (
"errors"
"fmt"
"sort"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
)
// RunStageResult describes this invocation's work, independently of prior state.
type RunStageResult struct {
Name string `json:"name"`
Attempted bool `json:"attempted"`
Outcome string `json:"outcome"`
Reason string `json:"reason,omitempty"`
}
// RunArtifact identifies durable output evidence without exposing manifest internals.
type RunArtifact struct {
Stage string `json:"stage"`
Source string `json:"source,omitempty"`
Kind string `json:"kind"`
Path string `json:"path"`
ProducerRunID string `json:"producer_run_id,omitempty"`
Evidence string `json:"evidence"`
}
func finalizeInvocation(runtime *invocationRuntime, operationErr error) (*RunSummary, error) {
evidence := runtime.manifest
resultErr := operationErr
if operationErr != nil {
resultErr = persistSessionTerminalFailure(runtime.ctx, runtime.manifestStore, runtime.manifestPath, runtime.manifest, operationErr, runtime.dependencies.now())
// Read durable authority after a failure: checkpoint saves can succeed
// even when terminal persistence fails.
var err error
evidence, err = runtime.manifestStore.Load(runtime.ctx, runtime.manifestPath)
if err != nil {
resultErr = errors.Join(resultErr, fmt.Errorf("read durable run evidence: %w", err))
evidence = nil
}
}
return &RunSummary{
SessionID: runtime.identity.SessionID, RunID: runtime.identity.RunID,
ManifestPath: runtime.manifestPath, StageNames: runtime.stageNames,
Executed: runtime.executed, Skipped: runtime.skipped,
StageResults: runtime.stageResults, AvailableArtifacts: durableRunArtifacts(evidence),
Complete: resultErr == nil,
}, resultErr
}
func durableRunArtifacts(m *manifest.Manifest) []RunArtifact {
out := []RunArtifact{}
if m == nil {
return out
}
names := make([]string, 0, len(m.Stages))
for name := range m.Stages {
names = append(names, name)
}
sort.Strings(names)
for _, name := range names {
record := m.Stages[name]
if record == nil {
continue
}
seen := map[string]bool{}
add := func(value manifest.ArtifactRecord, evidence string) {
key := value.SourceID + "\x00" + value.LocalPath
if seen[key] {
return
}
seen[key] = true
out = append(out, RunArtifact{Stage: name, Source: value.SourceID, Kind: value.Kind, Path: value.LocalPath, ProducerRunID: value.ProducerRunID, Evidence: evidence})
}
if record.Status == manifest.StatusSucceeded {
for _, value := range record.Outputs {
add(value, "stage_result")
}
}
if record.HasVersionedTranscribeState() {
keys := make([]string, 0, len(record.TranscribeSpeakers))
for key := range record.TranscribeSpeakers {
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
add(record.TranscribeSpeakers[key].Output, "speaker_checkpoint")
}
}
if record.HasVersionedAnalyzeState() {
keys := make([]string, 0, len(record.AnalyzeArtifacts))
for key := range record.AnalyzeArtifacts {
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
value := record.AnalyzeArtifacts[key]
if value.Status == manifest.AnalyzeArtifactCurrent && value.Output != nil {
add(*value.Output, "artifact_checkpoint")
}
}
}
}
return out
}

View File

@@ -0,0 +1,94 @@
package app
import (
"context"
"errors"
"testing"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/whisperx"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
"gitea.maximumdirect.net/eric/narratio/internal/stage"
)
type terminalCountingStore struct {
manifest.Store
terminalSaves int
terminalErr error
}
func (s *terminalCountingStore) Save(ctx context.Context, path string, m *manifest.Manifest) error {
if m.LastError != nil {
s.terminalSaves++
if s.terminalErr != nil {
return s.terminalErr
}
}
return s.Store.Save(ctx, path, m)
}
func TestPartialRunRetainsAllFailureCausesAndDurableSuccess(t *testing.T) {
for _, compositionFailure := range []bool{false, true} {
for _, saveFails := range []bool{false, true} {
t.Run(map[bool]string{false: "stage", true: "composition"}[compositionFailure]+map[bool]string{false: " saved", true: " save failed"}[saveFails], func(t *testing.T) {
cfg := testConfig(t)
cause := errors.New("operation unavailable")
saveErr := errors.New("terminal persistence unavailable")
releaseErr := errors.New("lock release failed")
store := &terminalCountingStore{Store: &manifest.LocalStore{}}
if saveFails {
store.terminalErr = saveErr
}
local := &releaseErrorArtifactStore{Store: artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root), err: releaseErr}
firstRuns := 0
stages := []stage.Stage{countingStage{name: "prepare", runs: &firstRuns}, failingStage{name: "transcribe", err: cause}, countingStage{name: "merge", runs: new(int)}}
deps := invocationDependencies{newWhisperX: func(*config.Config) (whisperx.Client, error) {
if compositionFailure {
return nil, cause
}
return &whisperx.NoopClient{}, nil
}}
summary, err := executePlanWithDependencies(context.Background(), cfg, BoundedPlan{stages: stages}, RunOptions{Env: &Env{ManifestStore: store, ArtifactStore: local}}, deps)
if !errors.Is(err, cause) || !errors.Is(err, releaseErr) || (saveFails && !errors.Is(err, saveErr)) {
t.Fatalf("lost causes: %v", err)
}
if store.terminalSaves != 1 {
t.Fatalf("terminal saves=%d", store.terminalSaves)
}
if summary == nil || summary.Complete || len(summary.StageResults) != 3 {
t.Fatalf("summary=%#v", summary)
}
if compositionFailure {
for _, result := range summary.StageResults {
if result.Attempted || result.Outcome != "not_attempted" {
t.Fatalf("composition scheduled work: %#v", summary)
}
}
return
}
if firstRuns != 1 || summary.StageResults[0].Outcome != "completed" || summary.StageResults[1].Outcome != "failed" || !summary.StageResults[1].Attempted || summary.StageResults[2].Outcome != "not_attempted" {
t.Fatalf("partial results=%#v", summary)
}
durable, err := store.Load(context.Background(), summary.ManifestPath)
if err != nil {
t.Fatal(err)
}
if durable.Stages["prepare"].Status != manifest.StatusSucceeded {
t.Fatal("earlier success lost")
}
if saveFails {
if durable.Stages["transcribe"].Status != manifest.StatusRunning {
t.Fatal("failed terminal save advertised durable failure")
}
} else if durable.Stages["transcribe"].Status != manifest.StatusFailed {
t.Fatal("failed attempt not persisted")
}
retry, err := executeStages(context.Background(), cfg, []stage.Stage{countingStage{name: "prepare", runs: &firstRuns}, countingStage{name: "transcribe", runs: new(int)}}, RunOptions{})
if err != nil || firstRuns != 1 || retry.StageResults[0].Outcome != "reused" || !retry.Complete {
t.Fatalf("retry=%#v runs=%d err=%v", retry, firstRuns, err)
}
})
}
}
}

View File

@@ -28,6 +28,8 @@ type invocationRuntime struct {
artifactStore artifacts.Store
manifestPath string
manifest *manifest.Manifest
started bool
stageResults []RunStageResult
stageNames []string
executed []string
skipped []string
@@ -159,6 +161,7 @@ func initializeLockedInvocation(runtime *invocationRuntime) error {
if err := validateBoundedPrerequisites(runtime.plan, m); err != nil {
return fmt.Errorf("validate bounded run prerequisites under session lock: %w", err)
}
runtime.started = true
runtime.identity.applyToSessionManifest(m)
applyEffectiveConfigProvenance(m, runtime.cfg)
if err := runtime.manifestStore.Save(runtime.ctx, runtime.manifestPath, m); err != nil {
@@ -176,14 +179,3 @@ func (runtime *invocationRuntime) ensureRunWorkspaceFor(selected stage.Stage) er
}
return nil
}
func (runtime *invocationRuntime) persistTerminalFailure(err error) error {
return persistSessionTerminalFailure(
runtime.ctx,
runtime.manifestStore,
runtime.manifestPath,
runtime.manifest,
err,
runtime.dependencies.now(),
)
}

View File

@@ -10,40 +10,45 @@ import (
type stageExecutionOutcome struct {
executed bool
skipped bool
reused bool
reason string
}
func executeSelectedStage(runtime *invocationRuntime, selected stage.Stage) (stageExecutionOutcome, error) {
func executeSelectedStage(runtime *invocationRuntime, selected stage.Stage) (outcome stageExecutionOutcome, resultErr error) {
defer func() {
if resultErr == nil || !outcome.executed {
return
}
failedAt := runtime.dependencies.now()
runtime.manifest.MarkStageFailed(selected.Name(), failedAt, resultErr.Error())
_, invalidationErr := invalidateDependentSucceededStagesWithReason(runtime.manifest, selected.Name(), failedAt, staleReasonFailure)
if invalidationErr != nil {
resultErr = errors.Join(resultErr, fmt.Errorf("invalidate dependents after stage %q failure: %w", selected.Name(), invalidationErr))
}
}()
env := runtime.env
m := runtime.manifest
env.Force = runtime.opts.Force
semanticConfig, err := currentStageSemanticConfig(selected, env)
if err != nil {
return stageExecutionOutcome{}, runtime.persistTerminalFailure(
fmt.Errorf("fingerprint semantic configuration for stage %q: %w", selected.Name(), err),
)
return outcome, fmt.Errorf("fingerprint semantic configuration for stage %q: %w", selected.Name(), err)
}
decision := decideStageExecution(recordedStageStatus(m, selected.Name()), runtime.opts.Force, nil)
if decision.action == stageActionSkip {
validation, err := evaluateStageResume(runtime.ctx, selected, env, m, semanticConfig)
if err != nil {
return stageExecutionOutcome{}, runtime.persistTerminalFailure(
fmt.Errorf("validate resume for stage %q: %w", selected.Name(), err),
)
return outcome, fmt.Errorf("validate resume for stage %q: %w", selected.Name(), err)
}
decision = decideStageExecution(recordedStageStatus(m, selected.Name()), runtime.opts.Force, validation)
if decision.markSelectedStale {
staleAt := runtime.dependencies.now()
if _, err := applyPreExecutionDecision(m, selected.Name(), decision, staleAt); err != nil {
return stageExecutionOutcome{}, runtime.persistTerminalFailure(
fmt.Errorf("invalidate dependents after resume validation for stage %q: %w", selected.Name(), err),
)
return outcome, fmt.Errorf("invalidate dependents after resume validation for stage %q: %w", selected.Name(), err)
}
if err := runtime.manifestStore.Save(runtime.ctx, runtime.manifestPath, m); err != nil {
return stageExecutionOutcome{}, runtime.persistTerminalFailure(
fmt.Errorf("save manifest after resume validation for stage %q: %w", selected.Name(), err),
)
return outcome, fmt.Errorf("save manifest after resume validation for stage %q: %w", selected.Name(), err)
}
env.Logger.Info("stage result is not resumable", "stage", selected.Name(), "reason", decision.selectedStaleReason)
}
@@ -51,28 +56,22 @@ func executeSelectedStage(runtime *invocationRuntime, selected stage.Stage) (sta
if decision.action == stageActionSkip {
env.Logger.Info("skipping stage", "stage", selected.Name(), "reason", "already_succeeded", "force", runtime.opts.Force)
return stageExecutionOutcome{skipped: true}, nil
return stageExecutionOutcome{skipped: true, reused: true, reason: "already_succeeded"}, nil
}
if err := runtime.ensureRunWorkspaceFor(selected); err != nil {
return stageExecutionOutcome{}, runtime.persistTerminalFailure(err)
return outcome, err
}
priorOutcome := capturePriorStageOutcome(m, selected.Name())
attempted := stageExecutionOutcome{executed: true}
outcome.executed = true
startedAt := runtime.dependencies.now()
if _, err := beginStageAttempt(m, selected.Name(), decision, startedAt); err != nil {
return stageExecutionOutcome{}, runtime.persistTerminalFailure(
fmt.Errorf("invalidate dependents before forced stage %q: %w", selected.Name(), err),
)
return outcome, fmt.Errorf("invalidate dependents before forced stage %q: %w", selected.Name(), err)
}
env.Logger.Info("starting stage", "stage", selected.Name())
if err := runtime.manifestStore.Save(runtime.ctx, runtime.manifestPath, m); err != nil {
operationErr := fmt.Errorf("save manifest before stage %q: %w", selected.Name(), err)
m.MarkStageFailed(selected.Name(), runtime.dependencies.now(), operationErr.Error())
if _, invalidationErr := invalidateDependentSucceededStagesWithReason(m, selected.Name(), runtime.dependencies.now(), staleReasonFailure); invalidationErr != nil {
operationErr = errors.Join(operationErr, fmt.Errorf("invalidate dependents after stage %q persistence failure: %w", selected.Name(), invalidationErr))
}
return stageExecutionOutcome{}, runtime.persistTerminalFailure(operationErr)
return outcome, operationErr
}
env.Logger.Debug("manifest saved", "stage", selected.Name(), "transition", "running", "path", runtime.manifestPath)
@@ -81,49 +80,33 @@ func executeSelectedStage(runtime *invocationRuntime, selected stage.Stage) (sta
err = validateStageResult(result)
}
if err != nil {
failedAt := runtime.dependencies.now()
m.MarkStageFailed(selected.Name(), failedAt, err.Error())
if _, invalidationErr := invalidateDependentSucceededStagesWithReason(m, selected.Name(), failedAt, staleReasonFailure); invalidationErr != nil {
err = errors.Join(err, fmt.Errorf("invalidate dependents after stage %q failure: %w", selected.Name(), invalidationErr))
}
env.Logger.Info("stage failed", "stage", selected.Name(), "error", err)
return stageExecutionOutcome{}, runtime.persistTerminalFailure(fmt.Errorf("stage %q failed: %w", selected.Name(), err))
return outcome, fmt.Errorf("stage %q failed: %w", selected.Name(), err)
}
if result != nil && result.Disposition == stage.StageDispositionSkipped {
skippedAt := runtime.dependencies.now()
if _, err := applyCompletedStageOutcome(m, selected.Name(), result, nil, semanticConfig, priorOutcome, skippedAt); err != nil {
return stageExecutionOutcome{}, runtime.persistTerminalFailure(
fmt.Errorf("invalidate dependents after stage %q self-skip: %w", selected.Name(), err),
)
return outcome, fmt.Errorf("invalidate dependents after stage %q self-skip: %w", selected.Name(), err)
}
if err := runtime.manifestStore.Save(runtime.ctx, runtime.manifestPath, m); err != nil {
return stageExecutionOutcome{}, runtime.persistTerminalFailure(
fmt.Errorf("save manifest after self-skip %q: %w", selected.Name(), err),
)
return outcome, fmt.Errorf("save manifest after self-skip %q: %w", selected.Name(), err)
}
env.Logger.Debug("manifest saved", "stage", selected.Name(), "transition", "skipped", "path", runtime.manifestPath)
env.Logger.Info("stage skipped", "stage", selected.Name(), "reason", result.SkipReason)
attempted.skipped = true
return attempted, nil
outcome.skipped = true
outcome.reason = result.SkipReason
return outcome, nil
}
outputs := mapResultOutputs(selected.Name(), result, runtime.identity.RunID)
succeededAt := runtime.dependencies.now()
if _, err := applyCompletedStageOutcome(m, selected.Name(), result, outputs, semanticConfig, priorOutcome, succeededAt); err != nil {
return stageExecutionOutcome{}, runtime.persistTerminalFailure(
fmt.Errorf("invalidate dependents after changed stage %q result: %w", selected.Name(), err),
)
return outcome, fmt.Errorf("invalidate dependents after changed stage %q result: %w", selected.Name(), err)
}
if err := runtime.manifestStore.Save(runtime.ctx, runtime.manifestPath, m); err != nil {
operationErr := fmt.Errorf("save manifest after stage %q: %w", selected.Name(), err)
failedAt := runtime.dependencies.now()
m.MarkStageFailed(selected.Name(), failedAt, operationErr.Error())
if _, invalidationErr := invalidateDependentSucceededStagesWithReason(m, selected.Name(), failedAt, staleReasonFailure); invalidationErr != nil {
operationErr = errors.Join(operationErr, fmt.Errorf("invalidate dependents after stage %q persistence failure: %w", selected.Name(), invalidationErr))
}
return stageExecutionOutcome{}, runtime.persistTerminalFailure(operationErr)
return outcome, operationErr
}
env.Logger.Debug("manifest saved", "stage", selected.Name(), "transition", "succeeded", "path", runtime.manifestPath)
env.Logger.Info("stage succeeded", "stage", selected.Name())
return attempted, nil
return outcome, nil
}

View File

@@ -18,10 +18,11 @@ type terminalSessionStore struct {
}
type successfulStageSaveFailingStore struct {
delegate manifest.Store
stageName string
dependent string
failed bool
delegate manifest.Store
stageName string
failStatus manifest.StageStatus
dependent string
failed bool
}
func (s *successfulStageSaveFailingStore) Create(ctx context.Context, sessionID string) (*manifest.Manifest, error) {
@@ -34,7 +35,11 @@ func (s *successfulStageSaveFailingStore) Load(ctx context.Context, path string)
func (s *successfulStageSaveFailingStore) Save(ctx context.Context, path string, m *manifest.Manifest) error {
record := m.Stages[s.stageName]
if !s.failed && record != nil && record.Status == manifest.StatusSucceeded {
status := s.failStatus
if status == "" {
status = manifest.StatusSucceeded
}
if !s.failed && record != nil && record.Status == status {
s.failed = true
m.MarkStageSucceeded(s.dependent, time.Now().UTC(), nil)
return errors.New("injected successful-stage save failure")
@@ -166,3 +171,19 @@ func TestExecuteStagesSuccessfulSaveFailureMarksStageFailedAndInvalidatesDepende
t.Fatalf("normalize stage = %#v, want stale", got)
}
}
func TestExecuteStagesSelfSkipSaveFailureIsNotReportedComplete(t *testing.T) {
cfg := testConfig(t)
store := &successfulStageSaveFailingStore{delegate: &manifest.LocalStore{}, stageName: "polish", dependent: "normalize", failStatus: manifest.StatusSkipped}
summary, err := executeStages(context.Background(), cfg, []stage.Stage{resultStage{name: "polish", result: &stage.StageResult{Disposition: stage.StageDispositionSkipped, SkipReason: "disabled"}}}, RunOptions{Env: &Env{ManifestStore: store}})
if err == nil || summary == nil || summary.Complete || len(summary.Skipped) != 0 || summary.StageResults[0].Outcome != "failed" {
t.Fatalf("summary=%#v error=%v", summary, err)
}
saved, err := store.Load(context.Background(), summary.ManifestPath)
if err != nil {
t.Fatal(err)
}
if saved.Stages["polish"].Status != manifest.StatusFailed || saved.Stages["normalize"].Status != manifest.StatusStale {
t.Fatalf("unpersisted skip preserved as completion: %#v", saved.Stages)
}
}

View File

@@ -499,7 +499,7 @@ func TestExecuteStagesPublishSkipsRequiredUnselectedConfiguredOutput(t *testing.
},
},
}
cfg.ArtifactCatalog = &config.ArtifactCatalog{Artifacts: cfg.Pipeline.Scriptorium.Artifacts}
cfg.ArtifactCatalog = &config.ArtifactCatalog{Artifacts: cfg.Pipeline.Scriptorium.Artifacts, PublishOutputs: cfg.Pipeline.Publish.Outputs}
cfg.ArtifactCatalog.Artifacts["player_handout"] = config.ScriptoriumArtifactConfig{PromptID: "handout", OutputPath: "artifacts/handout.md"}
store := &manifest.LocalStore{}
@@ -860,8 +860,8 @@ func TestExecuteStagesFailureUpdatesManifest(t *testing.T) {
if err == nil {
t.Fatal("expected error, got nil")
}
if summary != nil {
t.Fatalf("summary = %#v, want nil on failure", summary)
if summary == nil || summary.Complete || len(summary.StageResults) != 3 || summary.StageResults[0].Outcome != "completed" || summary.StageResults[1].Outcome != "failed" || !summary.StageResults[1].Attempted || summary.StageResults[2].Outcome != "not_attempted" {
t.Fatalf("incorrect partial summary: %#v", summary)
}
if !strings.Contains(err.Error(), "stage \"transcribe\" failed") {
t.Fatalf("error = %q, want stage failure", err.Error())
@@ -1191,8 +1191,8 @@ func TestExecuteStagesRejectsSkippedResultWithOutputs(t *testing.T) {
if err == nil {
t.Fatal("executeStages() error = nil, want invalid skipped result failure")
}
if summary != nil {
t.Fatalf("summary = %#v, want nil", summary)
if summary == nil || summary.Complete || len(summary.StageResults) != 1 || summary.StageResults[0].Outcome != "failed" {
t.Fatalf("incorrect partial summary: %#v", summary)
}
if !strings.Contains(err.Error(), "skipped result contains 1 output") {
t.Fatalf("error = %q, want skipped-output validation", err)

View File

@@ -88,11 +88,24 @@ func TestExecuteStagesRetriesOnlyUncommittedTranscribeSpeakers(t *testing.T) {
}}
transcribe := mustSelectStage(t, "transcribe")
if _, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{
summary, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{
Env: &Env{ManifestStore: baseStore, WhisperX: failing},
}); !errors.Is(err, cause) {
})
if !errors.Is(err, cause) {
t.Fatalf("failed executeStages() error = %v, want %v", err, cause)
}
if summary == nil || summary.Complete || summary.StageResults[0].Outcome != "failed" {
t.Fatalf("partial report = %#v", summary)
}
checkpoints := 0
for _, artifact := range summary.AvailableArtifacts {
if artifact.Evidence == "speaker_checkpoint" {
checkpoints++
}
}
if checkpoints != 2 {
t.Fatalf("reported speaker checkpoints = %d, want 2", checkpoints)
}
failed := loadTranscribeRecoveryManifest(t, baseStore, cfg)
if failed.Stages["transcribe"].Status != manifest.StatusFailed {
t.Fatalf("transcribe status = %q, want failed", failed.Stages["transcribe"].Status)

View File

@@ -104,8 +104,8 @@ func TestExecuteStagesDefaultWiringWhisperXFailureMarksManifestFailed(t *testing
if err == nil {
t.Fatal("expected error, got nil")
}
if summary != nil {
t.Fatalf("summary = %#v, want nil on failure", summary)
if summary == nil || summary.Complete || len(summary.StageResults) != 2 || summary.StageResults[0].Outcome != "completed" || summary.StageResults[1].Outcome != "failed" {
t.Fatalf("incorrect partial summary: %#v", summary)
}
if calls.Load() != 2 {
t.Fatalf("whisperx calls = %d, want 2 (retry + initial)", calls.Load())