Preserve partial run results and terminal failure causes
This commit is contained in:
@@ -604,6 +604,18 @@ actions accurately instead of losing all result data.
|
||||
|
||||
## Stage 15: Preserve partial run results and terminal failures
|
||||
|
||||
**Completed.** Runner summaries retain the full selected plan, attempted/failed,
|
||||
completed, reused/self-skipped, and unattempted outcomes, plus durable output and
|
||||
speaker/artifact checkpoint evidence. One finalizer owns terminal persistence
|
||||
and summary construction; failed completion/self-skip saves remain failures,
|
||||
earlier successes survive, and original, persistence, and release causes stay
|
||||
inspectable. Native reports follow resource release, honor quiet visibility,
|
||||
and cannot undo completed work when their writer fails. Runner, checkpoint,
|
||||
retry, composition, native-report, focused race, configuration/stage/artifact,
|
||||
build, and diff checks pass with Go 1.27.4. Updated an existing publication test
|
||||
fixture to supply its separately derived publication rules; remaining legacy
|
||||
CLI conversions stay assigned to final cleanup.
|
||||
|
||||
Read: `runner.go`, `runner_stage_execution.go`, `runner_runtime.go`,
|
||||
`runner_finalization.go`, `run_control.go`, and manifest checkpoint contracts.
|
||||
|
||||
|
||||
@@ -20,14 +20,16 @@ type executionInput struct {
|
||||
}
|
||||
|
||||
type runReport struct {
|
||||
Campaign string `json:"campaign"`
|
||||
SessionID string `json:"session_id"`
|
||||
RunID string `json:"run_id"`
|
||||
Stages []string `json:"stages"`
|
||||
Executed []string `json:"executed"`
|
||||
Skipped []string `json:"skipped"`
|
||||
ArtifactTargets []string `json:"artifact_targets"`
|
||||
Complete bool `json:"complete"`
|
||||
Campaign string `json:"campaign"`
|
||||
SessionID string `json:"session_id"`
|
||||
RunID string `json:"run_id"`
|
||||
Stages []string `json:"stages"`
|
||||
Executed []string `json:"executed"`
|
||||
Skipped []string `json:"skipped"`
|
||||
ArtifactTargets []string `json:"artifact_targets"`
|
||||
StageResults []RunStageResult `json:"stage_results"`
|
||||
AvailableArtifacts []RunArtifact `json:"available_artifacts"`
|
||||
Complete bool `json:"complete"`
|
||||
}
|
||||
|
||||
func checkExecutionInput(input *executionInput) error {
|
||||
@@ -137,5 +139,5 @@ func executeSelectedPipeline(ctx context.Context, input *executionInput, deps co
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return &runReport{Campaign: config.CampaignID(cfg.Campaign), SessionID: summary.SessionID, RunID: summary.RunID, Stages: input.plan.Names(), Executed: summary.Executed, Skipped: summary.Skipped, ArtifactTargets: effective.Keys(), Complete: err == nil}, err
|
||||
return &runReport{Campaign: config.CampaignID(cfg.Campaign), SessionID: summary.SessionID, RunID: summary.RunID, Stages: input.plan.Names(), Executed: summary.Executed, Skipped: summary.Skipped, ArtifactTargets: effective.Keys(), StageResults: summary.StageResults, AvailableArtifacts: summary.AvailableArtifacts, Complete: err == nil}, err
|
||||
}
|
||||
|
||||
@@ -4,7 +4,12 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/stage"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
@@ -101,3 +106,102 @@ func TestExecutionCommandsRejectInvalidSelectionBeforeBootstrap(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type releaseObservedStore struct {
|
||||
artifacts.Store
|
||||
released bool
|
||||
}
|
||||
|
||||
func (s *releaseObservedStore) ReleaseSessionLock(lock *artifacts.LockHandle) error {
|
||||
err := s.Store.ReleaseSessionLock(lock)
|
||||
s.released = true
|
||||
return err
|
||||
}
|
||||
|
||||
type failingRunReportWriter struct {
|
||||
t *testing.T
|
||||
store *releaseObservedStore
|
||||
err error
|
||||
}
|
||||
|
||||
func (w failingRunReportWriter) Write([]byte) (int, error) {
|
||||
if !w.store.released {
|
||||
w.t.Fatal("report written before lock release")
|
||||
}
|
||||
return 0, w.err
|
||||
}
|
||||
|
||||
func TestExecutionCommandReportFailurePreservesCompletedWork(t *testing.T) {
|
||||
pipeline, campaign, session := writeValidConfigFiles(t, t.TempDir())
|
||||
writeErr := errors.New("report destination unavailable")
|
||||
var saved *RunSummary
|
||||
var local *releaseObservedStore
|
||||
app, err := newApplication(commandDependencies{}, func(ctx context.Context, cfg *config.Config, _ BoundedPlan, opts RunOptions) (*RunSummary, error) {
|
||||
local = &releaseObservedStore{Store: artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root)}
|
||||
opts.Env = &Env{ArtifactStore: local}
|
||||
var err error
|
||||
saved, err = executeStages(ctx, cfg, []stage.Stage{resultStage{name: "prepare"}}, opts)
|
||||
return saved, err
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Defer resolving the store until Report invokes Write.
|
||||
writer := runReportWriteFunc(func(p []byte) (int, error) {
|
||||
return (failingRunReportWriter{t: t, store: local, err: writeErr}).Write(p)
|
||||
})
|
||||
var diagnostic bytes.Buffer
|
||||
outcome := app.Execute(context.Background(), []string{"run", "2026-05-03", "--stage", "prepare", "--config", pipeline, "--campaign-file", campaign, "--session-file", session, "--format", "json"}, commandkit.Streams{Out: writer, Err: &diagnostic})
|
||||
if outcome.Status != 1 || !errors.Is(outcome.Cause, writeErr) {
|
||||
t.Fatalf("outcome=%#v diagnostics=%s", outcome, diagnostic.String())
|
||||
}
|
||||
if saved == nil || !saved.Complete {
|
||||
t.Fatalf("completed domain result=%#v", saved)
|
||||
}
|
||||
durable, err := (&manifest.LocalStore{}).Load(context.Background(), saved.ManifestPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if durable.Stages["prepare"].Status != manifest.StatusSucceeded || durable.LastError != nil {
|
||||
t.Fatalf("report failure changed domain state: %#v", durable)
|
||||
}
|
||||
}
|
||||
|
||||
type runReportWriteFunc func([]byte) (int, error)
|
||||
|
||||
func (f runReportWriteFunc) Write(p []byte) (int, error) { return f(p) }
|
||||
|
||||
func TestExecutionCommandPresentsPartialFailureThroughNativeVisibility(t *testing.T) {
|
||||
pipeline, campaign, session := writeValidConfigFiles(t, t.TempDir())
|
||||
cause := errors.New("synthetic terminal operation failure")
|
||||
app, err := newApplication(commandDependencies{}, func(_ context.Context, cfg *config.Config, _ BoundedPlan, _ RunOptions) (*RunSummary, error) {
|
||||
return &RunSummary{SessionID: cfg.Session.SessionID, RunID: "synthetic-run", Executed: []string{"prepare", "transcribe"}, StageResults: []RunStageResult{{Name: "prepare", Attempted: true, Outcome: "completed"}, {Name: "transcribe", Attempted: true, Outcome: "failed"}}}, cause
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, quiet := range []bool{false, true} {
|
||||
args := []string{"run", "2026-05-03", "--config", pipeline, "--campaign-file", campaign, "--session-file", session, "--format", "json"}
|
||||
if quiet {
|
||||
args = append(args, "--quiet")
|
||||
}
|
||||
var out, diagnostic bytes.Buffer
|
||||
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &out, Err: &diagnostic})
|
||||
if outcome.Status != 1 || !errors.Is(outcome.Cause, cause) || strings.Count(diagnostic.String(), cause.Error()) != 1 {
|
||||
t.Fatalf("outcome=%#v diagnostics=%s", outcome, diagnostic.String())
|
||||
}
|
||||
if quiet {
|
||||
if out.Len() != 0 {
|
||||
t.Fatal("quiet emitted a report")
|
||||
}
|
||||
continue
|
||||
}
|
||||
var report runReport
|
||||
if err := json.Unmarshal(out.Bytes(), &report); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if report.Complete || len(report.StageResults) != 2 || report.StageResults[1].Outcome != "failed" {
|
||||
t.Fatalf("report=%#v", report)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,12 +26,15 @@ type RunOptions struct {
|
||||
}
|
||||
|
||||
type RunSummary struct {
|
||||
SessionID string
|
||||
RunID string
|
||||
ManifestPath string
|
||||
StageNames []string
|
||||
Executed []string
|
||||
Skipped []string
|
||||
SessionID string
|
||||
RunID string
|
||||
ManifestPath string
|
||||
StageNames []string
|
||||
Executed []string
|
||||
Skipped []string
|
||||
StageResults []RunStageResult
|
||||
AvailableArtifacts []RunArtifact
|
||||
Complete bool
|
||||
}
|
||||
|
||||
type pipelineExecutor func(context.Context, *config.Config, BoundedPlan, RunOptions) (*RunSummary, error)
|
||||
@@ -62,7 +65,20 @@ func executePlanWithDependencies(ctx context.Context, cfg *config.Config, plan B
|
||||
resultErr = errors.Join(resultErr, fmt.Errorf("release session lock: %w", releaseErr))
|
||||
}
|
||||
}
|
||||
if summary != nil {
|
||||
summary.Complete = resultErr == nil
|
||||
}
|
||||
}()
|
||||
defer func() {
|
||||
if runtime.started {
|
||||
summary, resultErr = finalizeInvocation(runtime, resultErr)
|
||||
}
|
||||
}()
|
||||
|
||||
for _, selected := range runtime.stages {
|
||||
runtime.stageNames = append(runtime.stageNames, selected.Name())
|
||||
runtime.stageResults = append(runtime.stageResults, RunStageResult{Name: selected.Name(), Outcome: "not_attempted"})
|
||||
}
|
||||
|
||||
if err := initializeLockedInvocation(runtime); err != nil {
|
||||
return nil, err
|
||||
@@ -71,20 +87,32 @@ func executePlanWithDependencies(ctx context.Context, cfg *config.Config, plan B
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, selected := range runtime.stages {
|
||||
runtime.stageNames = append(runtime.stageNames, selected.Name())
|
||||
for index, selected := range runtime.stages {
|
||||
outcome, err := executeSelectedStage(runtime, selected)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
result := &runtime.stageResults[index]
|
||||
result.Attempted = outcome.executed
|
||||
result.Reason = outcome.reason
|
||||
switch {
|
||||
case err != nil:
|
||||
result.Outcome = "failed"
|
||||
case outcome.reused:
|
||||
result.Outcome = "reused"
|
||||
case outcome.skipped:
|
||||
result.Outcome = "skipped"
|
||||
default:
|
||||
result.Outcome = "completed"
|
||||
}
|
||||
if outcome.executed {
|
||||
runtime.executed = append(runtime.executed, selected.Name())
|
||||
}
|
||||
if outcome.skipped {
|
||||
if outcome.skipped && err == nil {
|
||||
runtime.skipped = append(runtime.skipped, selected.Name())
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return finalizeInvocation(runtime)
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func stagesContainAny(stages []stage.Stage, names ...string) bool {
|
||||
|
||||
@@ -14,21 +14,21 @@ func composeInvocation(runtime *invocationRuntime) error {
|
||||
if env.WhisperX == nil && stagesContainAny(stages, "transcribe") {
|
||||
client, err := dependencies.newWhisperX(env.Config)
|
||||
if err != nil {
|
||||
return runtime.persistTerminalFailure(fmt.Errorf("initialize whisperx client: %w", err))
|
||||
return fmt.Errorf("initialize whisperx client: %w", err)
|
||||
}
|
||||
env.WhisperX = client
|
||||
}
|
||||
if env.Seriatim == nil && stagesContainAny(stages, "merge", "normalize", "trim", "render") {
|
||||
runner, err := dependencies.newSeriatim(env.Config)
|
||||
if err != nil {
|
||||
return runtime.persistTerminalFailure(fmt.Errorf("initialize seriatim runner: %w", err))
|
||||
return fmt.Errorf("initialize seriatim runner: %w", err)
|
||||
}
|
||||
env.Seriatim = runner
|
||||
}
|
||||
if env.Audita == nil && stagesContainAny(stages, "polish") {
|
||||
runner, err := dependencies.newAudita(env.Config)
|
||||
if err != nil {
|
||||
return runtime.persistTerminalFailure(fmt.Errorf("initialize audita runner: %w", err))
|
||||
return fmt.Errorf("initialize audita runner: %w", err)
|
||||
}
|
||||
env.Audita = runner
|
||||
}
|
||||
@@ -41,7 +41,7 @@ func composeInvocation(runtime *invocationRuntime) error {
|
||||
if env.ObjectStore == nil && needsObjectStoreForRun(env.Config, stages, runtime.effectiveArtifacts) {
|
||||
objectStore, err := dependencies.newObjectStore(runtime.ctx, env.Config)
|
||||
if err != nil {
|
||||
return runtime.persistTerminalFailure(err)
|
||||
return err
|
||||
}
|
||||
env.ObjectStore = objectStore
|
||||
}
|
||||
|
||||
@@ -1,12 +1,106 @@
|
||||
package app
|
||||
|
||||
func finalizeInvocation(runtime *invocationRuntime) (*RunSummary, error) {
|
||||
return &RunSummary{
|
||||
SessionID: runtime.identity.SessionID,
|
||||
RunID: runtime.identity.RunID,
|
||||
ManifestPath: runtime.manifestPath,
|
||||
StageNames: runtime.stageNames,
|
||||
Executed: runtime.executed,
|
||||
Skipped: runtime.skipped,
|
||||
}, nil
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
)
|
||||
|
||||
// RunStageResult describes this invocation's work, independently of prior state.
|
||||
type RunStageResult struct {
|
||||
Name string `json:"name"`
|
||||
Attempted bool `json:"attempted"`
|
||||
Outcome string `json:"outcome"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
}
|
||||
|
||||
// RunArtifact identifies durable output evidence without exposing manifest internals.
|
||||
type RunArtifact struct {
|
||||
Stage string `json:"stage"`
|
||||
Source string `json:"source,omitempty"`
|
||||
Kind string `json:"kind"`
|
||||
Path string `json:"path"`
|
||||
ProducerRunID string `json:"producer_run_id,omitempty"`
|
||||
Evidence string `json:"evidence"`
|
||||
}
|
||||
|
||||
func finalizeInvocation(runtime *invocationRuntime, operationErr error) (*RunSummary, error) {
|
||||
evidence := runtime.manifest
|
||||
resultErr := operationErr
|
||||
if operationErr != nil {
|
||||
resultErr = persistSessionTerminalFailure(runtime.ctx, runtime.manifestStore, runtime.manifestPath, runtime.manifest, operationErr, runtime.dependencies.now())
|
||||
// Read durable authority after a failure: checkpoint saves can succeed
|
||||
// even when terminal persistence fails.
|
||||
var err error
|
||||
evidence, err = runtime.manifestStore.Load(runtime.ctx, runtime.manifestPath)
|
||||
if err != nil {
|
||||
resultErr = errors.Join(resultErr, fmt.Errorf("read durable run evidence: %w", err))
|
||||
evidence = nil
|
||||
}
|
||||
}
|
||||
return &RunSummary{
|
||||
SessionID: runtime.identity.SessionID, RunID: runtime.identity.RunID,
|
||||
ManifestPath: runtime.manifestPath, StageNames: runtime.stageNames,
|
||||
Executed: runtime.executed, Skipped: runtime.skipped,
|
||||
StageResults: runtime.stageResults, AvailableArtifacts: durableRunArtifacts(evidence),
|
||||
Complete: resultErr == nil,
|
||||
}, resultErr
|
||||
}
|
||||
|
||||
func durableRunArtifacts(m *manifest.Manifest) []RunArtifact {
|
||||
out := []RunArtifact{}
|
||||
if m == nil {
|
||||
return out
|
||||
}
|
||||
names := make([]string, 0, len(m.Stages))
|
||||
for name := range m.Stages {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, name := range names {
|
||||
record := m.Stages[name]
|
||||
if record == nil {
|
||||
continue
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
add := func(value manifest.ArtifactRecord, evidence string) {
|
||||
key := value.SourceID + "\x00" + value.LocalPath
|
||||
if seen[key] {
|
||||
return
|
||||
}
|
||||
seen[key] = true
|
||||
out = append(out, RunArtifact{Stage: name, Source: value.SourceID, Kind: value.Kind, Path: value.LocalPath, ProducerRunID: value.ProducerRunID, Evidence: evidence})
|
||||
}
|
||||
if record.Status == manifest.StatusSucceeded {
|
||||
for _, value := range record.Outputs {
|
||||
add(value, "stage_result")
|
||||
}
|
||||
}
|
||||
if record.HasVersionedTranscribeState() {
|
||||
keys := make([]string, 0, len(record.TranscribeSpeakers))
|
||||
for key := range record.TranscribeSpeakers {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, key := range keys {
|
||||
add(record.TranscribeSpeakers[key].Output, "speaker_checkpoint")
|
||||
}
|
||||
}
|
||||
if record.HasVersionedAnalyzeState() {
|
||||
keys := make([]string, 0, len(record.AnalyzeArtifacts))
|
||||
for key := range record.AnalyzeArtifacts {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, key := range keys {
|
||||
value := record.AnalyzeArtifacts[key]
|
||||
if value.Status == manifest.AnalyzeArtifactCurrent && value.Output != nil {
|
||||
add(*value.Output, "artifact_checkpoint")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
94
internal/app/runner_partial_result_test.go
Normal file
94
internal/app/runner_partial_result_test.go
Normal file
@@ -0,0 +1,94 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/whisperx"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/stage"
|
||||
)
|
||||
|
||||
type terminalCountingStore struct {
|
||||
manifest.Store
|
||||
terminalSaves int
|
||||
terminalErr error
|
||||
}
|
||||
|
||||
func (s *terminalCountingStore) Save(ctx context.Context, path string, m *manifest.Manifest) error {
|
||||
if m.LastError != nil {
|
||||
s.terminalSaves++
|
||||
if s.terminalErr != nil {
|
||||
return s.terminalErr
|
||||
}
|
||||
}
|
||||
return s.Store.Save(ctx, path, m)
|
||||
}
|
||||
|
||||
func TestPartialRunRetainsAllFailureCausesAndDurableSuccess(t *testing.T) {
|
||||
for _, compositionFailure := range []bool{false, true} {
|
||||
for _, saveFails := range []bool{false, true} {
|
||||
t.Run(map[bool]string{false: "stage", true: "composition"}[compositionFailure]+map[bool]string{false: " saved", true: " save failed"}[saveFails], func(t *testing.T) {
|
||||
cfg := testConfig(t)
|
||||
cause := errors.New("operation unavailable")
|
||||
saveErr := errors.New("terminal persistence unavailable")
|
||||
releaseErr := errors.New("lock release failed")
|
||||
store := &terminalCountingStore{Store: &manifest.LocalStore{}}
|
||||
if saveFails {
|
||||
store.terminalErr = saveErr
|
||||
}
|
||||
local := &releaseErrorArtifactStore{Store: artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root), err: releaseErr}
|
||||
firstRuns := 0
|
||||
stages := []stage.Stage{countingStage{name: "prepare", runs: &firstRuns}, failingStage{name: "transcribe", err: cause}, countingStage{name: "merge", runs: new(int)}}
|
||||
deps := invocationDependencies{newWhisperX: func(*config.Config) (whisperx.Client, error) {
|
||||
if compositionFailure {
|
||||
return nil, cause
|
||||
}
|
||||
return &whisperx.NoopClient{}, nil
|
||||
}}
|
||||
summary, err := executePlanWithDependencies(context.Background(), cfg, BoundedPlan{stages: stages}, RunOptions{Env: &Env{ManifestStore: store, ArtifactStore: local}}, deps)
|
||||
if !errors.Is(err, cause) || !errors.Is(err, releaseErr) || (saveFails && !errors.Is(err, saveErr)) {
|
||||
t.Fatalf("lost causes: %v", err)
|
||||
}
|
||||
if store.terminalSaves != 1 {
|
||||
t.Fatalf("terminal saves=%d", store.terminalSaves)
|
||||
}
|
||||
if summary == nil || summary.Complete || len(summary.StageResults) != 3 {
|
||||
t.Fatalf("summary=%#v", summary)
|
||||
}
|
||||
if compositionFailure {
|
||||
for _, result := range summary.StageResults {
|
||||
if result.Attempted || result.Outcome != "not_attempted" {
|
||||
t.Fatalf("composition scheduled work: %#v", summary)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
if firstRuns != 1 || summary.StageResults[0].Outcome != "completed" || summary.StageResults[1].Outcome != "failed" || !summary.StageResults[1].Attempted || summary.StageResults[2].Outcome != "not_attempted" {
|
||||
t.Fatalf("partial results=%#v", summary)
|
||||
}
|
||||
durable, err := store.Load(context.Background(), summary.ManifestPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if durable.Stages["prepare"].Status != manifest.StatusSucceeded {
|
||||
t.Fatal("earlier success lost")
|
||||
}
|
||||
if saveFails {
|
||||
if durable.Stages["transcribe"].Status != manifest.StatusRunning {
|
||||
t.Fatal("failed terminal save advertised durable failure")
|
||||
}
|
||||
} else if durable.Stages["transcribe"].Status != manifest.StatusFailed {
|
||||
t.Fatal("failed attempt not persisted")
|
||||
}
|
||||
retry, err := executeStages(context.Background(), cfg, []stage.Stage{countingStage{name: "prepare", runs: &firstRuns}, countingStage{name: "transcribe", runs: new(int)}}, RunOptions{})
|
||||
if err != nil || firstRuns != 1 || retry.StageResults[0].Outcome != "reused" || !retry.Complete {
|
||||
t.Fatalf("retry=%#v runs=%d err=%v", retry, firstRuns, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -28,6 +28,8 @@ type invocationRuntime struct {
|
||||
artifactStore artifacts.Store
|
||||
manifestPath string
|
||||
manifest *manifest.Manifest
|
||||
started bool
|
||||
stageResults []RunStageResult
|
||||
stageNames []string
|
||||
executed []string
|
||||
skipped []string
|
||||
@@ -159,6 +161,7 @@ func initializeLockedInvocation(runtime *invocationRuntime) error {
|
||||
if err := validateBoundedPrerequisites(runtime.plan, m); err != nil {
|
||||
return fmt.Errorf("validate bounded run prerequisites under session lock: %w", err)
|
||||
}
|
||||
runtime.started = true
|
||||
runtime.identity.applyToSessionManifest(m)
|
||||
applyEffectiveConfigProvenance(m, runtime.cfg)
|
||||
if err := runtime.manifestStore.Save(runtime.ctx, runtime.manifestPath, m); err != nil {
|
||||
@@ -176,14 +179,3 @@ func (runtime *invocationRuntime) ensureRunWorkspaceFor(selected stage.Stage) er
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (runtime *invocationRuntime) persistTerminalFailure(err error) error {
|
||||
return persistSessionTerminalFailure(
|
||||
runtime.ctx,
|
||||
runtime.manifestStore,
|
||||
runtime.manifestPath,
|
||||
runtime.manifest,
|
||||
err,
|
||||
runtime.dependencies.now(),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -10,40 +10,45 @@ import (
|
||||
type stageExecutionOutcome struct {
|
||||
executed bool
|
||||
skipped bool
|
||||
reused bool
|
||||
reason string
|
||||
}
|
||||
|
||||
func executeSelectedStage(runtime *invocationRuntime, selected stage.Stage) (stageExecutionOutcome, error) {
|
||||
func executeSelectedStage(runtime *invocationRuntime, selected stage.Stage) (outcome stageExecutionOutcome, resultErr error) {
|
||||
defer func() {
|
||||
if resultErr == nil || !outcome.executed {
|
||||
return
|
||||
}
|
||||
failedAt := runtime.dependencies.now()
|
||||
runtime.manifest.MarkStageFailed(selected.Name(), failedAt, resultErr.Error())
|
||||
_, invalidationErr := invalidateDependentSucceededStagesWithReason(runtime.manifest, selected.Name(), failedAt, staleReasonFailure)
|
||||
if invalidationErr != nil {
|
||||
resultErr = errors.Join(resultErr, fmt.Errorf("invalidate dependents after stage %q failure: %w", selected.Name(), invalidationErr))
|
||||
}
|
||||
}()
|
||||
env := runtime.env
|
||||
m := runtime.manifest
|
||||
env.Force = runtime.opts.Force
|
||||
|
||||
semanticConfig, err := currentStageSemanticConfig(selected, env)
|
||||
if err != nil {
|
||||
return stageExecutionOutcome{}, runtime.persistTerminalFailure(
|
||||
fmt.Errorf("fingerprint semantic configuration for stage %q: %w", selected.Name(), err),
|
||||
)
|
||||
return outcome, fmt.Errorf("fingerprint semantic configuration for stage %q: %w", selected.Name(), err)
|
||||
}
|
||||
decision := decideStageExecution(recordedStageStatus(m, selected.Name()), runtime.opts.Force, nil)
|
||||
|
||||
if decision.action == stageActionSkip {
|
||||
validation, err := evaluateStageResume(runtime.ctx, selected, env, m, semanticConfig)
|
||||
if err != nil {
|
||||
return stageExecutionOutcome{}, runtime.persistTerminalFailure(
|
||||
fmt.Errorf("validate resume for stage %q: %w", selected.Name(), err),
|
||||
)
|
||||
return outcome, fmt.Errorf("validate resume for stage %q: %w", selected.Name(), err)
|
||||
}
|
||||
decision = decideStageExecution(recordedStageStatus(m, selected.Name()), runtime.opts.Force, validation)
|
||||
if decision.markSelectedStale {
|
||||
staleAt := runtime.dependencies.now()
|
||||
if _, err := applyPreExecutionDecision(m, selected.Name(), decision, staleAt); err != nil {
|
||||
return stageExecutionOutcome{}, runtime.persistTerminalFailure(
|
||||
fmt.Errorf("invalidate dependents after resume validation for stage %q: %w", selected.Name(), err),
|
||||
)
|
||||
return outcome, fmt.Errorf("invalidate dependents after resume validation for stage %q: %w", selected.Name(), err)
|
||||
}
|
||||
if err := runtime.manifestStore.Save(runtime.ctx, runtime.manifestPath, m); err != nil {
|
||||
return stageExecutionOutcome{}, runtime.persistTerminalFailure(
|
||||
fmt.Errorf("save manifest after resume validation for stage %q: %w", selected.Name(), err),
|
||||
)
|
||||
return outcome, fmt.Errorf("save manifest after resume validation for stage %q: %w", selected.Name(), err)
|
||||
}
|
||||
env.Logger.Info("stage result is not resumable", "stage", selected.Name(), "reason", decision.selectedStaleReason)
|
||||
}
|
||||
@@ -51,28 +56,22 @@ func executeSelectedStage(runtime *invocationRuntime, selected stage.Stage) (sta
|
||||
|
||||
if decision.action == stageActionSkip {
|
||||
env.Logger.Info("skipping stage", "stage", selected.Name(), "reason", "already_succeeded", "force", runtime.opts.Force)
|
||||
return stageExecutionOutcome{skipped: true}, nil
|
||||
return stageExecutionOutcome{skipped: true, reused: true, reason: "already_succeeded"}, nil
|
||||
}
|
||||
if err := runtime.ensureRunWorkspaceFor(selected); err != nil {
|
||||
return stageExecutionOutcome{}, runtime.persistTerminalFailure(err)
|
||||
return outcome, err
|
||||
}
|
||||
|
||||
priorOutcome := capturePriorStageOutcome(m, selected.Name())
|
||||
attempted := stageExecutionOutcome{executed: true}
|
||||
outcome.executed = true
|
||||
startedAt := runtime.dependencies.now()
|
||||
if _, err := beginStageAttempt(m, selected.Name(), decision, startedAt); err != nil {
|
||||
return stageExecutionOutcome{}, runtime.persistTerminalFailure(
|
||||
fmt.Errorf("invalidate dependents before forced stage %q: %w", selected.Name(), err),
|
||||
)
|
||||
return outcome, fmt.Errorf("invalidate dependents before forced stage %q: %w", selected.Name(), err)
|
||||
}
|
||||
env.Logger.Info("starting stage", "stage", selected.Name())
|
||||
if err := runtime.manifestStore.Save(runtime.ctx, runtime.manifestPath, m); err != nil {
|
||||
operationErr := fmt.Errorf("save manifest before stage %q: %w", selected.Name(), err)
|
||||
m.MarkStageFailed(selected.Name(), runtime.dependencies.now(), operationErr.Error())
|
||||
if _, invalidationErr := invalidateDependentSucceededStagesWithReason(m, selected.Name(), runtime.dependencies.now(), staleReasonFailure); invalidationErr != nil {
|
||||
operationErr = errors.Join(operationErr, fmt.Errorf("invalidate dependents after stage %q persistence failure: %w", selected.Name(), invalidationErr))
|
||||
}
|
||||
return stageExecutionOutcome{}, runtime.persistTerminalFailure(operationErr)
|
||||
return outcome, operationErr
|
||||
}
|
||||
env.Logger.Debug("manifest saved", "stage", selected.Name(), "transition", "running", "path", runtime.manifestPath)
|
||||
|
||||
@@ -81,49 +80,33 @@ func executeSelectedStage(runtime *invocationRuntime, selected stage.Stage) (sta
|
||||
err = validateStageResult(result)
|
||||
}
|
||||
if err != nil {
|
||||
failedAt := runtime.dependencies.now()
|
||||
m.MarkStageFailed(selected.Name(), failedAt, err.Error())
|
||||
if _, invalidationErr := invalidateDependentSucceededStagesWithReason(m, selected.Name(), failedAt, staleReasonFailure); invalidationErr != nil {
|
||||
err = errors.Join(err, fmt.Errorf("invalidate dependents after stage %q failure: %w", selected.Name(), invalidationErr))
|
||||
}
|
||||
env.Logger.Info("stage failed", "stage", selected.Name(), "error", err)
|
||||
return stageExecutionOutcome{}, runtime.persistTerminalFailure(fmt.Errorf("stage %q failed: %w", selected.Name(), err))
|
||||
return outcome, fmt.Errorf("stage %q failed: %w", selected.Name(), err)
|
||||
}
|
||||
if result != nil && result.Disposition == stage.StageDispositionSkipped {
|
||||
skippedAt := runtime.dependencies.now()
|
||||
if _, err := applyCompletedStageOutcome(m, selected.Name(), result, nil, semanticConfig, priorOutcome, skippedAt); err != nil {
|
||||
return stageExecutionOutcome{}, runtime.persistTerminalFailure(
|
||||
fmt.Errorf("invalidate dependents after stage %q self-skip: %w", selected.Name(), err),
|
||||
)
|
||||
return outcome, fmt.Errorf("invalidate dependents after stage %q self-skip: %w", selected.Name(), err)
|
||||
}
|
||||
if err := runtime.manifestStore.Save(runtime.ctx, runtime.manifestPath, m); err != nil {
|
||||
return stageExecutionOutcome{}, runtime.persistTerminalFailure(
|
||||
fmt.Errorf("save manifest after self-skip %q: %w", selected.Name(), err),
|
||||
)
|
||||
return outcome, fmt.Errorf("save manifest after self-skip %q: %w", selected.Name(), err)
|
||||
}
|
||||
env.Logger.Debug("manifest saved", "stage", selected.Name(), "transition", "skipped", "path", runtime.manifestPath)
|
||||
env.Logger.Info("stage skipped", "stage", selected.Name(), "reason", result.SkipReason)
|
||||
attempted.skipped = true
|
||||
return attempted, nil
|
||||
outcome.skipped = true
|
||||
outcome.reason = result.SkipReason
|
||||
return outcome, nil
|
||||
}
|
||||
|
||||
outputs := mapResultOutputs(selected.Name(), result, runtime.identity.RunID)
|
||||
succeededAt := runtime.dependencies.now()
|
||||
if _, err := applyCompletedStageOutcome(m, selected.Name(), result, outputs, semanticConfig, priorOutcome, succeededAt); err != nil {
|
||||
return stageExecutionOutcome{}, runtime.persistTerminalFailure(
|
||||
fmt.Errorf("invalidate dependents after changed stage %q result: %w", selected.Name(), err),
|
||||
)
|
||||
return outcome, fmt.Errorf("invalidate dependents after changed stage %q result: %w", selected.Name(), err)
|
||||
}
|
||||
if err := runtime.manifestStore.Save(runtime.ctx, runtime.manifestPath, m); err != nil {
|
||||
operationErr := fmt.Errorf("save manifest after stage %q: %w", selected.Name(), err)
|
||||
failedAt := runtime.dependencies.now()
|
||||
m.MarkStageFailed(selected.Name(), failedAt, operationErr.Error())
|
||||
if _, invalidationErr := invalidateDependentSucceededStagesWithReason(m, selected.Name(), failedAt, staleReasonFailure); invalidationErr != nil {
|
||||
operationErr = errors.Join(operationErr, fmt.Errorf("invalidate dependents after stage %q persistence failure: %w", selected.Name(), invalidationErr))
|
||||
}
|
||||
return stageExecutionOutcome{}, runtime.persistTerminalFailure(operationErr)
|
||||
return outcome, operationErr
|
||||
}
|
||||
env.Logger.Debug("manifest saved", "stage", selected.Name(), "transition", "succeeded", "path", runtime.manifestPath)
|
||||
env.Logger.Info("stage succeeded", "stage", selected.Name())
|
||||
return attempted, nil
|
||||
return outcome, nil
|
||||
}
|
||||
|
||||
@@ -18,10 +18,11 @@ type terminalSessionStore struct {
|
||||
}
|
||||
|
||||
type successfulStageSaveFailingStore struct {
|
||||
delegate manifest.Store
|
||||
stageName string
|
||||
dependent string
|
||||
failed bool
|
||||
delegate manifest.Store
|
||||
stageName string
|
||||
failStatus manifest.StageStatus
|
||||
dependent string
|
||||
failed bool
|
||||
}
|
||||
|
||||
func (s *successfulStageSaveFailingStore) Create(ctx context.Context, sessionID string) (*manifest.Manifest, error) {
|
||||
@@ -34,7 +35,11 @@ func (s *successfulStageSaveFailingStore) Load(ctx context.Context, path string)
|
||||
|
||||
func (s *successfulStageSaveFailingStore) Save(ctx context.Context, path string, m *manifest.Manifest) error {
|
||||
record := m.Stages[s.stageName]
|
||||
if !s.failed && record != nil && record.Status == manifest.StatusSucceeded {
|
||||
status := s.failStatus
|
||||
if status == "" {
|
||||
status = manifest.StatusSucceeded
|
||||
}
|
||||
if !s.failed && record != nil && record.Status == status {
|
||||
s.failed = true
|
||||
m.MarkStageSucceeded(s.dependent, time.Now().UTC(), nil)
|
||||
return errors.New("injected successful-stage save failure")
|
||||
@@ -166,3 +171,19 @@ func TestExecuteStagesSuccessfulSaveFailureMarksStageFailedAndInvalidatesDepende
|
||||
t.Fatalf("normalize stage = %#v, want stale", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteStagesSelfSkipSaveFailureIsNotReportedComplete(t *testing.T) {
|
||||
cfg := testConfig(t)
|
||||
store := &successfulStageSaveFailingStore{delegate: &manifest.LocalStore{}, stageName: "polish", dependent: "normalize", failStatus: manifest.StatusSkipped}
|
||||
summary, err := executeStages(context.Background(), cfg, []stage.Stage{resultStage{name: "polish", result: &stage.StageResult{Disposition: stage.StageDispositionSkipped, SkipReason: "disabled"}}}, RunOptions{Env: &Env{ManifestStore: store}})
|
||||
if err == nil || summary == nil || summary.Complete || len(summary.Skipped) != 0 || summary.StageResults[0].Outcome != "failed" {
|
||||
t.Fatalf("summary=%#v error=%v", summary, err)
|
||||
}
|
||||
saved, err := store.Load(context.Background(), summary.ManifestPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if saved.Stages["polish"].Status != manifest.StatusFailed || saved.Stages["normalize"].Status != manifest.StatusStale {
|
||||
t.Fatalf("unpersisted skip preserved as completion: %#v", saved.Stages)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -499,7 +499,7 @@ func TestExecuteStagesPublishSkipsRequiredUnselectedConfiguredOutput(t *testing.
|
||||
},
|
||||
},
|
||||
}
|
||||
cfg.ArtifactCatalog = &config.ArtifactCatalog{Artifacts: cfg.Pipeline.Scriptorium.Artifacts}
|
||||
cfg.ArtifactCatalog = &config.ArtifactCatalog{Artifacts: cfg.Pipeline.Scriptorium.Artifacts, PublishOutputs: cfg.Pipeline.Publish.Outputs}
|
||||
|
||||
cfg.ArtifactCatalog.Artifacts["player_handout"] = config.ScriptoriumArtifactConfig{PromptID: "handout", OutputPath: "artifacts/handout.md"}
|
||||
store := &manifest.LocalStore{}
|
||||
@@ -860,8 +860,8 @@ func TestExecuteStagesFailureUpdatesManifest(t *testing.T) {
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
if summary != nil {
|
||||
t.Fatalf("summary = %#v, want nil on failure", summary)
|
||||
if summary == nil || summary.Complete || len(summary.StageResults) != 3 || summary.StageResults[0].Outcome != "completed" || summary.StageResults[1].Outcome != "failed" || !summary.StageResults[1].Attempted || summary.StageResults[2].Outcome != "not_attempted" {
|
||||
t.Fatalf("incorrect partial summary: %#v", summary)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "stage \"transcribe\" failed") {
|
||||
t.Fatalf("error = %q, want stage failure", err.Error())
|
||||
@@ -1191,8 +1191,8 @@ func TestExecuteStagesRejectsSkippedResultWithOutputs(t *testing.T) {
|
||||
if err == nil {
|
||||
t.Fatal("executeStages() error = nil, want invalid skipped result failure")
|
||||
}
|
||||
if summary != nil {
|
||||
t.Fatalf("summary = %#v, want nil", summary)
|
||||
if summary == nil || summary.Complete || len(summary.StageResults) != 1 || summary.StageResults[0].Outcome != "failed" {
|
||||
t.Fatalf("incorrect partial summary: %#v", summary)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "skipped result contains 1 output") {
|
||||
t.Fatalf("error = %q, want skipped-output validation", err)
|
||||
|
||||
@@ -88,11 +88,24 @@ func TestExecuteStagesRetriesOnlyUncommittedTranscribeSpeakers(t *testing.T) {
|
||||
}}
|
||||
transcribe := mustSelectStage(t, "transcribe")
|
||||
|
||||
if _, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{
|
||||
summary, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{
|
||||
Env: &Env{ManifestStore: baseStore, WhisperX: failing},
|
||||
}); !errors.Is(err, cause) {
|
||||
})
|
||||
if !errors.Is(err, cause) {
|
||||
t.Fatalf("failed executeStages() error = %v, want %v", err, cause)
|
||||
}
|
||||
if summary == nil || summary.Complete || summary.StageResults[0].Outcome != "failed" {
|
||||
t.Fatalf("partial report = %#v", summary)
|
||||
}
|
||||
checkpoints := 0
|
||||
for _, artifact := range summary.AvailableArtifacts {
|
||||
if artifact.Evidence == "speaker_checkpoint" {
|
||||
checkpoints++
|
||||
}
|
||||
}
|
||||
if checkpoints != 2 {
|
||||
t.Fatalf("reported speaker checkpoints = %d, want 2", checkpoints)
|
||||
}
|
||||
failed := loadTranscribeRecoveryManifest(t, baseStore, cfg)
|
||||
if failed.Stages["transcribe"].Status != manifest.StatusFailed {
|
||||
t.Fatalf("transcribe status = %q, want failed", failed.Stages["transcribe"].Status)
|
||||
|
||||
@@ -104,8 +104,8 @@ func TestExecuteStagesDefaultWiringWhisperXFailureMarksManifestFailed(t *testing
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
if summary != nil {
|
||||
t.Fatalf("summary = %#v, want nil on failure", summary)
|
||||
if summary == nil || summary.Complete || len(summary.StageResults) != 2 || summary.StageResults[0].Outcome != "completed" || summary.StageResults[1].Outcome != "failed" {
|
||||
t.Fatalf("incorrect partial summary: %#v", summary)
|
||||
}
|
||||
if calls.Load() != 2 {
|
||||
t.Fatalf("whisperx calls = %d, want 2 (retry + initial)", calls.Load())
|
||||
|
||||
Reference in New Issue
Block a user