From 735dfc1834996da643d00101250e2107cb1d0e9c Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Wed, 11 Nov 2026 04:31:44 +0000 Subject: [PATCH] Report correction repair operations and outcomes --- docs/integrations/output-files.md | 14 +++-- docs/roadmap/implementation.md | 2 + internal/core/reporting/markdown.go | 10 ++++ internal/core/reporting/report.go | 10 ++++ internal/framework/outcomes/outcomes.go | 10 ++-- internal/framework/processreport/report.go | 38 ++++++++++++-- .../framework/processreport/report_test.go | 52 +++++++++++++++++++ internal/framework/runner/repair.go | 29 +++++++++++ internal/framework/runner/repair_test.go | 3 ++ internal/framework/runner/runner.go | 1 + 10 files changed, 159 insertions(+), 10 deletions(-) diff --git a/docs/integrations/output-files.md b/docs/integrations/output-files.md index a5d2423..fb2cb85 100644 --- a/docs/integrations/output-files.md +++ b/docs/integrations/output-files.md @@ -48,7 +48,7 @@ remain in opt-in debug files. An `errors` record with `scope: work_unit` identifies zero-based `stage_index` and `chunk_index`, optional `stage_id`, module key/instance, initial segment range, applied-correction count, and a `failures` sequence. Each failure identifies the operation -(`proposal`, `validation`, or `validation_input`), optional validator and +(`proposal`, `validation`, `validation_input`, or `correction_repair`), optional validator and zero-based batch index, safe reason/message, sorted proposal indices when known, and execution counts when available. Multiple failed batches in one work unit form one error record. `scope: run` instead records phase, safe reason @@ -62,7 +62,9 @@ may still be approved or rejected normally. Each recovery identifies its work unit, optional `stage_id`, and successful operation, with sorted `reasons`, a safe message, available execution counts, and sorted `filtered_proposals` details. One recovery can contain `output_repaired`, -`service_recovered`, and `proposals_filtered` together. Filtering covers +`service_recovered`, `correction_revised`, and `proposals_filtered` together. +`correction_revised` means a repair yielded a fully validated revision; its +application outcome is recorded in the Markdown report. Filtering covers defective response proposals; low confidence, protected-term rejection, ordinary editorial rejection, and an apply-time conflict caused by an earlier accepted correction are not filtering recoveries. Failed operations are errors, @@ -84,8 +86,12 @@ the one-based position and module, plus the operator ID when supplied. Completed counts. `—` means not started and `n*` means interrupted with n facts recorded. An empty stage or chunk set is identified as having no work units. Short reason breakdowns count records containing each reason; a recovery can have several reasons, so those -counts need not sum to the recovery total. The report refers to both YAML -files. It does not include transcript excerpts or correction lists. +counts need not sum to the recovery total. When a repair was attempted, a +Correction repair table lists its stage, chunk, +original proposal index, semantic attempt count, and final outcome (`applied`, +`approved_skipped`, `withdrawn`, `rejected`, `exhausted`, `unavailable`, or +`interrupted`). The report refers to both YAML files. It does not include +transcript excerpts or correction lists. ## Debug-Only Files diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index c67fce7..e4b1062 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -276,6 +276,8 @@ validator responses remains [deferred](future.md#domain-aware-response-recovery) ## Stage 7 — Account for operations and publish compact ordinary outcomes +**Completed.** + 1. Add `correction_repair` as an operation kind in `internal/framework/outcomes`. For each repair call retain its proposal identity, attempt number, operation counts, and safe terminal reason. diff --git a/internal/core/reporting/markdown.go b/internal/core/reporting/markdown.go index 9c6ed97..7f24818 100644 --- a/internal/core/reporting/markdown.go +++ b/internal/core/reporting/markdown.go @@ -53,6 +53,16 @@ func Markdown(snapshot Snapshot, secrets ...string) []byte { fmt.Fprintln(&b) writeMatrix(&b, snapshot, r, true) fmt.Fprintln(&b) + if len(snapshot.RepairOutcomes) > 0 { + fmt.Fprintln(&b, "## Correction repair") + fmt.Fprintln(&b) + fmt.Fprintln(&b, "| Stage | Chunk | Proposal index | Attempts | Outcome |") + fmt.Fprintln(&b, "| ---: | ---: | ---: | ---: | --- |") + for _, row := range snapshot.RepairOutcomes { + fmt.Fprintf(&b, "| %d | %d | %d | %d | %s |\n", row.StageIndex+1, row.ChunkIndex+1, row.ProposalIndex, row.Attempts, safe(row.Outcome)) + } + fmt.Fprintln(&b) + } fmt.Fprintln(&b, "Legend: `—` = not started; `n*` = interrupted with n recorded errors or recoveries. Completed error cells are 0 or 1.") fmt.Fprintln(&b) writeReasons(&b, "Error reasons", errorReasons(snapshot.Errors), safe) diff --git a/internal/core/reporting/report.go b/internal/core/reporting/report.go index c0d9b8e..23145b7 100644 --- a/internal/core/reporting/report.go +++ b/internal/core/reporting/report.go @@ -129,6 +129,15 @@ type WorkUnit struct { RecoveryCount int } +// RepairOutcome is a compact correction repair row. +type RepairOutcome struct { + StageIndex int + ChunkIndex int + ProposalIndex int + Attempts int + Outcome string +} + // Snapshot is built once from processing state and shared by presentations. type Snapshot struct { RunID string @@ -139,6 +148,7 @@ type Snapshot struct { OutputPath string StageNames []string WorkUnits []WorkUnit + RepairOutcomes []RepairOutcome Summary Summary Errors []ErrorRecord Recoveries []RecoveryRecord diff --git a/internal/framework/outcomes/outcomes.go b/internal/framework/outcomes/outcomes.go index 957b762..cc50bcf 100644 --- a/internal/framework/outcomes/outcomes.go +++ b/internal/framework/outcomes/outcomes.go @@ -7,9 +7,10 @@ package outcomes type OperationKind string const ( - OperationProposal OperationKind = "proposal" - OperationValidation OperationKind = "validation" - OperationValidationInput OperationKind = "validation_input" + OperationProposal OperationKind = "proposal" + OperationCorrectionRepair OperationKind = "correction_repair" + OperationValidation OperationKind = "validation" + OperationValidationInput OperationKind = "validation_input" ) // Reference is the stable processing identity of an operation. Proposal indices @@ -22,6 +23,8 @@ type Reference struct { ModuleInstance string `json:"module_instance"` SectionIndex int `json:"section_index"` Kind OperationKind `json:"operation"` + ProposalIndex *int `json:"proposal_index,omitempty"` + RepairAttempt *int `json:"repair_attempt,omitempty"` ValidatorName string `json:"validator_name,omitempty"` ValidatorOrder *int `json:"validator_order,omitempty"` BatchIndex *int `json:"batch_index,omitempty"` @@ -61,6 +64,7 @@ const ( type RecoveryReason string const ( + RecoveryCorrectionRevised RecoveryReason = "correction_revised" RecoveryOutputRepaired RecoveryReason = "output_repaired" RecoveryServiceRecovered RecoveryReason = "service_recovered" RecoveryProposalsFiltered RecoveryReason = "proposals_filtered" diff --git a/internal/framework/processreport/report.go b/internal/framework/processreport/report.go index aa02fb3..273c43f 100644 --- a/internal/framework/processreport/report.go +++ b/internal/framework/processreport/report.go @@ -123,7 +123,23 @@ func Build(input BuildInput) reporting.Snapshot { StartSegmentID: &start, EndSegmentID: &end, CorrectionsApplied: &applied, Failures: failures}) } snapshot.WorkUnits = append(snapshot.WorkUnits, row) + for _, repair := range unit.Repairs { + if len(repair.Attempts) == 0 { + continue + } + snapshot.RepairOutcomes = append(snapshot.RepairOutcomes, reporting.RepairOutcome{StageIndex: unit.ModuleIndex, ChunkIndex: unit.Section.Index, ProposalIndex: repair.ProposalIndex, Attempts: len(repair.Attempts), Outcome: repair.Outcome}) + } } + sort.SliceStable(snapshot.RepairOutcomes, func(i, j int) bool { + a, b := snapshot.RepairOutcomes[i], snapshot.RepairOutcomes[j] + if a.StageIndex != b.StageIndex { + return a.StageIndex < b.StageIndex + } + if a.ChunkIndex != b.ChunkIndex { + return a.ChunkIndex < b.ChunkIndex + } + return a.ProposalIndex < b.ProposalIndex + }) runErrors := len(input.RunFailures) snapshot.Summary = reporting.Summary{ WorkUnitsTotal: &total, WorkUnitsStarted: &started, WorkUnitsCompleted: &completed, @@ -247,15 +263,31 @@ func operationLess(a, b outcomes.Operation) bool { return 2 } } - if rank(a.Reference.Kind) != rank(b.Reference.Kind) && (a.Reference.Kind == outcomes.OperationProposal || b.Reference.Kind == outcomes.OperationProposal) { - return rank(a.Reference.Kind) < rank(b.Reference.Kind) - } order := func(p *int) int { if p == nil { return -1 } return *p } + aRepair, bRepair := a.Reference.RepairAttempt != nil, b.Reference.RepairAttempt != nil + if aRepair != bRepair { + return !aRepair + } + if aRepair { + if order(a.Reference.ProposalIndex) != order(b.Reference.ProposalIndex) { + return order(a.Reference.ProposalIndex) < order(b.Reference.ProposalIndex) + } + if order(a.Reference.RepairAttempt) != order(b.Reference.RepairAttempt) { + return order(a.Reference.RepairAttempt) < order(b.Reference.RepairAttempt) + } + if (a.Reference.Kind == outcomes.OperationCorrectionRepair) != (b.Reference.Kind == outcomes.OperationCorrectionRepair) { + return a.Reference.Kind == outcomes.OperationCorrectionRepair + } + } + if rank(a.Reference.Kind) != rank(b.Reference.Kind) && (a.Reference.Kind == outcomes.OperationProposal || b.Reference.Kind == outcomes.OperationProposal) { + return rank(a.Reference.Kind) < rank(b.Reference.Kind) + } + if order(a.Reference.ValidatorOrder) != order(b.Reference.ValidatorOrder) { return order(a.Reference.ValidatorOrder) < order(b.Reference.ValidatorOrder) } diff --git a/internal/framework/processreport/report_test.go b/internal/framework/processreport/report_test.go index 333ac9c..b282f35 100644 --- a/internal/framework/processreport/report_test.go +++ b/internal/framework/processreport/report_test.go @@ -2,6 +2,7 @@ package processreport import ( "reflect" + "sort" "strings" "testing" "time" @@ -137,3 +138,54 @@ func TestBuildInitializationNoWorkAndRunFailure(t *testing.T) { } func intPtr(value int) *int { return &value } + +func TestRepairOperationsAndCompactOutcomes(t *testing.T) { + zero, one, two := 0, 1, 2 + operations := []outcomes.Operation{ + {Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ProposalIndex: &two, RepairAttempt: &one, ValidatorOrder: &zero}, Succeeded: true}, + {Reference: outcomes.Reference{Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &two, RepairAttempt: &one}, Succeeded: true, Counts: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 2, StructuralRepairs: 1}, RecoveryReasons: []outcomes.RecoveryReason{outcomes.RecoveryOutputRepaired, outcomes.RecoveryCorrectionRevised}}, + {Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ValidatorOrder: &zero}, Succeeded: true}, + {Reference: outcomes.Reference{Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &one, RepairAttempt: &one}, FailureReason: outcomes.ReasonProviderFailed, ProposalIndices: []int{1}}, + {Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true}, + } + units := []runner.WorkUnitOutcome{{ModuleIndex: 0, Section: contracts.SectionMetadata{Index: 0}, State: outcomes.WorkCompleted, CorrectionsApplied: 1, Operations: operations}} + for i, disposition := range []string{"applied", "approved_skipped", "withdrawn", "rejected", "exhausted", "unavailable", "interrupted"} { + units[0].Repairs = append(units[0].Repairs, runner.RepairOutcome{ProposalIndex: 6 - i, Outcome: disposition, Attempts: []runner.RepairAttempt{{}}}) + } + snapshot := Build(BuildInput{RunID: "r", ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar"}}, RunOutput: &runner.RunOutput{InventoryAvailable: true, WorkUnits: units}}) + if *snapshot.Summary.LLMOperations != 5 || *snapshot.Summary.ProposalResponses != 1 || *snapshot.Summary.Recoveries != 1 || *snapshot.Summary.WorkUnitsWithErrors != 1 || *snapshot.Summary.CorrectionsApplied != 1 { + t.Fatalf("summary: %+v", snapshot.Summary) + } + if len(snapshot.Errors) != 1 || snapshot.Errors[0].Failures[0].Operation != "correction_repair" || len(snapshot.Recoveries) != 1 || !reflect.DeepEqual(snapshot.Recoveries[0].Reasons, []string{"correction_revised", "output_repaired"}) { + t.Fatalf("reports: %+v %+v", snapshot.Errors, snapshot.Recoveries) + } + if len(snapshot.RepairOutcomes) != 7 || snapshot.RepairOutcomes[0].ProposalIndex != 0 || snapshot.RepairOutcomes[6].ProposalIndex != 6 { + t.Fatalf("outcomes: %+v", snapshot.RepairOutcomes) + } + text := string(reporting.Markdown(snapshot)) + for _, disposition := range []string{"applied", "approved_skipped", "withdrawn", "rejected", "exhausted", "unavailable", "interrupted"} { + if !strings.Contains(text, "| "+disposition+" |") { + t.Errorf("missing %s in %s", disposition, text) + } + } + summary := Build(BuildInput{SummaryOnly: true, RunOutput: &runner.RunOutput{InventoryAvailable: true, WorkUnits: units}}) + if !reflect.DeepEqual(summary.Summary, snapshot.Summary) || len(summary.RepairOutcomes) != 0 { + t.Fatalf("summary-only: %+v", summary) + } +} + +func TestRepairOperationOrderingKeepsAttemptsTogether(t *testing.T) { + zero, one, two := 0, 1, 2 + operations := []outcomes.Operation{ + {Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ProposalIndex: &one, RepairAttempt: &two, ValidatorOrder: &zero}}, + {Reference: outcomes.Reference{Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &two, RepairAttempt: &one}}, + {Reference: outcomes.Reference{Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &one, RepairAttempt: &two}}, + {Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ValidatorOrder: &zero}}, + {Reference: outcomes.Reference{Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &one, RepairAttempt: &one}}, + {Reference: outcomes.Reference{Kind: outcomes.OperationProposal}}, + } + sort.SliceStable(operations, func(i, j int) bool { return operationLess(operations[i], operations[j]) }) + if operations[0].Reference.Kind != outcomes.OperationProposal || operations[1].Reference.Kind != outcomes.OperationValidation || *operations[2].Reference.ProposalIndex != 1 || *operations[2].Reference.RepairAttempt != 1 || *operations[3].Reference.RepairAttempt != 2 || operations[3].Reference.Kind != outcomes.OperationCorrectionRepair || operations[4].Reference.Kind != outcomes.OperationValidation || *operations[5].Reference.ProposalIndex != 2 { + t.Fatalf("order: %+v", operations) + } +} diff --git a/internal/framework/runner/repair.go b/internal/framework/runner/repair.go index f267797..0115edf 100644 --- a/internal/framework/runner/repair.go +++ b/internal/framework/runner/repair.go @@ -21,6 +21,7 @@ type RepairAttempt struct { Counts outcomes.ExecutionCounts FailureReason outcomes.ReasonCode Succeeded bool + Approved bool Operations []outcomes.Operation } @@ -137,6 +138,11 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput, record.Revision = &revisionSnapshot record.Decisions = validated.decisions record.Rejected = validated.rejected + for i := range validated.operations { + proposalIndex, attemptNumber := original.ProposalIndex, attempt+1 + validated.operations[i].Reference.ProposalIndex = &proposalIndex + validated.operations[i].Reference.RepairAttempt = &attemptNumber + } record.Operations = validated.operations branch.Attempts = append(branch.Attempts, record) initial.operations = append(initial.operations, validated.operations...) @@ -146,6 +152,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput, return initial, branches, validationErr } if len(validated.approved) == 1 { + branch.Attempts[len(branch.Attempts)-1].Approved = true branch.Outcome = "approved" initial.approved = append(initial.approved, current) retained := initial.filtered[:0] @@ -177,3 +184,25 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput, } return initial, branches, nil } + +func repairOperations(input validateSectionCandidatesInput, branches []RepairOutcome) []outcomes.Operation { + var result []outcomes.Operation + section := 0 + if input.SectionIndex != nil { + section = *input.SectionIndex + } + for _, branch := range branches { + for i, attempt := range branch.Attempts { + index, number := branch.ProposalIndex, i+1 + operation := outcomes.Operation{Reference: outcomes.Reference{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: section, Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &index, RepairAttempt: &number}, Counts: attempt.Counts, Succeeded: attempt.Succeeded, FailureReason: attempt.FailureReason, ProposalIndices: []int{index}} + if attempt.Succeeded { + operation.RecoveryReasons = outcomes.ExecutionRecoveries(attempt.Counts) + if attempt.Approved { + operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryCorrectionRevised) + } + } + result = append(result, operation) + } + } + return result +} diff --git a/internal/framework/runner/repair_test.go b/internal/framework/runner/repair_test.go index 5814d00..72fbfea 100644 --- a/internal/framework/runner/repair_test.go +++ b/internal/framework/runner/repair_test.go @@ -121,6 +121,9 @@ func TestRunnerAppliesRevisionsInOriginalOrder(t *testing.T) { if out.FinalTranscript.Segments[0].Text != "dog" || len(out.WorkUnits) != 1 || len(out.WorkUnits[0].Repairs) != 1 || out.WorkUnits[0].Repairs[0].Outcome != "approved_skipped" || out.WorkUnits[0].CorrectionsApplied != 1 || len(out.ModuleResults[0].SkippedChanges) != 1 || out.ModuleResults[0].SkippedChanges[0].ProposalIndex != 1 { t.Fatalf("output=%+v section=%d", out, sectionIndex) } + if len(out.WorkUnits[0].Operations) != 1 || out.WorkUnits[0].Operations[0].Reference.Kind != outcomes.OperationCorrectionRepair || len(out.WorkUnits[0].Operations[0].RecoveryReasons) != 1 || out.WorkUnits[0].Operations[0].RecoveryReasons[0] != outcomes.RecoveryCorrectionRevised { + t.Fatalf("repair operations: %+v", out.WorkUnits[0].Operations) + } } func TestRepairRevalidatesEarlierApprovalAndRetainsUnrelatedApprovalOnFailure(t *testing.T) { diff --git a/internal/framework/runner/runner.go b/internal/framework/runner/runner.go index dcba057..b9b5e1e 100644 --- a/internal/framework/runner/runner.go +++ b/internal/framework/runner/runner.go @@ -564,6 +564,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) } repairInput := validateSectionCandidatesInput{Spec: input.Spec, Policy: input.Policy, WorkingTranscript: transcriptFromSection(sectionResult.section), SectionIndex: §ionMetadata.Index, Validators: validatorsOrdered, SectionEnriched: enriched, Exchange: sectionResult.exchange, ValidationLLMClient: input.ValidationClient, DiagnosticsDir: input.ValidationDiagnosticsDir, GlossaryView: input.GlossaryView, ReportDiagnosticError: input.ReportDiagnosticError} validated, repairs, err = repairCandidates(runCtx, repairInput, sectionResult.references, validated, promptID) + validated.operations = append(validated.operations, repairOperations(repairInput, repairs)...) } setErr(err) validationResults <- sectionValidationResult{