Report correction repair operations and outcomes

This commit is contained in:
2026-11-11 04:31:44 +00:00
parent d927c61199
commit 735dfc1834
10 changed files with 159 additions and 10 deletions

View File

@@ -48,7 +48,7 @@ remain in opt-in debug files.
An `errors` record with `scope: work_unit` identifies zero-based `stage_index`
and `chunk_index`, optional `stage_id`, module key/instance, initial segment range, applied-correction
count, and a `failures` sequence. Each failure identifies the operation
(`proposal`, `validation`, or `validation_input`), optional validator and
(`proposal`, `validation`, `validation_input`, or `correction_repair`), optional validator and
zero-based batch index, safe reason/message, sorted proposal indices when
known, and execution counts when available. Multiple failed batches in one
work unit form one error record. `scope: run` instead records phase, safe reason
@@ -62,7 +62,9 @@ may still be approved or rejected normally.
Each recovery identifies its work unit, optional `stage_id`, and successful operation, with sorted
`reasons`, a safe message, available execution counts, and sorted
`filtered_proposals` details. One recovery can contain `output_repaired`,
`service_recovered`, and `proposals_filtered` together. Filtering covers
`service_recovered`, `correction_revised`, and `proposals_filtered` together.
`correction_revised` means a repair yielded a fully validated revision; its
application outcome is recorded in the Markdown report. Filtering covers
defective response proposals; low confidence, protected-term rejection,
ordinary editorial rejection, and an apply-time conflict caused by an earlier
accepted correction are not filtering recoveries. Failed operations are errors,
@@ -84,8 +86,12 @@ the one-based position and module, plus the operator ID when supplied. Completed
counts. `—` means not started and `n*` means interrupted with n facts recorded.
An empty stage or chunk set is identified as having no work units. Short reason breakdowns count
records containing each reason; a recovery can have several reasons, so those
counts need not sum to the recovery total. The report refers to both YAML
files. It does not include transcript excerpts or correction lists.
counts need not sum to the recovery total. When a repair was attempted, a
Correction repair table lists its stage, chunk,
original proposal index, semantic attempt count, and final outcome (`applied`,
`approved_skipped`, `withdrawn`, `rejected`, `exhausted`, `unavailable`, or
`interrupted`). The report refers to both YAML files. It does not include
transcript excerpts or correction lists.
## Debug-Only Files

View File

@@ -276,6 +276,8 @@ validator responses remains [deferred](future.md#domain-aware-response-recovery)
## Stage 7 — Account for operations and publish compact ordinary outcomes
**Completed.**
1. Add `correction_repair` as an operation kind in
`internal/framework/outcomes`. For each repair call retain its proposal
identity, attempt number, operation counts, and safe terminal reason.

View File

@@ -53,6 +53,16 @@ func Markdown(snapshot Snapshot, secrets ...string) []byte {
fmt.Fprintln(&b)
writeMatrix(&b, snapshot, r, true)
fmt.Fprintln(&b)
if len(snapshot.RepairOutcomes) > 0 {
fmt.Fprintln(&b, "## Correction repair")
fmt.Fprintln(&b)
fmt.Fprintln(&b, "| Stage | Chunk | Proposal index | Attempts | Outcome |")
fmt.Fprintln(&b, "| ---: | ---: | ---: | ---: | --- |")
for _, row := range snapshot.RepairOutcomes {
fmt.Fprintf(&b, "| %d | %d | %d | %d | %s |\n", row.StageIndex+1, row.ChunkIndex+1, row.ProposalIndex, row.Attempts, safe(row.Outcome))
}
fmt.Fprintln(&b)
}
fmt.Fprintln(&b, "Legend: `—` = not started; `n*` = interrupted with n recorded errors or recoveries. Completed error cells are 0 or 1.")
fmt.Fprintln(&b)
writeReasons(&b, "Error reasons", errorReasons(snapshot.Errors), safe)

View File

@@ -129,6 +129,15 @@ type WorkUnit struct {
RecoveryCount int
}
// RepairOutcome is a compact correction repair row.
type RepairOutcome struct {
StageIndex int
ChunkIndex int
ProposalIndex int
Attempts int
Outcome string
}
// Snapshot is built once from processing state and shared by presentations.
type Snapshot struct {
RunID string
@@ -139,6 +148,7 @@ type Snapshot struct {
OutputPath string
StageNames []string
WorkUnits []WorkUnit
RepairOutcomes []RepairOutcome
Summary Summary
Errors []ErrorRecord
Recoveries []RecoveryRecord

View File

@@ -7,9 +7,10 @@ package outcomes
type OperationKind string
const (
OperationProposal OperationKind = "proposal"
OperationValidation OperationKind = "validation"
OperationValidationInput OperationKind = "validation_input"
OperationProposal OperationKind = "proposal"
OperationCorrectionRepair OperationKind = "correction_repair"
OperationValidation OperationKind = "validation"
OperationValidationInput OperationKind = "validation_input"
)
// Reference is the stable processing identity of an operation. Proposal indices
@@ -22,6 +23,8 @@ type Reference struct {
ModuleInstance string `json:"module_instance"`
SectionIndex int `json:"section_index"`
Kind OperationKind `json:"operation"`
ProposalIndex *int `json:"proposal_index,omitempty"`
RepairAttempt *int `json:"repair_attempt,omitempty"`
ValidatorName string `json:"validator_name,omitempty"`
ValidatorOrder *int `json:"validator_order,omitempty"`
BatchIndex *int `json:"batch_index,omitempty"`
@@ -61,6 +64,7 @@ const (
type RecoveryReason string
const (
RecoveryCorrectionRevised RecoveryReason = "correction_revised"
RecoveryOutputRepaired RecoveryReason = "output_repaired"
RecoveryServiceRecovered RecoveryReason = "service_recovered"
RecoveryProposalsFiltered RecoveryReason = "proposals_filtered"

View File

@@ -123,7 +123,23 @@ func Build(input BuildInput) reporting.Snapshot {
StartSegmentID: &start, EndSegmentID: &end, CorrectionsApplied: &applied, Failures: failures})
}
snapshot.WorkUnits = append(snapshot.WorkUnits, row)
for _, repair := range unit.Repairs {
if len(repair.Attempts) == 0 {
continue
}
snapshot.RepairOutcomes = append(snapshot.RepairOutcomes, reporting.RepairOutcome{StageIndex: unit.ModuleIndex, ChunkIndex: unit.Section.Index, ProposalIndex: repair.ProposalIndex, Attempts: len(repair.Attempts), Outcome: repair.Outcome})
}
}
sort.SliceStable(snapshot.RepairOutcomes, func(i, j int) bool {
a, b := snapshot.RepairOutcomes[i], snapshot.RepairOutcomes[j]
if a.StageIndex != b.StageIndex {
return a.StageIndex < b.StageIndex
}
if a.ChunkIndex != b.ChunkIndex {
return a.ChunkIndex < b.ChunkIndex
}
return a.ProposalIndex < b.ProposalIndex
})
runErrors := len(input.RunFailures)
snapshot.Summary = reporting.Summary{
WorkUnitsTotal: &total, WorkUnitsStarted: &started, WorkUnitsCompleted: &completed,
@@ -247,15 +263,31 @@ func operationLess(a, b outcomes.Operation) bool {
return 2
}
}
if rank(a.Reference.Kind) != rank(b.Reference.Kind) && (a.Reference.Kind == outcomes.OperationProposal || b.Reference.Kind == outcomes.OperationProposal) {
return rank(a.Reference.Kind) < rank(b.Reference.Kind)
}
order := func(p *int) int {
if p == nil {
return -1
}
return *p
}
aRepair, bRepair := a.Reference.RepairAttempt != nil, b.Reference.RepairAttempt != nil
if aRepair != bRepair {
return !aRepair
}
if aRepair {
if order(a.Reference.ProposalIndex) != order(b.Reference.ProposalIndex) {
return order(a.Reference.ProposalIndex) < order(b.Reference.ProposalIndex)
}
if order(a.Reference.RepairAttempt) != order(b.Reference.RepairAttempt) {
return order(a.Reference.RepairAttempt) < order(b.Reference.RepairAttempt)
}
if (a.Reference.Kind == outcomes.OperationCorrectionRepair) != (b.Reference.Kind == outcomes.OperationCorrectionRepair) {
return a.Reference.Kind == outcomes.OperationCorrectionRepair
}
}
if rank(a.Reference.Kind) != rank(b.Reference.Kind) && (a.Reference.Kind == outcomes.OperationProposal || b.Reference.Kind == outcomes.OperationProposal) {
return rank(a.Reference.Kind) < rank(b.Reference.Kind)
}
if order(a.Reference.ValidatorOrder) != order(b.Reference.ValidatorOrder) {
return order(a.Reference.ValidatorOrder) < order(b.Reference.ValidatorOrder)
}

View File

@@ -2,6 +2,7 @@ package processreport
import (
"reflect"
"sort"
"strings"
"testing"
"time"
@@ -137,3 +138,54 @@ func TestBuildInitializationNoWorkAndRunFailure(t *testing.T) {
}
func intPtr(value int) *int { return &value }
func TestRepairOperationsAndCompactOutcomes(t *testing.T) {
zero, one, two := 0, 1, 2
operations := []outcomes.Operation{
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ProposalIndex: &two, RepairAttempt: &one, ValidatorOrder: &zero}, Succeeded: true},
{Reference: outcomes.Reference{Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &two, RepairAttempt: &one}, Succeeded: true, Counts: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 2, StructuralRepairs: 1}, RecoveryReasons: []outcomes.RecoveryReason{outcomes.RecoveryOutputRepaired, outcomes.RecoveryCorrectionRevised}},
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ValidatorOrder: &zero}, Succeeded: true},
{Reference: outcomes.Reference{Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &one, RepairAttempt: &one}, FailureReason: outcomes.ReasonProviderFailed, ProposalIndices: []int{1}},
{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true},
}
units := []runner.WorkUnitOutcome{{ModuleIndex: 0, Section: contracts.SectionMetadata{Index: 0}, State: outcomes.WorkCompleted, CorrectionsApplied: 1, Operations: operations}}
for i, disposition := range []string{"applied", "approved_skipped", "withdrawn", "rejected", "exhausted", "unavailable", "interrupted"} {
units[0].Repairs = append(units[0].Repairs, runner.RepairOutcome{ProposalIndex: 6 - i, Outcome: disposition, Attempts: []runner.RepairAttempt{{}}})
}
snapshot := Build(BuildInput{RunID: "r", ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar"}}, RunOutput: &runner.RunOutput{InventoryAvailable: true, WorkUnits: units}})
if *snapshot.Summary.LLMOperations != 5 || *snapshot.Summary.ProposalResponses != 1 || *snapshot.Summary.Recoveries != 1 || *snapshot.Summary.WorkUnitsWithErrors != 1 || *snapshot.Summary.CorrectionsApplied != 1 {
t.Fatalf("summary: %+v", snapshot.Summary)
}
if len(snapshot.Errors) != 1 || snapshot.Errors[0].Failures[0].Operation != "correction_repair" || len(snapshot.Recoveries) != 1 || !reflect.DeepEqual(snapshot.Recoveries[0].Reasons, []string{"correction_revised", "output_repaired"}) {
t.Fatalf("reports: %+v %+v", snapshot.Errors, snapshot.Recoveries)
}
if len(snapshot.RepairOutcomes) != 7 || snapshot.RepairOutcomes[0].ProposalIndex != 0 || snapshot.RepairOutcomes[6].ProposalIndex != 6 {
t.Fatalf("outcomes: %+v", snapshot.RepairOutcomes)
}
text := string(reporting.Markdown(snapshot))
for _, disposition := range []string{"applied", "approved_skipped", "withdrawn", "rejected", "exhausted", "unavailable", "interrupted"} {
if !strings.Contains(text, "| "+disposition+" |") {
t.Errorf("missing %s in %s", disposition, text)
}
}
summary := Build(BuildInput{SummaryOnly: true, RunOutput: &runner.RunOutput{InventoryAvailable: true, WorkUnits: units}})
if !reflect.DeepEqual(summary.Summary, snapshot.Summary) || len(summary.RepairOutcomes) != 0 {
t.Fatalf("summary-only: %+v", summary)
}
}
func TestRepairOperationOrderingKeepsAttemptsTogether(t *testing.T) {
zero, one, two := 0, 1, 2
operations := []outcomes.Operation{
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ProposalIndex: &one, RepairAttempt: &two, ValidatorOrder: &zero}},
{Reference: outcomes.Reference{Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &two, RepairAttempt: &one}},
{Reference: outcomes.Reference{Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &one, RepairAttempt: &two}},
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ValidatorOrder: &zero}},
{Reference: outcomes.Reference{Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &one, RepairAttempt: &one}},
{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}},
}
sort.SliceStable(operations, func(i, j int) bool { return operationLess(operations[i], operations[j]) })
if operations[0].Reference.Kind != outcomes.OperationProposal || operations[1].Reference.Kind != outcomes.OperationValidation || *operations[2].Reference.ProposalIndex != 1 || *operations[2].Reference.RepairAttempt != 1 || *operations[3].Reference.RepairAttempt != 2 || operations[3].Reference.Kind != outcomes.OperationCorrectionRepair || operations[4].Reference.Kind != outcomes.OperationValidation || *operations[5].Reference.ProposalIndex != 2 {
t.Fatalf("order: %+v", operations)
}
}

View File

@@ -21,6 +21,7 @@ type RepairAttempt struct {
Counts outcomes.ExecutionCounts
FailureReason outcomes.ReasonCode
Succeeded bool
Approved bool
Operations []outcomes.Operation
}
@@ -137,6 +138,11 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
record.Revision = &revisionSnapshot
record.Decisions = validated.decisions
record.Rejected = validated.rejected
for i := range validated.operations {
proposalIndex, attemptNumber := original.ProposalIndex, attempt+1
validated.operations[i].Reference.ProposalIndex = &proposalIndex
validated.operations[i].Reference.RepairAttempt = &attemptNumber
}
record.Operations = validated.operations
branch.Attempts = append(branch.Attempts, record)
initial.operations = append(initial.operations, validated.operations...)
@@ -146,6 +152,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
return initial, branches, validationErr
}
if len(validated.approved) == 1 {
branch.Attempts[len(branch.Attempts)-1].Approved = true
branch.Outcome = "approved"
initial.approved = append(initial.approved, current)
retained := initial.filtered[:0]
@@ -177,3 +184,25 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
}
return initial, branches, nil
}
func repairOperations(input validateSectionCandidatesInput, branches []RepairOutcome) []outcomes.Operation {
var result []outcomes.Operation
section := 0
if input.SectionIndex != nil {
section = *input.SectionIndex
}
for _, branch := range branches {
for i, attempt := range branch.Attempts {
index, number := branch.ProposalIndex, i+1
operation := outcomes.Operation{Reference: outcomes.Reference{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: section, Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &index, RepairAttempt: &number}, Counts: attempt.Counts, Succeeded: attempt.Succeeded, FailureReason: attempt.FailureReason, ProposalIndices: []int{index}}
if attempt.Succeeded {
operation.RecoveryReasons = outcomes.ExecutionRecoveries(attempt.Counts)
if attempt.Approved {
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryCorrectionRevised)
}
}
result = append(result, operation)
}
}
return result
}

View File

@@ -121,6 +121,9 @@ func TestRunnerAppliesRevisionsInOriginalOrder(t *testing.T) {
if out.FinalTranscript.Segments[0].Text != "dog" || len(out.WorkUnits) != 1 || len(out.WorkUnits[0].Repairs) != 1 || out.WorkUnits[0].Repairs[0].Outcome != "approved_skipped" || out.WorkUnits[0].CorrectionsApplied != 1 || len(out.ModuleResults[0].SkippedChanges) != 1 || out.ModuleResults[0].SkippedChanges[0].ProposalIndex != 1 {
t.Fatalf("output=%+v section=%d", out, sectionIndex)
}
if len(out.WorkUnits[0].Operations) != 1 || out.WorkUnits[0].Operations[0].Reference.Kind != outcomes.OperationCorrectionRepair || len(out.WorkUnits[0].Operations[0].RecoveryReasons) != 1 || out.WorkUnits[0].Operations[0].RecoveryReasons[0] != outcomes.RecoveryCorrectionRevised {
t.Fatalf("repair operations: %+v", out.WorkUnits[0].Operations)
}
}
func TestRepairRevalidatesEarlierApprovalAndRetainsUnrelatedApprovalOnFailure(t *testing.T) {

View File

@@ -564,6 +564,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
}
repairInput := validateSectionCandidatesInput{Spec: input.Spec, Policy: input.Policy, WorkingTranscript: transcriptFromSection(sectionResult.section), SectionIndex: &sectionMetadata.Index, Validators: validatorsOrdered, SectionEnriched: enriched, Exchange: sectionResult.exchange, ValidationLLMClient: input.ValidationClient, DiagnosticsDir: input.ValidationDiagnosticsDir, GlossaryView: input.GlossaryView, ReportDiagnosticError: input.ReportDiagnosticError}
validated, repairs, err = repairCandidates(runCtx, repairInput, sectionResult.references, validated, promptID)
validated.operations = append(validated.operations, repairOperations(repairInput, repairs)...)
}
setErr(err)
validationResults <- sectionValidationResult{