From 04e0c2f913e2ad83733d552e662ba4f8764d7bb2 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Wed, 11 Nov 2026 12:31:08 +0000 Subject: [PATCH] Keep detailed correction repair history in debug runs --- docs/roadmap/remediation.md | 4 +- internal/framework/processreport/report.go | 4 +- .../framework/processreport/report_test.go | 2 +- internal/framework/runner/repair.go | 132 ++++++++---------- internal/framework/runner/repair_test.go | 4 +- internal/framework/runner/runner.go | 1 - 6 files changed, 66 insertions(+), 81 deletions(-) diff --git a/docs/roadmap/remediation.md b/docs/roadmap/remediation.md index 365731b..1d4eab7 100644 --- a/docs/roadmap/remediation.md +++ b/docs/roadmap/remediation.md @@ -2,7 +2,7 @@ **Updated:** 2026-11-11 -**Status:** 6 of nine findings resolved. Findings refer to implementation revision +**Status:** 7 of nine findings resolved. Findings refer to implementation revision `aa8073c`. See the [implementation plan](implementation.md). ## Review Result @@ -240,6 +240,8 @@ skipped by the existing conflict checks. ### R7 — Keep Detailed Repair History in Debug Runs (P3) +**Completed.** Ordinary runs retain a compact attempt count and disposition. Debug runs retain the revision history. Runner, reporting, and CLI tests pass with debug on and off. + **Affected plan:** Stage 7, item 3, and Stage 8, items 2–3. **Evidence:** [repairCandidates](../../internal/framework/runner/repair.go#L178) diff --git a/internal/framework/processreport/report.go b/internal/framework/processreport/report.go index 273c43f..204055f 100644 --- a/internal/framework/processreport/report.go +++ b/internal/framework/processreport/report.go @@ -124,10 +124,10 @@ func Build(input BuildInput) reporting.Snapshot { } snapshot.WorkUnits = append(snapshot.WorkUnits, row) for _, repair := range unit.Repairs { - if len(repair.Attempts) == 0 { + if repair.AttemptCount == 0 { continue } - snapshot.RepairOutcomes = append(snapshot.RepairOutcomes, reporting.RepairOutcome{StageIndex: unit.ModuleIndex, ChunkIndex: unit.Section.Index, ProposalIndex: repair.ProposalIndex, Attempts: len(repair.Attempts), Outcome: repair.Outcome}) + snapshot.RepairOutcomes = append(snapshot.RepairOutcomes, reporting.RepairOutcome{StageIndex: unit.ModuleIndex, ChunkIndex: unit.Section.Index, ProposalIndex: repair.ProposalIndex, Attempts: repair.AttemptCount, Outcome: repair.Outcome}) } } sort.SliceStable(snapshot.RepairOutcomes, func(i, j int) bool { diff --git a/internal/framework/processreport/report_test.go b/internal/framework/processreport/report_test.go index b282f35..28b4c0c 100644 --- a/internal/framework/processreport/report_test.go +++ b/internal/framework/processreport/report_test.go @@ -150,7 +150,7 @@ func TestRepairOperationsAndCompactOutcomes(t *testing.T) { } units := []runner.WorkUnitOutcome{{ModuleIndex: 0, Section: contracts.SectionMetadata{Index: 0}, State: outcomes.WorkCompleted, CorrectionsApplied: 1, Operations: operations}} for i, disposition := range []string{"applied", "approved_skipped", "withdrawn", "rejected", "exhausted", "unavailable", "interrupted"} { - units[0].Repairs = append(units[0].Repairs, runner.RepairOutcome{ProposalIndex: 6 - i, Outcome: disposition, Attempts: []runner.RepairAttempt{{}}}) + units[0].Repairs = append(units[0].Repairs, runner.RepairOutcome{ProposalIndex: 6 - i, Outcome: disposition, AttemptCount: 1}) } snapshot := Build(BuildInput{RunID: "r", ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar"}}, RunOutput: &runner.RunOutput{InventoryAvailable: true, WorkUnits: units}}) if *snapshot.Summary.LLMOperations != 5 || *snapshot.Summary.ProposalResponses != 1 || *snapshot.Summary.Recoveries != 1 || *snapshot.Summary.WorkUnitsWithErrors != 1 || *snapshot.Summary.CorrectionsApplied != 1 { diff --git a/internal/framework/runner/repair.go b/internal/framework/runner/repair.go index 3e5db13..c3e4c87 100644 --- a/internal/framework/runner/repair.go +++ b/internal/framework/runner/repair.go @@ -18,25 +18,20 @@ import ( "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators" ) -// RepairAttempt retains compact facts for a single semantic request. +// RepairAttempt holds optional debug evidence for one semantic request. type RepairAttempt struct { Artifacts llm.InteractionArtifacts Revision *proposals.EnrichedCorrectionProposal Decisions []ValidatorDecisionRecord - Rejected []ValidatorRejectedChange - Counts outcomes.ExecutionCounts FailureReason outcomes.ReasonCode - Succeeded bool - Approved bool - Operations []outcomes.Operation - Filtered []outcomes.FilteredProposal } -// RepairOutcome keeps the proposal identity through revision and application. +// RepairOutcome summarizes the branch and carries its lineage when debug is enabled. type RepairOutcome struct { ProposalIndex int - Attempts []RepairAttempt + AttemptCount int Outcome string + Attempts []RepairAttempt Original proposals.EnrichedCorrectionProposal Final proposals.EnrichedCorrectionProposal } @@ -68,12 +63,19 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput, } superseded := make(map[int]bool) var branches []RepairOutcome + var repairOps []outcomes.Operation + debug := input.DiagnosticsDir != "" + section := 0 + if input.SectionIndex != nil { + section = *input.SectionIndex + } finish := func(err error) (validateSectionCandidatesResult, []RepairOutcome, error) { if len(superseded) > 0 { initial.filtered = slices.DeleteFunc(initial.filtered, func(item outcomes.FilteredProposal) bool { return superseded[item.ProposalIndex] }) } + initial.operations = append(initial.operations, repairOps...) return initial, branches, err } @@ -82,7 +84,10 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput, if !actionable(original.ProposalIndex, rejected) { continue } - branch := RepairOutcome{ProposalIndex: original.ProposalIndex, Original: original, Final: original, Outcome: "interrupted"} + branch := RepairOutcome{ProposalIndex: original.ProposalIndex, Outcome: "interrupted"} + if debug { + branch.Original, branch.Final = original, original + } if err := ctx.Err(); err != nil { branches = append(branches, branch) return finish(err) @@ -122,40 +127,47 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput, var response proposal_generation.RepairResponse proposalIndex, attemptNumber := original.ProposalIndex, attempt+1 completion, callErr := input.Spec.RepairClient.CompleteStructured(ctx, structuredoutput.Request{PromptID: promptID, Variables: map[string]string{"RepairRequestJSON": payload}, Schema: responseschema.CorrectionRepairKey, Replay: replay, Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex, ProposalIndex: &proposalIndex, RepairAttempt: &attemptNumber}}, &response) - record := RepairAttempt{Counts: completion.Execution} - save := func(err error) { - if input.DiagnosticsDir == "" { + operation := outcomes.Operation{ + Reference: outcomes.Reference{ + StageIndex: &input.Spec.Index, StageID: input.Spec.ID, ModuleKey: input.Spec.ModuleKey, + ModuleInstance: input.Spec.InstanceName, SectionIndex: section, Kind: outcomes.OperationCorrectionRepair, + ProposalIndex: &proposalIndex, RepairAttempt: &attemptNumber, + }, + Counts: completion.Execution, ProposalIndices: []int{proposalIndex}, + } + finishAttempt := func(err error, revision *proposals.EnrichedCorrectionProposal, validated validateSectionCandidatesResult) { + branch.AttemptCount++ + repairOps = append(repairOps, operation) + if !debug { return } - secrets := repairSecrets(ctx, input) - section := 0 - if input.SectionIndex != nil { - section = *input.SectionIndex + record := RepairAttempt{Decisions: validated.decisions, FailureReason: operation.FailureReason} + if revision != nil { + snapshot := *revision + record.Revision = &snapshot } name := fmt.Sprintf("%s:section-%04d:proposal-%04d:attempt-%04d:repair", input.Spec.InstanceName, section, proposalIndex, attemptNumber) var failure any if err != nil { failure = map[string]string{"error": err.Error()} } - writer := llm.NewDiagnosticsWriter(filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName), secrets) - artifacts, writeErr := writer.WriteInteraction(name, map[string]any{"stage_index": input.Spec.Index, "stage_id": input.Spec.ID, "proposal_index": proposalIndex, "repair_attempt": attemptNumber, "prompt_id": promptID, "prompt": completion.Prompt, "feedback": findings}, completion.Prepared, map[string]any{"decoded": response, "final_content": completion.FinalContent, "revision": record.Revision, "decisions": record.Decisions, "rejections": record.Rejected}, failure) - if writeErr != nil { - if input.ReportDiagnosticError != nil { - input.ReportDiagnosticError(fmt.Errorf("repair interaction %s: %w", name, writeErr)) - } - return + writer := llm.NewDiagnosticsWriter(filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName), repairSecrets(ctx, input)) + artifacts, writeErr := writer.WriteInteraction(name, map[string]any{"stage_index": input.Spec.Index, "stage_id": input.Spec.ID, "proposal_index": proposalIndex, "repair_attempt": attemptNumber, "prompt_id": promptID, "prompt": completion.Prompt, "feedback": findings}, completion.Prepared, map[string]any{"decoded": response, "final_content": completion.FinalContent, "revision": record.Revision, "decisions": record.Decisions, "rejections": validated.rejected}, failure) + if writeErr == nil { + record.Artifacts = artifacts + } else if input.ReportDiagnosticError != nil { + input.ReportDiagnosticError(fmt.Errorf("repair interaction %s: %w", name, writeErr)) } - record.Artifacts = artifacts + branch.Attempts = append(branch.Attempts, record) } if callErr != nil { var failure *structuredoutput.Failure if errors.As(callErr, &failure) { - record.FailureReason = failure.Reason + operation.FailureReason = failure.Reason } else { - record.FailureReason = outcomes.ReasonInternalFailure + operation.FailureReason = outcomes.ReasonInternalFailure } - save(callErr) - branch.Attempts = append(branch.Attempts, record) + finishAttempt(callErr, nil, validateSectionCandidatesResult{}) if structuredoutput.IsRecoverable(callErr) { branch.Outcome = "unavailable" break @@ -165,18 +177,17 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput, } decoded, err := proposal_generation.DecodeRepairResponse(completion.Content, position+1) if err != nil { - record.FailureReason = outcomes.ReasonInvalidOutput - save(err) - branch.Attempts = append(branch.Attempts, record) + operation.FailureReason = outcomes.ReasonInvalidOutput + finishAttempt(err, nil, validateSectionCandidatesResult{}) branch.Outcome = "unavailable" break } - record.Succeeded = true + operation.Succeeded = true + operation.RecoveryReasons = outcomes.ExecutionRecoveries(operation.Counts) replay.Turns = append(replay.Turns, completion.RepairInstruction...) replay.Turns = append(replay.Turns, structuredoutput.Message{Role: "assistant", Content: completion.FinalContent}) if decoded.Action == "withdraw" { - save(nil) - branch.Attempts = append(branch.Attempts, record) + finishAttempt(nil, nil, validateSectionCandidatesResult{}) branch.Outcome = "withdrawn" break } @@ -188,35 +199,34 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput, } current = original current.CorrectionProposal = proposals.CorrectionProposal{TargetSegmentID: target, OriginalText: revision.OriginalText, CorrectedText: revision.CorrectedText, Confidence: revision.Confidence} - branch.Final = current + if debug { + branch.Final = current + } next := input next.RepairProposalIndex, next.RepairAttempt = &proposalIndex, &attemptNumber next.SectionEnriched = []proposals.EnrichedCorrectionProposal{current} validated, validationErr := validateSectionCandidates(ctx, next) - revisionSnapshot := current - record.Revision = &revisionSnapshot - record.Decisions = validated.decisions - record.Rejected = validated.rejected if len(validated.filtered) > 0 { - // A repair response contains one proposal. Attribute its first - // defect once, as for proposals in the original response. - record.Filtered = validated.filtered[:1] + // Each repair response contains only one proposal. + operation.Filtered = validated.filtered[:1] + operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryProposalsFiltered) } for i := range validated.operations { proposalIndex, attemptNumber := original.ProposalIndex, attempt+1 validated.operations[i].Reference.ProposalIndex = &proposalIndex validated.operations[i].Reference.RepairAttempt = &attemptNumber } - record.Operations = validated.operations - save(validationErr) - branch.Attempts = append(branch.Attempts, record) + approved := validationErr == nil && len(validated.approved) == 1 + if approved { + operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryCorrectionRevised) + } + finishAttempt(validationErr, ¤t, validated) initial.operations = append(initial.operations, validated.operations...) if validationErr != nil { branches = append(branches, branch) return finish(validationErr) } - if len(validated.approved) == 1 { - branch.Attempts[len(branch.Attempts)-1].Approved = true + if approved { branch.Outcome = "approved" initial.approved = append(initial.approved, current) superseded[original.ProposalIndex] = true @@ -244,32 +254,6 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput, return finish(nil) } -func repairOperations(input validateSectionCandidatesInput, branches []RepairOutcome) []outcomes.Operation { - var result []outcomes.Operation - section := 0 - if input.SectionIndex != nil { - section = *input.SectionIndex - } - for _, branch := range branches { - for i, attempt := range branch.Attempts { - index, number := branch.ProposalIndex, i+1 - operation := outcomes.Operation{Reference: outcomes.Reference{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: section, Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &index, RepairAttempt: &number}, Counts: attempt.Counts, Succeeded: attempt.Succeeded, FailureReason: attempt.FailureReason, ProposalIndices: []int{index}} - if attempt.Succeeded { - operation.RecoveryReasons = outcomes.ExecutionRecoveries(attempt.Counts) - if len(attempt.Filtered) > 0 { - operation.Filtered = attempt.Filtered - operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryProposalsFiltered) - } - if attempt.Approved { - operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryCorrectionRevised) - } - } - result = append(result, operation) - } - } - return result -} - func repairSecrets(ctx context.Context, input validateSectionCandidatesInput) []string { var secrets []string for _, client := range []contracts.StructuredLLMClient{input.Spec.RepairClient, input.ValidationLLMClient} { diff --git a/internal/framework/runner/repair_test.go b/internal/framework/runner/repair_test.go index 3c0a334..fd37f25 100644 --- a/internal/framework/runner/repair_test.go +++ b/internal/framework/runner/repair_test.go @@ -99,7 +99,7 @@ func TestRepairBranchTerminalOutcomes(t *testing.T) { if err != nil { t.Fatal(err) } - if len(branches) != 1 || branches[0].Outcome != tc.outcome || len(client.calls) != tc.calls { + if len(branches) != 1 || branches[0].Outcome != tc.outcome || len(client.calls) != tc.calls || branches[0].AttemptCount != tc.calls || len(branches[0].Attempts) != 0 { t.Fatalf("branches=%+v calls=%d", branches, len(client.calls)) } if tc.calls == 2 && len(client.calls[1].Replay.Turns) != 2 { @@ -192,7 +192,7 @@ func TestRepairInterruptionRetainsAttempt(t *testing.T) { t.Fatal(err) } _, branches, err := repairCandidates(context.Background(), input, []int{1}, initial, "repair") - if !errors.Is(err, context.Canceled) || len(branches) != 1 || branches[0].Outcome != "interrupted" || len(branches[0].Attempts) != 1 { + if !errors.Is(err, context.Canceled) || len(branches) != 1 || branches[0].Outcome != "interrupted" || branches[0].AttemptCount != 1 || len(branches[0].Attempts) != 0 { t.Fatalf("branches=%+v error=%v", branches, err) } } diff --git a/internal/framework/runner/runner.go b/internal/framework/runner/runner.go index 62ae30f..d370895 100644 --- a/internal/framework/runner/runner.go +++ b/internal/framework/runner/runner.go @@ -571,7 +571,6 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) promptID = provider.RepairPromptID() } validated, repairs, err = repairCandidates(runCtx, validationInput, sectionResult.references, validated, promptID) - validated.operations = append(validated.operations, repairOperations(validationInput, repairs)...) } setErr(err) validationResults <- sectionValidationResult{