eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Retain repair diagnostics and correction lineage

Browse Source
This commit is contained in:
Eric Rakestraw
2026-11-11 04:37:52 +00:00
parent 735dfc1834
commit f4c638313e
15 changed files with 308 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
internal/framework/processreport/correction_ledger.go
View File

@@ -4,6 +4,7 @@ import (
"path/filepath"
"sort"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
@@ -33,6 +34,9 @@ type CorrectionLedgerEntry struct {
SegmentID int `json:"segment_id,omitempty"`
OriginalText string `json:"original_text,omitempty"`
ProposedCorrectedText *string `json:"proposed_corrected_text,omitempty"`
AppliedSegmentID *int `json:"applied_segment_id,omitempty"`
AppliedOriginalText *string `json:"applied_original_text,omitempty"`
RepairAttempts []LedgerRepairAttempt `json:"repair_attempts,omitempty"`
AppliedCorrectedText *string `json:"applied_corrected_text,omitempty"`
ReplacementPolicy string `json:"replacement_policy,omitempty"`
Disposition string `json:"disposition"`
@@ -42,7 +46,20 @@ type CorrectionLedgerEntry struct {
LLMValidatorResults []LedgerValidatorDecisionRecord `json:"llm_validator_decisions,omitempty"`
}
type LedgerRepairAttempt struct {
Attempt int `json:"attempt"`
Revision *proposals.EnrichedCorrectionProposal `json:"revision,omitempty"`
FailureReason string `json:"failure_reason,omitempty"`
RequestMetadataPath string `json:"request_metadata_path,omitempty"`
RequestPayloadPath string `json:"request_payload_path,omitempty"`
ResponsePayloadPath string `json:"response_payload_path,omitempty"`
ErrorPayloadPath string `json:"error_payload_path,omitempty"`
DeterministicValidatorResults []LedgerValidatorDecisionRecord `json:"deterministic_validator_decisions,omitempty"`
LLMValidatorResults []LedgerValidatorDecisionRecord `json:"llm_validator_decisions,omitempty"`
}
type LedgerValidatorDecisionRecord struct {
ArtifactPath string `json:"artifact_path,omitempty"`
ValidatorKey string `json:"validator_key"`
Approved bool `json:"approved"`
Disposition validators.DecisionDisposition `json:"disposition"`
@@ -71,6 +88,12 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry
policy string
decisions map[int][]runner.ValidatorDecisionRecord
}
repairs := make(map[proposalIdentity]runner.RepairOutcome)
for _, unit := range runOutput.WorkUnits {
for _, repair := range unit.Repairs {
repairs[proposalIdentity{unit.ModuleInstance, repair.ProposalIndex}] = repair
}
}
seen := make(map[proposalIdentity]bool)
stages := make(map[string]stageEvidence, len(runOutput.ModuleResults))
for _, module := range runOutput.ModuleResults {
@@ -85,6 +108,9 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry
}
for _, change := range module.AppliedChanges {
if _, ok := repairs[proposalIdentity{module.ModuleInstance, change.ProposalIndex}]; ok {
continue
}
seen[proposalIdentity{module.ModuleInstance, change.ProposalIndex}] = true
entries = append(entries, CorrectionLedgerEntry{
StageIndex: module.StageIndex, StageID: module.StageID,
@@ -103,6 +129,9 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry
})
}
for _, change := range module.SkippedChanges {
if _, ok := repairs[proposalIdentity{module.ModuleInstance, change.ProposalIndex}]; ok {
continue
}
seen[proposalIdentity{module.ModuleInstance, change.ProposalIndex}] = true
entries = append(entries, CorrectionLedgerEntry{
StageIndex: module.StageIndex, StageID: module.StageID,
@@ -122,6 +151,12 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry
})
}
for _, rejection := range module.ValidatorRejected {
if seen[proposalIdentity{module.ModuleInstance, rejection.ProposalIndex}] {
continue
}
if _, ok := repairs[proposalIdentity{module.ModuleInstance, rejection.ProposalIndex}]; ok {
continue
}
disposition := correctionDispositionRejected
if rejection.Disposition == validators.DispositionUnavailable {
disposition = correctionDispositionUnavailable
@@ -156,6 +191,23 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry
})
}
}
for identity, repair := range repairs {
stage := stages[identity.instance]
original := repair.Original
entry := CorrectionLedgerEntry{StageIndex: stage.index, StageID: stage.id, RunID: runID, ModuleKey: original.ModuleKey, ModuleInstance: identity.instance, ProposalIndex: identity.index, SegmentID: original.TargetSegmentID, OriginalText: original.OriginalText, ProposedCorrectedText: &original.CorrectedText, ReplacementPolicy: stage.policy, Disposition: repair.Outcome, DeterministicValidatorResults: filterLedgerDecisions(stage.decisions[identity.index], false), LLMValidatorResults: filterLedgerDecisions(stage.decisions[identity.index], true)}
for i, attempt := range repair.Attempts {
item := LedgerRepairAttempt{Attempt: i + 1, Revision: attempt.Revision, FailureReason: string(attempt.FailureReason), RequestMetadataPath: attempt.Artifacts.RequestMetadataPath, RequestPayloadPath: attempt.Artifacts.RequestPayloadPath, ResponsePayloadPath: attempt.Artifacts.ResponsePayloadPath, ErrorPayloadPath: attempt.Artifacts.ErrorPayloadPath, DeterministicValidatorResults: filterLedgerDecisions(attempt.Decisions, false), LLMValidatorResults: filterLedgerDecisions(attempt.Decisions, true)}
entry.RepairAttempts = append(entry.RepairAttempts, item)
}
if repair.Outcome == "applied" {
segment, before, after := repair.Final.TargetSegmentID, repair.Final.OriginalText, repair.Final.CorrectedText
entry.AppliedSegmentID = &segment
entry.AppliedOriginalText = &before
entry.AppliedCorrectedText = &after
}
seen[identity] = true
entries = append(entries, entry)
}
for _, candidate := range runOutput.Candidates {
if seen[proposalIdentity{candidate.ModuleInstance, candidate.ProposalIndex}] {
continue
@@ -197,6 +249,7 @@ func filterLedgerDecisions(in []runner.ValidatorDecisionRecord, wantLLM bool) []
continue
}
out = append(out, LedgerValidatorDecisionRecord{
ArtifactPath: decision.DiagnosticArtifactPath,
ValidatorKey: decision.ValidatorName,
Approved: decision.Approved,
Disposition: decision.Disposition,

34
internal/framework/processreport/correction_ledger_test.go
View File

@@ -208,3 +208,37 @@ func TestCorrectionLedgerSerializesEmptyReplacementByDisposition(t *testing.T) {
}
}
}
func TestCorrectionLedgerCombinesRepairLineage(t *testing.T) {
original := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 3, ModuleKey: "grammar", ModuleInstance: "grammar_1"}}
revised := original
revised.TargetSegmentID = 2
revised.OriginalText = "uh "
revised.CorrectedText = ""
output := &runner.RunOutput{ModuleResults: []runner.ModuleResult{{StageIndex: 0, ModuleKey: "grammar", ModuleInstance: "grammar_1", Status: runner.ModuleStatusSuccess, ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, ValidatorDecisions: []runner.ValidatorDecisionRecord{{ProposalIndex: 3, ValidatorName: "confidence_threshold", ReasonCode: "low_confidence", Disposition: validators.DispositionRejected}}, ValidatorRejected: []runner.ValidatorRejectedChange{{ProposalIndex: 3, ReasonCode: "low_confidence"}}, AppliedChanges: []proposals.AppliedChange{{ProposalIndex: 3, TargetSegmentID: 2, OriginalText: "uh ", CorrectedText: ""}}}}, WorkUnits: []runner.WorkUnitOutcome{{ModuleInstance: "grammar_1", Repairs: []runner.RepairOutcome{{ProposalIndex: 3, Original: original, Final: revised, Outcome: "applied", Attempts: []runner.RepairAttempt{{Revision: &revised, Decisions: []runner.ValidatorDecisionRecord{{ProposalIndex: 3, ValidatorName: "editorial_review", Approved: true, Disposition: validators.DispositionApproved, DiagnosticArtifactPath: "validation.json"}}}}}}}}, Candidates: []proposals.EnrichedCorrectionProposal{original}}
ledger := BuildCorrectionLedger(CorrectionLedgerInput{RunOutput: output})
if len(ledger) != 1 {
t.Fatalf("ledger: %+v", ledger)
}
entry := ledger[0]
if entry.SegmentID != 1 || entry.OriginalText != "cat" || entry.ProposedCorrectedText == nil || *entry.ProposedCorrectedText != "dog" || entry.AppliedSegmentID == nil || *entry.AppliedSegmentID != 2 || entry.AppliedOriginalText == nil || *entry.AppliedOriginalText != "uh " || entry.AppliedCorrectedText == nil || *entry.AppliedCorrectedText != "" || len(entry.RepairAttempts) != 1 || len(entry.DeterministicValidatorResults) != 1 || len(entry.RepairAttempts[0].LLMValidatorResults) != 1 || entry.RepairAttempts[0].LLMValidatorResults[0].ArtifactPath != "validation.json" {
t.Fatalf("lineage: %+v", entry)
}
}
func TestInterruptedRepairLedgerKeepsEarlierDecisions(t *testing.T) {
original := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1, ModuleKey: "grammar", ModuleInstance: "grammar_1"}}
revision := original
revision.CorrectedText = "fox"
output := &runner.RunOutput{ModuleResults: []runner.ModuleResult{{ModuleKey: "grammar", ModuleInstance: "grammar_1", Status: runner.ModuleStatusFailed, ValidatorDecisions: []runner.ValidatorDecisionRecord{{ValidatorName: "confidence_threshold", ProposalIndex: 1, ReasonCode: "low_confidence", Disposition: validators.DispositionRejected}}}}, WorkUnits: []runner.WorkUnitOutcome{{ModuleInstance: "grammar_1", Repairs: []runner.RepairOutcome{{ProposalIndex: 1, Original: original, Final: revision, Outcome: "interrupted", Attempts: []runner.RepairAttempt{{Revision: &revision, Decisions: []runner.ValidatorDecisionRecord{{ProposalIndex: 1, ValidatorName: "editorial_review", Disposition: validators.DispositionUnavailable, ReasonCode: "provider_failed"}}}}}}}}, Candidates: []proposals.EnrichedCorrectionProposal{original}}
ledger := BuildCorrectionLedger(CorrectionLedgerInput{RunOutput: output})
var found *CorrectionLedgerEntry
for i := range ledger {
if ledger[i].Disposition == "interrupted" {
found = &ledger[i]
}
}
if found == nil || len(found.DeterministicValidatorResults) != 1 || len(found.RepairAttempts) != 1 || len(found.RepairAttempts[0].LLMValidatorResults) != 1 || found.AppliedCorrectedText != nil {
t.Fatalf("ledger: %+v", ledger)
}
}

19
internal/framework/processreport/llm.go
View File

@@ -20,14 +20,15 @@ func mapLLMSummary(source llm.ExecutionSummary) reporting.LLMSummary {
}
type executionCall struct {
CallID uint64 `json:"call_id"`
Context structuredoutput.DiagnosticContext `json:"context"`
Prompt structuredoutput.PromptMetadata `json:"prompt"`
Started time.Time `json:"started"`
Finished *time.Time `json:"finished,omitempty"`
ElapsedMS int64 `json:"elapsed_ms"`
Failure structuredoutput.FailureKind `json:"failure,omitempty"`
Summary reporting.LLMSummary `json:"summary"`
RequestedPromptID string `json:"requested_prompt_id,omitempty"`
CallID uint64 `json:"call_id"`
Context structuredoutput.DiagnosticContext `json:"context"`
Prompt structuredoutput.PromptMetadata `json:"prompt"`
Started time.Time `json:"started"`
Finished *time.Time `json:"finished,omitempty"`
ElapsedMS int64 `json:"elapsed_ms"`
Failure structuredoutput.FailureKind `json:"failure,omitempty"`
Summary reporting.LLMSummary `json:"summary"`
}
type executionEvent struct {
@@ -71,7 +72,7 @@ func BuildLLMExecution(snapshot llm.ObservationSnapshot) any {
value := call.Finished
finished = &value
}
calls = append(calls, executionCall{CallID: call.CallID, Context: call.Context, Prompt: call.Prompt, Started: call.Started, Finished: finished, ElapsedMS: call.WallDuration.Milliseconds(), Failure: call.Failure, Summary: mapLLMSummary(llm.Summarize(group))})
calls = append(calls, executionCall{RequestedPromptID: call.RequestedPromptID, CallID: call.CallID, Context: call.Context, Prompt: call.Prompt, Started: call.Started, Finished: finished, ElapsedMS: call.WallDuration.Milliseconds(), Failure: call.Failure, Summary: mapLLMSummary(llm.Summarize(group))})
}
for _, event := range snapshot.Events {
var generation, attempt *uint64