eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Retain repair diagnostics and correction lineage

Browse Source
This commit is contained in:
Eric Rakestraw
2026-11-11 04:37:52 +00:00
parent 735dfc1834
commit f4c638313e
15 changed files with 308 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
internal/framework/validators/deterministic.go
View File

@@ -3,6 +3,7 @@ package validators
import (
"context"
"fmt"
"math"
"strings"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
@@ -104,7 +105,7 @@ func (v NonEmptyResultingSegmentValidator) Validate(_ context.Context, req Reque
decisions = append(decisions, approval(c.ProposalIndex))
continue
}
preview := proposals.PreviewProposalForSegment(&segment, c.CorrectionProposal, req.ReplacementPolicy)
preview := previewForEvidence(req, &segment, c.CorrectionProposal)
if preview.SkipReason == proposals.SkipReasonEmptyResultingText {
decisions = append(decisions, rejection(c.ProposalIndex, ReasonEmptyResultingText, "proposal would leave the segment empty"))
continue
@@ -154,7 +155,7 @@ func ValidateProtectedTerms(req Request) (Result, error) {
if segment.ID != c.TargetSegmentID {
continue
}
preview := proposals.PreviewProposalForSegment(segment, c.CorrectionProposal, req.ReplacementPolicy)
preview := previewForEvidence(req, segment, c.CorrectionProposal)
if preview.Applicable {
if glossaryStage {
// A correction span may sit inside a longer exempt name.
@@ -177,3 +178,10 @@ func ValidateProtectedTerms(req Request) (Result, error) {
}
return Result{ValidatorName: "protected_terms", Decisions: decisions}, nil
}
func previewForEvidence(req Request, segment *schema.Segment, correction proposals.CorrectionProposal) proposals.SegmentPreviewResult {
if req.IndependentEvidence && (math.IsNaN(correction.Confidence) || correction.Confidence < 0 || correction.Confidence > 1) {
correction.Confidence = 0.5
}
return proposals.PreviewProposalForSegment(segment, correction, req.ReplacementPolicy)
}

21
internal/framework/validators/llm_validators.go
View File

@@ -11,7 +11,6 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/stagename"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
@@ -128,7 +127,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
completion, err = req.LLMClient.CompleteStructured(ctx, StructuredCompletionRequest{
PromptID: promptMetadata.PromptID,
Variables: variables, Schema: responseschema.ValidatorDecisionSetKey,
Context: structuredoutput.DiagnosticContext{StageIndex: req.StageIndex, StageID: req.StageID, Role: "validation", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: req.SectionIndex, BatchIndex: &batch.BatchIndex, ValidatorName: v.name},
Context: structuredoutput.DiagnosticContext{StageIndex: req.StageIndex, StageID: req.StageID, Role: "validation", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: req.SectionIndex, BatchIndex: &batch.BatchIndex, ValidatorName: v.name, ProposalIndex: req.RepairProposalIndex, RepairAttempt: req.RepairAttempt},
}, &response)
batchIndex := batch.BatchIndex
operation := outcomes.Operation{
@@ -144,6 +143,9 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
if req.SectionIndex != nil {
stageInstance = fmt.Sprintf("%s:section-%04d", stageInstance, *req.SectionIndex)
}
if req.RepairProposalIndex != nil && req.RepairAttempt != nil {
stageInstance = fmt.Sprintf("%s:proposal-%04d:attempt-%04d", stageInstance, *req.RepairProposalIndex, *req.RepairAttempt)
}
stage := stagename.ValidatorBatch(stageInstance, v.name, batch.BatchIndex)
responseSchema := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey)
responsePayload := map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model}
@@ -155,12 +157,13 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
stage,
map[string]any{
"stage_index": req.StageIndex, "stage_id": req.StageID, "module_key": req.ModuleKey,
"validator_name": v.name,
"validator_type": v.validatorType,
"batch_index": batch.BatchIndex,
"proposal_indices": batchProposalIndices(batch.Items),
"prompt": completion.Prompt,
"response_schema": responseSchema.DiagnosticsMap(),
"validator_name": v.name,
"validator_type": v.validatorType,
"batch_index": batch.BatchIndex,
"proposal_indices": batchProposalIndices(batch.Items),
"repair_proposal_index": req.RepairProposalIndex, "repair_attempt": req.RepairAttempt,
"prompt": completion.Prompt,
"response_schema": responseSchema.DiagnosticsMap(),
},
completion.Prepared,
responsePayload,
@@ -263,7 +266,7 @@ func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorT
}
seg := segments[position]
preview := proposals.PreviewProposalForSegment(&seg, p.CorrectionProposal, req.ReplacementPolicy)
preview := previewForEvidence(req, &seg, p.CorrectionProposal)
if !preview.Applicable {
immediate = append(immediate, rejection(p.ProposalIndex, string(preview.SkipReason), "proposal is not previewable for LLM validation"))
continue

3
internal/framework/validators/models.go
View File

@@ -36,6 +36,9 @@ type Request struct {
StageID *string `json:"stage_id,omitempty"`
// ReportDiagnosticError may be called concurrently; it must not affect processing.
ReportDiagnosticError func(error) `json:"-"`
IndependentEvidence bool `json:"-"`
RepairProposalIndex *int `json:"repair_proposal_index,omitempty"`
RepairAttempt *int `json:"repair_attempt,omitempty"`
SectionIndex *int `json:"section_index,omitempty"`
WorkingTranscript *schema.Transcript `json:"-"`
CandidateProposal []proposals.EnrichedCorrectionProposal `json:"candidate_proposals"`