140 lines
9.0 KiB
Go
140 lines
9.0 KiB
Go
package runner
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
|
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
|
|
)
|
|
|
|
func TestFeedbackCollectsIndependentFindingsAndUsableSemanticInput(t *testing.T) {
|
|
candidate := func(index, target int, original, replacement string, confidence float64) proposals.EnrichedCorrectionProposal {
|
|
return proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: target, OriginalText: original, CorrectedText: replacement, Confidence: confidence}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index}}
|
|
}
|
|
var seen []int
|
|
semantic := validatormetadata.Wrap(fakeValidator{name: "semantic", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
|
var decisions []validators.Decision
|
|
for _, p := range req.CandidateProposal {
|
|
seen = append(seen, p.ProposalIndex)
|
|
decisions = append(decisions, validators.Decision{ProposalIndex: p.ProposalIndex, Disposition: validators.DispositionRejected, ReasonCode: "semantic_rejection", Message: "meaning changed"})
|
|
}
|
|
return validators.Result{Decisions: decisions}, nil
|
|
}}, validatormetadata.ExecutionClassLLMBacked)
|
|
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", Settings: config.Stage{ConfidenceThreshold: .8, LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh teh cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.ConfidenceThresholdValidator{}, semantic}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate(0, 1, "cat", "dog", .2), candidate(1, 0, "teh", "the", .2), candidate(2, 1, "teh", "the", .9), candidate(3, 1, "cat", "cat", .9)}}
|
|
result, err := validateSectionCandidates(context.Background(), input)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
reasons := map[int]map[string]bool{}
|
|
for _, d := range result.rejected {
|
|
if reasons[d.ProposalIndex] == nil {
|
|
reasons[d.ProposalIndex] = map[string]bool{}
|
|
}
|
|
reasons[d.ProposalIndex][d.ReasonCode] = true
|
|
}
|
|
if !reasons[0][validators.ReasonLowConfidence] || !reasons[0]["semantic_rejection"] || !reasons[1][validators.ReasonInvalidTargetSegment] || !reasons[2][string(proposals.SkipReasonAmbiguousOriginal)] || !reasons[3][validators.ReasonNoEffect] {
|
|
t.Fatalf("findings = %+v", reasons)
|
|
}
|
|
if len(result.approved) != 0 || len(seen) != 1 || seen[0] != 0 {
|
|
t.Fatalf("approved=%v semantic input=%v", result.approved, seen)
|
|
}
|
|
}
|
|
|
|
func TestFeedbackKeepsUnavailableAndZeroBudgetShortlists(t *testing.T) {
|
|
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .9}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
|
|
for _, budget := range []int{0, 1} {
|
|
t.Run(fmt.Sprint(budget), func(t *testing.T) {
|
|
var later int
|
|
first := fakeValidator{name: "first", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
|
var decisions []validators.Decision
|
|
for _, p := range req.CandidateProposal {
|
|
decisions = append(decisions, validators.Decision{ProposalIndex: p.ProposalIndex, Disposition: validators.DispositionUnavailable, ReasonCode: "unavailable"})
|
|
}
|
|
return validators.Result{Decisions: decisions}, nil
|
|
}}
|
|
second := fakeValidator{name: "second", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
|
later = len(req.CandidateProposal)
|
|
var decisions []validators.Decision
|
|
for _, p := range req.CandidateProposal {
|
|
decisions = append(decisions, validators.Decision{ProposalIndex: p.ProposalIndex, Approved: true, Disposition: validators.DispositionApproved})
|
|
}
|
|
return validators.Result{Decisions: decisions}, nil
|
|
}}
|
|
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{first, second}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}}
|
|
result, err := validateSectionCandidates(context.Background(), input)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(result.approved) != 0 || len(result.rejected) != 1 || result.rejected[0].Disposition != validators.DispositionUnavailable || later != budget {
|
|
t.Fatalf("result=%+v later=%d", result, later)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRepairFindingsUseSafeUnavailableEvidence(t *testing.T) {
|
|
rejected := []ValidatorRejectedChange{{ValidatorName: "confidence_threshold", ProposalIndex: 2, ReasonCode: "low_confidence", Message: "confidence 0.2 below threshold 0.8"}, {ValidatorName: "editorial_review", ProposalIndex: 2, Disposition: validators.DispositionUnavailable, ReasonCode: "provider_failed", Message: "provider diagnostic secret"}, {ValidatorName: "other", ProposalIndex: 3, ReasonCode: "other"}}
|
|
findings := repairFindings(2, rejected)
|
|
if len(findings) != 2 || findings[0].Explanation != rejected[0].Message || findings[1].Status != "unavailable" || findings[1].Explanation != "assessment was unavailable" {
|
|
t.Fatalf("findings: %+v", findings)
|
|
}
|
|
}
|
|
|
|
func TestRevalidationArtifactsUseProposalAndAttemptCoordinates(t *testing.T) {
|
|
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{
|
|
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}},
|
|
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}},
|
|
}}
|
|
validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
dir := t.TempDir()
|
|
attempt := 1
|
|
var paths []string
|
|
for _, index := range []int{2, 3} {
|
|
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .9}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index}}
|
|
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar_1", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validator}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, ValidationLLMClient: client, DiagnosticsDir: dir, RepairProposalIndex: &index, RepairAttempt: &attempt}
|
|
result, err := validateSectionCandidates(context.Background(), input)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(result.decisions) != 1 || result.decisions[0].DiagnosticArtifactPath == "" {
|
|
t.Fatalf("decisions: %+v", result.decisions)
|
|
}
|
|
paths = append(paths, result.decisions[0].DiagnosticArtifactPath)
|
|
}
|
|
if paths[0] == paths[1] {
|
|
t.Fatalf("colliding paths: %v", paths)
|
|
}
|
|
for _, path := range paths {
|
|
if _, err := os.Stat(path); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestFeedbackUsesIndependentPreviewWhenConfidenceInvalid(t *testing.T) {
|
|
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}}}
|
|
validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: 2}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
|
|
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validator}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, ValidationLLMClient: client}
|
|
result, err := validateSectionCandidates(context.Background(), input)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(result.approved) != 0 || len(result.decisions) != 2 || result.decisions[1].ValidatorName != "editorial_review" || !result.decisions[1].Approved {
|
|
t.Fatalf("result: %+v", result)
|
|
}
|
|
}
|