From aa8073cab1cade3c8b34f4a9417b96c3676c9e56 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Wed, 11 Nov 2026 04:47:43 +0000 Subject: [PATCH] Validate correction repair through the processing command --- .../correction_repair_integration_test.go | 334 ++++++++++++++++++ docs/config.md | 20 +- docs/internal/llm-runtime.md | 8 +- docs/internal/modules.md | 7 + docs/internal/prompts.md | 12 +- docs/internal/validators.md | 5 +- docs/operations.md | 8 +- docs/roadmap/correction-repair.md | 4 +- docs/roadmap/future.md | 8 +- docs/roadmap/implementation.md | 2 + examples/README.md | 10 +- internal/framework/llm/native_adapter_test.go | 7 +- internal/framework/proposals/preview.go | 4 +- internal/framework/runner/feedback.go | 2 +- internal/framework/runner/feedback_test.go | 18 + internal/framework/runner/repair.go | 33 +- internal/framework/runner/repair_test.go | 38 ++ .../framework/validators/deterministic.go | 2 +- .../framework/validators/protected_terms.go | 30 +- 19 files changed, 511 insertions(+), 41 deletions(-) create mode 100644 cmd/audita/correction_repair_integration_test.go diff --git a/cmd/audita/correction_repair_integration_test.go b/cmd/audita/correction_repair_integration_test.go new file mode 100644 index 0000000..b2a6a0b --- /dev/null +++ b/cmd/audita/correction_repair_integration_test.go @@ -0,0 +1,334 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "reflect" + "strings" + "sync" + "testing" + "time" + + "gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting" +) + +type repairWireMessage struct { + Role string `json:"role"` + Content string `json:"content"` +} +type repairWireRequest struct { + Messages []repairWireMessage `json:"messages"` + ResponseFormat struct { + JSONSchema struct { + Schema struct { + Properties map[string]any `json:"properties"` + } `json:"schema"` + } `json:"json_schema"` + } `json:"response_format"` +} +type correctionProvider struct { + mu sync.Mutex + requests []repairWireRequest + attempts map[int]int + entered chan struct{} + wait bool +} + +func (p *correctionProvider) ServeHTTP(w http.ResponseWriter, r *http.Request) { + var req repairWireRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, "bad request", 400) + return + } + _, repair := req.ResponseFormat.JSONSchema.Schema.Properties["proposal_ref"] + _, validation := req.ResponseFormat.JSONSchema.Schema.Properties["validations"] + p.mu.Lock() + p.requests = append(p.requests, req) + p.mu.Unlock() + content := `{"corrections":[{"id":1,"original_text":"hello","corrected_text":"Hello","confidence":0.99},{"id":2,"original_text":"second","corrected_text":"Second","confidence":0.1},{"id":3,"original_text":"third","corrected_text":"Third","confidence":0.1}]}` + if validation { + var indices []int + for _, message := range req.Messages { + if pos := strings.LastIndex(message.Content, "Corrections to validate:"); pos >= 0 { + var items []struct { + CorrectionIndex int `json:"correction_index"` + } + if err := json.Unmarshal([]byte(strings.TrimSpace(message.Content[pos+len("Corrections to validate:"):])), &items); err != nil { + http.Error(w, "bad validation", 400) + return + } + for _, item := range items { + indices = append(indices, item.CorrectionIndex) + } + } + } + var decisions []map[string]any + for _, index := range indices { + decisions = append(decisions, map[string]any{"correction_index": index, "approved": true, "confidence": 0.99, "reason": "controlled"}) + } + raw, _ := json.Marshal(map[string]any{"validations": decisions}) + content = string(raw) + } + if repair { + if p.wait { + select { + case p.entered <- struct{}{}: + default: + } + <-r.Context().Done() + return + } + ref := 2 + if strings.Contains(req.Messages[len(req.Messages)-1].Content, `"proposal_ref":3`) { + ref = 3 + } + p.mu.Lock() + p.attempts[ref]++ + attempt := p.attempts[ref] + p.mu.Unlock() + if ref == 3 { + http.Error(w, `{"error":{"message":"context too long"}}`, http.StatusBadRequest) + return + } + confidence := 0.1 + if attempt > 1 { + confidence = 0.99 + } + content = fmt.Sprintf(`{"proposal_ref":2,"action":"revise","correction":{"id":2,"original_text":"second","corrected_text":"Second","confidence":%v}}`, confidence) + } + raw, _ := json.Marshal(content) + w.Header().Set("Content-Type", "application/json") + fmt.Fprintf(w, `{"model":"fixture","choices":[{"message":{"content":%s}}]}`, raw) +} +func setupCorrectionFixture(t *testing.T, attempts int, replacement, wait bool) (processFixture, *correctionProvider) { + t.Helper() + f := newProcessFixture(t, "normal", "[grammar]") + f.setSegments(t, "hello world", "second text", "third text") + provider := &correctionProvider{attempts: map[int]int{}, entered: make(chan struct{}, 1), wait: wait} + server := httptest.NewServer(provider) + t.Cleanup(server.Close) + resources := fmt.Sprintf("profiles:\n - id: proposal\n endpoint: %s/v1\n model: fixture\n - id: validation\n endpoint: %s/v1\n model: fixture\n", server.URL, server.URL) + if replacement { + resources += "collections:\n audita:\n prompts:\n - id: modules.grammar.repair\n messages:\n - role: user\n content_file: repair.txt\n output:\n format: json\n validation_mode: json_schema\n schema_path: correction_repair.v1.json\n" + if err := os.WriteFile(filepath.Join(f.dir, "repair.txt"), []byte("OVERRIDE {{ .RepairRequestJSON }}"), 0600); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(filepath.Join(f.dir, "prompts.yml"), []byte(resources), 0600); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(f.config) + if err != nil { + t.Fatal(err) + } + text := strings.Replace(string(data), "correction_repair_attempts: 0", fmt.Sprintf("correction_repair_attempts: %d", attempts), 1) + text += "chunking:\n target_sections: 1\n min_section_tokens: 1\nnormalization:\n max_segment_gap: 0\n" + if err := os.WriteFile(f.config, []byte(text), 0600); err != nil { + t.Fatal(err) + } + return f, provider +} + +func TestSubprocessCorrectionRepairConversation(t *testing.T) { + for _, tc := range []struct { + name string + debug, replacement bool + }{{"default prompt without debug", false, false}, {"replacement with debug", true, true}} { + t.Run(tc.name, func(t *testing.T) { + f, provider := setupCorrectionFixture(t, 2, tc.replacement, false) + args := []string{"--verbose"} + if tc.debug { + args = append(args, "--debug") + } + code, stdout, stderr := f.run(t, true, args...) + if code != 0 || stdout != "" { + t.Fatalf("code=%d stdout=%s stderr=%s", code, stdout, stderr) + } + output := f.readTranscript(t, f.output) + if len(output.Segments) != 3 || output.Segments[0].Text != "Hello world" || output.Segments[1].Text != "Second text" || output.Segments[2].Text != "third text" { + t.Fatalf("transcript: %+v", output) + } + report, err := os.ReadFile(filepath.Join(f.dir, "report.md")) + if err != nil { + t.Fatal(err) + } + for _, fragment := range []string{"## Correction repair", "| 1 | 1 | 1 | 2 | applied |", "| 1 | 1 | 2 | 1 | unavailable |"} { + if !strings.Contains(string(report), fragment) { + t.Errorf("missing %q in %s", fragment, report) + } + } + doc := f.errors(t) + recoveries := f.recoveries(t) + if doc.Summary.LLMOperations == nil || *doc.Summary.LLMOperations != 10 || *doc.Summary.CorrectionsApplied != 2 || *doc.Summary.Recoveries != 1 || !reflect.DeepEqual(doc.Summary, recoveries.Summary) || !strings.Contains(stderr, "2 corrections applied; 1 work units with errors; 1 recoveries") { + t.Fatalf("counts: errors=%+v recoveries=%+v stderr=%s", doc.Summary, recoveries.Summary, stderr) + } + if len(doc.Errors) != 1 || len(doc.Errors[0].Failures) != 1 || doc.Errors[0].Failures[0].Operation != "correction_repair" || doc.Errors[0].Failures[0].Reason != "provider_failed" { + t.Fatalf("errors: %+v", doc) + } + provider.mu.Lock() + requests := append([]repairWireRequest(nil), provider.requests...) + provider.mu.Unlock() + var base []repairWireMessage + var branches [][]repairWireMessage + for _, req := range requests { + if _, ok := req.ResponseFormat.JSONSchema.Schema.Properties["corrections"]; ok { + base = req.Messages + } + if _, ok := req.ResponseFormat.JSONSchema.Schema.Properties["proposal_ref"]; ok { + branches = append(branches, req.Messages) + } + } + if len(base) != 2 || len(branches) != 3 || len(branches[0]) != 4 || len(branches[1]) != 6 || len(branches[2]) != 4 { + t.Fatalf("request lengths base=%d branches=%v", len(base), branches) + } + for _, messages := range branches { + for i := range base { + if messages[i] != base[i] { + t.Fatalf("changed prefix: %+v", messages) + } + } + if messages[2].Role != "assistant" || !strings.Contains(messages[2].Content, `"corrections"`) { + t.Fatalf("missing initial response: %+v", messages) + } + } + if branches[1][3] != branches[0][3] || branches[1][4].Role != "assistant" || strings.Contains(branches[2][3].Content, `"proposal_ref":2`) { + t.Fatalf("branch history: %+v", branches) + } + if tc.replacement && !strings.Contains(branches[0][3].Content, "OVERRIDE") { + t.Fatal("replacement not used") + } + if tc.debug { + _, ledger := f.debugArtifacts(t) + if len(ledger) != 3 { + t.Fatalf("ledger: %+v", ledger) + } + } + }) + } +} + +func TestSubprocessRepairDisabledAndOutputSelections(t *testing.T) { + for _, tc := range []struct { + name string + reporting, debug bool + }{{"neither", false, false}, {"debug only", false, true}, {"report only", true, false}} { + t.Run(tc.name, func(t *testing.T) { + f, provider := setupCorrectionFixture(t, 0, false, false) + if !tc.reporting { + file, err := os.OpenFile(f.config, os.O_APPEND|os.O_WRONLY, 0600) + if err != nil { + t.Fatal(err) + } + fmt.Fprintln(file, "reporting:\n enabled: false") + file.Close() + } + var args []string + if tc.debug { + args = append(args, "--debug") + } + code, _, stderr := f.run(t, true, args...) + if code != 0 { + t.Fatalf("code=%d stderr=%s", code, stderr) + } + provider.mu.Lock() + requests := append([]repairWireRequest(nil), provider.requests...) + provider.mu.Unlock() + for _, req := range requests { + if _, ok := req.ResponseFormat.JSONSchema.Schema.Properties["proposal_ref"]; ok { + t.Fatal("repair request with zero budget") + } + } + _, err := os.Stat(filepath.Join(f.dir, "report.md")) + if tc.reporting != (err == nil) { + t.Fatalf("report selection: %v", err) + } + if tc.debug { + _, ledger := f.debugArtifacts(t) + if len(ledger) == 0 { + t.Fatal("debug ledger missing") + } + } else if _, err := os.Stat(filepath.Join(f.dir, "runs")); !os.IsNotExist(err) { + t.Fatalf("unexpected debug dir: %v", err) + } + }) + } +} + +func TestSubprocessCancellationDuringCorrectionRepair(t *testing.T) { + f, provider := setupCorrectionFixture(t, 2, false, true) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + args := []string{"-test.run=^TestProcessSubprocessHelper$", "--", "process", f.transcript, "--config", f.config, "--output", f.output, "--debug"} + command := exec.CommandContext(ctx, os.Args[0], args...) + command.Env = append(os.Environ(), "AUDITA_HELPER_PROCESS=1") + var stderr bytes.Buffer + command.Stderr = &stderr + if err := command.Start(); err != nil { + t.Fatal(err) + } + select { + case <-provider.entered: + case <-ctx.Done(): + t.Fatalf("repair not reached: %v", ctx.Err()) + } + if err := command.Process.Signal(os.Interrupt); err != nil { + t.Fatal(err) + } + err := command.Wait() + if ctx.Err() != nil || err == nil { + t.Fatalf("cancellation: %v ctx=%v stderr=%s", err, ctx.Err(), stderr.String()) + } + if _, err := os.Stat(f.output); !os.IsNotExist(err) { + t.Fatalf("transcript published: %v", err) + } + doc := f.errors(t) + if doc.Status != reporting.StatusFailed || len(doc.Errors) == 0 { + t.Fatalf("failed report: %+v", doc) + } + data, err := os.ReadFile(filepath.Join(f.dir, "report.md")) + if err != nil || !strings.Contains(string(data), "| 1 | 1 | 1 | 1 | interrupted |") { + t.Fatalf("partial report %s %v", data, err) + } + _, ledger := f.debugArtifacts(t) + if len(ledger) == 0 { + t.Fatal("partial ledger missing") + } +} + +func TestSubprocessRepairWithoutOrdinaryReporting(t *testing.T) { + for _, debug := range []bool{false, true} { + t.Run(fmt.Sprintf("debug=%t", debug), func(t *testing.T) { + f, _ := setupCorrectionFixture(t, 2, false, false) + file, err := os.OpenFile(f.config, os.O_APPEND|os.O_WRONLY, 0600) + if err != nil { + t.Fatal(err) + } + fmt.Fprintln(file, "reporting:\n enabled: false") + file.Close() + args := []string{"--verbose"} + if debug { + args = append(args, "--debug") + } + code, _, stderr := f.run(t, true, args...) + if code != 0 || !strings.Contains(stderr, "2 corrections applied; 1 work units with errors; 1 recoveries") { + t.Fatalf("code=%d stderr=%s", code, stderr) + } + if _, err := os.Stat(filepath.Join(f.dir, "report.md")); !os.IsNotExist(err) { + t.Fatalf("unexpected report: %v", err) + } + if debug { + _, ledger := f.debugArtifacts(t) + if len(ledger) != 3 { + t.Fatalf("ledger: %+v", ledger) + } + } + }) + } +} diff --git a/docs/config.md b/docs/config.md index d0a9d18..779513c 100644 --- a/docs/config.md +++ b/docs/config.md @@ -77,11 +77,29 @@ configuration. | `reporting.enabled`, `reporting.stdout_dir` | `true`, `.` | Ordinary report capture and default directory for stdout output. | | `diagnostics.work_dir` | `/tmp/audita` | Parent directory for optional debug capture. | +For example, an entry can disable correction repair or select another profile: + +```yaml +llm: + proposal_profile: local + correction_repair_attempts: 2 +pipeline: + stages: + - module: grammar + llm: + correction_repair_profile: review + - module: spoken_word + llm: + correction_repair_attempts: 0 +``` + Processing profiles are selected explicitly; no profile or glossary is selected automatically. The validation selection falls back after root and per-entry settings are combined. An explicit root correction repair profile is inherited by each entry unless overridden or cleared with null; otherwise the entry uses its effective proposal profile. The repair profile is checked only when its -attempt budget is positive. Null does not disable validation. Numeric zero and false +attempt budget is positive. Each correction repair attempt is a logical call +that can also use the configured transient retries and structural repairs. Null +does not disable validation. Numeric zero and false retain their field meaning, subject to validation. Inspect recorded origins and default dependencies using the [native commands](cli.md). diff --git a/docs/internal/llm-runtime.md b/docs/internal/llm-runtime.md index bec8409..7fe4030 100644 --- a/docs/internal/llm-runtime.md +++ b/docs/internal/llm-runtime.md @@ -16,7 +16,13 @@ credential source names for artifact redaction without inspecting unrelated environment values. See the [integration contract](../integrations/openai-compatible-llm.md) for failure behavior and the [configuration reference](../config.md) for sources. -Proposal calls can retain the original prompt, effective variables, prepared messages, and final response for section-local repair replay. The adapter renders the repair definition with the selected profile, appends branch history to the original prompt, and checks the prepared prefix before execution. The requested repair definition is retained in call attribution alongside the native base-prompt metadata. +When correction repair is enabled, the proposal call retains its prompt, +effective variables, prepared messages, and full final response for section-local +replay. The adapter renders the repair definition with the selected profile, +appends only that branch’s history, and checks the original prepared prefix +before execution. The complete conversation is sent to the selected provider; +provider rejection ends that branch as unavailable. The requested repair +definition is retained in call attribution alongside native base-prompt metadata. The observation collector tracks logical calls and execution events. Ordinary reports use compact safe outcomes; request and response evidence is optional diff --git a/docs/internal/modules.md b/docs/internal/modules.md index 00b0a7a..fbc61c7 100644 --- a/docs/internal/modules.md +++ b/docs/internal/modules.md @@ -13,6 +13,9 @@ Modules implement `contracts.TranscriptModule`: - `Validators()` - `Propose(ctx, req)` +Built-in modules also expose `RepairPromptID()` for the runner to select their +repair definitions; proposal-only callers retain the existing interface. + Runner resolves configured module specs to module instances through `internal/framework/modules`. ## Built-In Modules @@ -61,6 +64,10 @@ Module packages call the built-in chain resolver at construction. ## Failure and Outcome Behavior +Each actionable rejection can be revised or withdrawn within its original +section. A revision retains the original proposal identity and must pass the +full validator chain before application. + Module construction failures are fatal. Recoverable proposal model failures record an unsuccessful operation and yield no candidates for that section; other sections and module instances continue. Fatal adapter or application diff --git a/docs/internal/prompts.md b/docs/internal/prompts.md index 7a94c5e..4d3500b 100644 --- a/docs/internal/prompts.md +++ b/docs/internal/prompts.md @@ -11,9 +11,10 @@ and `audita/modules.grammar.proposal`; the corresponding repair IDs replace PromptKit composes embedded defaults with operator prompt sources. Shared instructions come from the embedded fragments in `assets/shared`; the request -adapter fills absent `HardeningPolicy`, `ProposalContract`, `RepairContract`, and -`ValidationContract` variables without replacing values explicitly supplied by -the caller. Processing supplies Audita's response-schema contract separately. +shared-variable helper fills absent `HardeningPolicy`, `ProposalContract`, +`RepairContract`, and `ValidationContract` variables. The processing adapter +supplies the embedded instruction fragments for those names. Processing supplies +Audita's response-schema contract separately. The operator's prompt definition controls which variables its template uses. For a standard request preview or validation, select a native PromptKit profile @@ -23,7 +24,10 @@ proposal definitions use `TranscriptDescriptionBlock`, `GlossaryJSON`, and the separate spelling list defines protection. Preview defaults the spelling list to `[]` when omitted. The validator definitions use `TranscriptDescriptionBlock` and `PayloadJSON`. The latter is supplied candidate JSON and does not require a -proposal inference call. An operator replacement may use different variables. +proposal inference call. Repair definitions use `RepairContract` and +`RepairRequestJSON`, containing the original proposal reference, current +correction, and ordered findings. An operator replacement may use different +variables. See the [upstream CLI reference](https://forge.maximumdirect.net/base-layer/commandkit/src/commit/f369d6a78253b6e6fcb1e023fb2e9c3af1d20ff0/docs/cli.md) for the standard preview, validation, and smoke commands. diff --git a/docs/internal/validators.md b/docs/internal/validators.md index fa03188..01cc20f 100644 --- a/docs/internal/validators.md +++ b/docs/internal/validators.md @@ -49,7 +49,10 @@ For each module section: When correction repair is enabled, the runner also gathers independent findings for rejected candidates when a validator has usable evidence. Unavailable assessments remain unavailable, and only candidates that pass every applicable -check can advance. A zero repair budget retains the original shortlist flow. +check can advance. A known rejection can enter repair even when another +assessment is unavailable; an unavailable assessment alone does not. Revised +candidates run the applicable checks again. A zero repair budget retains the +original shortlist flow. Decision cardinality is enforced inside Audita: each candidate proposal must receive exactly one outcome per validator it reaches. The runner still rejects diff --git a/docs/operations.md b/docs/operations.md index 522b6a1..ed4dd25 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -12,6 +12,10 @@ show complete projects. Processing preflight checks every configured glossary correction and selected profile before output preparation and model work. Empty protection and an empty pipeline are valid. A completed run can contain recoverable model errors while still applying independently validated edits. +Correction repair sends the complete original proposal exchange and branch +history to the selected repair provider. A provider rejection ends that branch +as unavailable; other approved edits can still be applied. See +[configuration](config.md#settings) for the attempt budget and profile selection. ## Outputs and lifecycle @@ -48,7 +52,9 @@ Optional debug preparation or writes can produce redacted warnings without changing correction decisions or the required output result. Debug material can include transcript content, prompts, responses, and correction decisions. It is not anonymized; protect it according to the input's sensitivity. Configured -credential values are redacted at the diagnostic boundary. +credential values are redacted at the diagnostic boundary. Repair interactions +and revalidation artifacts include proposal and attempt coordinates; the +correction ledger connects them to the original edit. ## Recovery diff --git a/docs/roadmap/correction-repair.md b/docs/roadmap/correction-repair.md index 654f5c6..7ab4b1c 100644 --- a/docs/roadmap/correction-repair.md +++ b/docs/roadmap/correction-repair.md @@ -2,8 +2,8 @@ **Updated:** 2026-11-11 -**Status:** Feature direction and all six decisions accepted. Implementation -is pending. See the [implementation plan](implementation.md). +**Status:** Implemented. See the [implementation plan](implementation.md) +and the [pipeline guide](../internal/pipeline.md). ## Objective diff --git a/docs/roadmap/future.md b/docs/roadmap/future.md index 63278a4..ce5a5cd 100644 --- a/docs/roadmap/future.md +++ b/docs/roadmap/future.md @@ -11,9 +11,9 @@ implementing every capability. The implemented [partial-success decision reconciliation](../internal/validators.md#llm-validator-batching) preserves usable decisions without another model call. Revision of rejected -correction proposals is now an agreed feature direction in the -[correction-repair roadmap](correction-repair.md), with its feature decisions accepted. -That work includes substantive validator feedback and explicit proposal withdrawal. +correction proposals is implemented with substantive +validator feedback and explicit proposal withdrawal; see the +[pipeline guide](../internal/pipeline.md). It is distinct from recovering an unavailable validator decision. Consider bounded feedback and regeneration for model responses that satisfy the @@ -32,7 +32,7 @@ shared library capabilities where appropriate; do not introduce an uncoordinated outer retry loop that multiplies existing retry/repair budgets. Editorial rejection remains a valid decision about the evaluated proposal. The -correction-repair roadmap owns any subsequent revision of that proposal and full +implemented correction repair loop handles subsequent revision and full revalidation; this deferred entry concerns unavailable decisions. Decision recovery must preserve required validation, candidate identity, partial-success safety, and truthful error/recovery reporting. Validate with controlled responses that diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 2e9cab5..baa205a 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -352,6 +352,8 @@ validator responses remains [deferred](future.md#domain-aware-response-recovery) ## Stage 9 — Validate the assembled command and update canonical documentation +**Completed.** + 1. Exercise the normal processing command with controlled provider responses, real configuration and prompt loading, and the output session. Cover a mixed proposal set with two independent repair branches, a second turn diff --git a/examples/README.md b/examples/README.md index 0c5cf1f..1af6a42 100644 --- a/examples/README.md +++ b/examples/README.md @@ -25,7 +25,15 @@ audita promptkit show prepared audita/validators.meaning_reversal_review \ --var 'TranscriptDescriptionBlock=' ``` -This renders the supplied candidate without inference. The native command help +To preview the repair instruction and its output schema without inference: + +```sh +audita promptkit show prepared audita/modules.grammar.repair \ + --config config.yml --profile local \ + --var 'RepairRequestJSON={"proposal_ref":1,"correction":{"id":1,"original_text":" ","corrected_text":"","confidence":0.9},"findings":[]}' +``` + +These previews render supplied data without inference. The native command help and [CLI reference](../docs/cli.md) describe additional inspection and validation. ## Service and overlay diff --git a/internal/framework/llm/native_adapter_test.go b/internal/framework/llm/native_adapter_test.go index cf09afc..506b834 100644 --- a/internal/framework/llm/native_adapter_test.go +++ b/internal/framework/llm/native_adapter_test.go @@ -112,7 +112,12 @@ func TestRoleClientExecutionBudgetsAndFailureClasses(t *testing.T) { t.Fatal(err) } var result map[string]any - response, err := client.CompleteStructured(context.Background(), request, &result) + capturing := request + capturing.CaptureExchange = true + response, err := client.CompleteStructured(context.Background(), capturing, &result) + if tc.success && (response.Exchange == nil || response.Exchange.FinalContent != tc.outputs[len(tc.outputs)-1]) { + t.Fatalf("captured final response: %+v", response.Exchange) + } if (err == nil) != tc.success || calls != tc.wantCalls { t.Fatalf("error=%v calls=%d", err, calls) } diff --git a/internal/framework/proposals/preview.go b/internal/framework/proposals/preview.go index 6f2d8c6..e207b78 100644 --- a/internal/framework/proposals/preview.go +++ b/internal/framework/proposals/preview.go @@ -65,7 +65,7 @@ func PreviewProposalForSegment(segment *schema.Segment, proposal CorrectionPropo corrected := strings.Replace(segment.Text, proposal.OriginalText, proposal.CorrectedText, 1) if strings.TrimSpace(corrected) == "" { - return SegmentPreviewResult{SkipReason: SkipReasonEmptyResultingText} + return SegmentPreviewResult{SkipReason: SkipReasonEmptyResultingText, CorrectedSegmentText: corrected} } return SegmentPreviewResult{ Applicable: true, @@ -76,7 +76,7 @@ func PreviewProposalForSegment(segment *schema.Segment, proposal CorrectionPropo case ReplacementPolicyReplaceAll: corrected := strings.ReplaceAll(segment.Text, proposal.OriginalText, proposal.CorrectedText) if strings.TrimSpace(corrected) == "" { - return SegmentPreviewResult{SkipReason: SkipReasonEmptyResultingText} + return SegmentPreviewResult{SkipReason: SkipReasonEmptyResultingText, CorrectedSegmentText: corrected} } return SegmentPreviewResult{ Applicable: true, diff --git a/internal/framework/runner/feedback.go b/internal/framework/runner/feedback.go index 6e4a50d..3cc7895 100644 --- a/internal/framework/runner/feedback.go +++ b/internal/framework/runner/feedback.go @@ -51,7 +51,7 @@ func feedbackCandidates(input validateSectionCandidatesInput, validator contract case "protected_terms": applicable = usableSpan if input.Spec.ModuleKey == modulecatalog.KeyGlossary || input.Spec.ModuleKey == modulecatalog.KeySpokenWord { - applicable = preview.Applicable + applicable = preview.Applicable || preview.SkipReason == proposals.SkipReasonEmptyResultingText } case "non_empty_resulting_segment": applicable = preview.Applicable || preview.SkipReason == proposals.SkipReasonEmptyResultingText diff --git a/internal/framework/runner/feedback_test.go b/internal/framework/runner/feedback_test.go index d881ba2..6a08f96 100644 --- a/internal/framework/runner/feedback_test.go +++ b/internal/framework/runner/feedback_test.go @@ -4,9 +4,11 @@ import ( "context" "fmt" "os" + "strings" "testing" "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" + "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" @@ -137,3 +139,19 @@ func TestFeedbackUsesIndependentPreviewWhenConfidenceInvalid(t *testing.T) { t.Fatalf("result: %+v", result) } } + +func TestFeedbackReportsProtectionAndBlankResult(t *testing.T) { + identity := glossary.Identity{Collection: "party", ID: "jesters"} + protected := fakeValidator{name: "protected_terms", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { + return validators.ValidateProtectedTerms(req) + }} + candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "Jesters", CorrectedText: "", Confidence: .9}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}} + input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "spoken_word", ProtectedTerms: []glossary.Term{{Spelling: "Jesters", Identities: []glossary.Identity{identity}}}, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Jesters"}}}, Validators: []contracts.Validator{protected, validators.NonEmptyResultingSegmentValidator{}}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}} + result, err := validateSectionCandidates(context.Background(), input) + if err != nil { + t.Fatal(err) + } + if len(result.rejected) != 2 || result.rejected[0].ReasonCode != validators.ReasonProtectedGlossaryTerm || !strings.Contains(result.rejected[0].Message, "Jesters") || result.rejected[1].ReasonCode != validators.ReasonEmptyResultingText { + t.Fatalf("findings: %+v", result.rejected) + } +} diff --git a/internal/framework/runner/repair.go b/internal/framework/runner/repair.go index fbb2f4a..22de4e8 100644 --- a/internal/framework/runner/repair.go +++ b/internal/framework/runner/repair.go @@ -6,6 +6,8 @@ import ( "fmt" "path/filepath" + "gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction" + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation" @@ -84,7 +86,15 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput, } } current := original - findings := repairFindings(original.ProposalIndex, initial.rejected) + safeFindings := func(rejected []ValidatorRejectedChange) []proposal_generation.RepairFinding { + findings := repairFindings(original.ProposalIndex, rejected) + redactor := redaction.New(repairSecrets(ctx, input)) + for i := range findings { + findings[i].Explanation = redactor.String(findings[i].Explanation) + } + return findings + } + findings := safeFindings(initial.rejected) replay := &structuredoutput.Replay{Exchange: input.Exchange} for attempt := 0; attempt < input.Spec.Settings.LLM.CorrectionRepairAttempts; attempt++ { payload, err := (proposal_generation.RepairRequest{ProposalRef: position + 1, Correction: proposal_generation.StructuredCorrectionProposal{SegmentReference: localRef, OriginalText: current.OriginalText, CorrectedText: current.CorrectedText, Confidence: current.Confidence}, Findings: findings}).JSON() @@ -100,12 +110,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput, if input.DiagnosticsDir == "" { return } - var secrets []string - if source, ok := input.Spec.RepairClient.(interface { - ConfiguredSecrets(context.Context) []string - }); ok { - secrets = source.ConfiguredSecrets(ctx) - } + secrets := repairSecrets(ctx, input) section := 0 if input.SectionIndex != nil { section = *input.SectionIndex @@ -201,7 +206,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput, initial.filtered = retained break } - findings = repairFindings(original.ProposalIndex, validated.rejected) + findings = safeFindings(validated.rejected) if actionable(original.ProposalIndex, validated.rejected) { if attempt+1 == input.Spec.Settings.LLM.CorrectionRepairAttempts { branch.Outcome = "exhausted" @@ -243,3 +248,15 @@ func repairOperations(input validateSectionCandidatesInput, branches []RepairOut } return result } + +func repairSecrets(ctx context.Context, input validateSectionCandidatesInput) []string { + var secrets []string + for _, client := range []contracts.StructuredLLMClient{input.Spec.RepairClient, input.ValidationLLMClient} { + if source, ok := client.(interface { + ConfiguredSecrets(context.Context) []string + }); ok { + secrets = append(secrets, source.ConfiguredSecrets(ctx)...) + } + } + return secrets +} diff --git a/internal/framework/runner/repair_test.go b/internal/framework/runner/repair_test.go index 95b7aab..c2a1997 100644 --- a/internal/framework/runner/repair_test.go +++ b/internal/framework/runner/repair_test.go @@ -83,6 +83,7 @@ func TestRepairBranchTerminalOutcomes(t *testing.T) { }{ {"withdraw", []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`}, nil, 1, "withdrawn", 1}, {"exhausted", []string{`{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":"cat","corrected_text":"dog","confidence":0.1}}`, `{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":"cat","corrected_text":"dog","confidence":0.2}}`}, nil, 2, "exhausted", 2}, + {"no-effect revision", []string{`{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":"cat","corrected_text":"cat","confidence":0.9}}`}, nil, 2, "rejected", 1}, {"wrong reference", []string{`{"proposal_ref":2,"action":"withdraw","correction":null}`}, nil, 1, "unavailable", 1}, {"provider failure", nil, &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonProviderFailed, Message: "provider failed"}, 1, "unavailable", 1}, } { @@ -240,3 +241,40 @@ func TestRepairArtifactFailureDoesNotChangeOutcome(t *testing.T) { t.Fatalf("branches=%+v warnings=%d err=%v", branches, warnings, err) } } + +func TestNoEffectAndUnavailableOnlyDoNotStartRepair(t *testing.T) { + client := &repairClient{} + for _, rejected := range [][]ValidatorRejectedChange{ + {{ProposalIndex: 0, Disposition: validators.DispositionRejected, ReasonCode: validators.ReasonNoEffect}}, + {{ProposalIndex: 0, Disposition: validators.DispositionUnavailable, ReasonCode: "provider_failed"}}, + } { + candidate := proposals.EnrichedCorrectionProposal{ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}} + input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}} + _, branches, err := repairCandidates(context.Background(), input, nil, validateSectionCandidatesResult{rejected: rejected}, "repair") + if err != nil || len(branches) != 0 || len(client.calls) != 0 { + t.Fatalf("branches=%+v calls=%d err=%v", branches, len(client.calls), err) + } + } +} + +func TestRepairFeedbackRedactsValidationCredentials(t *testing.T) { + client := &repairClient{outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`}} + validation := &repairClient{secrets: []string{"validation-secret"}} + candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .9}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}} + input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, ValidationLLMClient: validation, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}, DiagnosticsDir: t.TempDir()} + initial := validateSectionCandidatesResult{rejected: []ValidatorRejectedChange{{ProposalIndex: 0, ValidatorName: "editorial_review", Disposition: validators.DispositionRejected, ReasonCode: "rejected", Message: "model echoed validation-secret"}}} + _, branches, err := repairCandidates(context.Background(), input, []int{1}, initial, "repair") + if err != nil { + t.Fatal(err) + } + if strings.Contains(client.calls[0].Variables["RepairRequestJSON"], "validation-secret") { + t.Fatalf("credential in feedback: %s", client.calls[0].Variables["RepairRequestJSON"]) + } + data, err := os.ReadFile(branches[0].Attempts[0].Artifacts.RequestMetadataPath) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(data), "validation-secret") { + t.Fatalf("credential in artifact: %s", data) + } +} diff --git a/internal/framework/validators/deterministic.go b/internal/framework/validators/deterministic.go index 94b3bf9..97addf3 100644 --- a/internal/framework/validators/deterministic.go +++ b/internal/framework/validators/deterministic.go @@ -156,7 +156,7 @@ func ValidateProtectedTerms(req Request) (Result, error) { continue } preview := previewForEvidence(req, segment, c.CorrectionProposal) - if preview.Applicable { + if preview.Applicable || (req.IndependentEvidence && preview.SkipReason == proposals.SkipReasonEmptyResultingText) { if glossaryStage { // A correction span may sit inside a longer exempt name. reason = vocab.violationReason(segment.Text, preview.CorrectedSegmentText) diff --git a/internal/framework/validators/protected_terms.go b/internal/framework/validators/protected_terms.go index 0814597..11342e0 100644 --- a/internal/framework/validators/protected_terms.go +++ b/internal/framework/validators/protected_terms.go @@ -1,6 +1,7 @@ package validators import ( + "fmt" "regexp" "sort" "strings" @@ -95,32 +96,35 @@ func (v ProtectedVocabulary) occurrences(text string) []protectedOccurrence { } func validateIdentityPreservation(beforeByID, afterByID map[glossary.Identity][]protectedOccurrence) string { + var spellings []string for identity, beforeItems := range beforeByID { - if len(afterByID[identity]) < len(beforeItems) { - return "proposal may alter protected glossary terminology" + if len(afterByID[identity]) < len(beforeItems) && len(beforeItems) > 0 { + spellings = append(spellings, beforeItems[0].canonical) } } - return "" + return protectedSpellingMessage(spellings) } func validateCapitalizationTransitions(beforeByID, afterByID map[glossary.Identity][]protectedOccurrence) string { + var spellings []string for identity, afterItems := range afterByID { beforeItems := beforeByID[identity] for i, afterItem := range afterItems { - if i >= len(beforeItems) { + if i >= len(beforeItems) || afterItem.text == beforeItems[i].text || afterItem.text == afterItem.canonical { continue } - beforeItem := beforeItems[i] - if afterItem.text == beforeItem.text { - continue - } - if afterItem.text == afterItem.canonical { - continue - } - return "proposal may alter protected glossary terminology" + spellings = append(spellings, afterItem.canonical) } } - return "" + return protectedSpellingMessage(spellings) +} + +func protectedSpellingMessage(spellings []string) string { + if len(spellings) == 0 { + return "" + } + sort.Strings(spellings) + return fmt.Sprintf("proposal may alter protected glossary spelling %q", spellings[0]) } // preservesRepeatedIdentities permits a spoken-word proposal to reduce repeated