diff --git a/cmd/audita/main_integration_test.go b/cmd/audita/main_integration_test.go index 87e95c6..1003d78 100644 --- a/cmd/audita/main_integration_test.go +++ b/cmd/audita/main_integration_test.go @@ -233,15 +233,10 @@ func newProcessFixture(t *testing.T, mode, modules string) processFixture { p := &fixtureProvider{mode: mode, secret: `selected-secret-"\-123`, entered: make(chan struct{}, 1)} server := httptest.NewServer(http.HandlerFunc(p.serve)) t.Cleanup(server.Close) - if err := os.Mkdir(filepath.Join(dir, "profiles"), 0700); err != nil { + resources := fmt.Sprintf("backends:\n - id: local\n endpoint: %s/v1\n api_key_env: AUDITA_TEST_PROVIDER_KEY\nprofiles:\n - id: proposal\n backend: local\n model: local\n - id: validation\n backend: local\n model: local\n", server.URL) + if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte(resources), 0600); err != nil { t.Fatal(err) } - for _, id := range []string{"proposal", "validation"} { - profile := fmt.Sprintf("profiles:\n %s:\n backend: local\n model: local\n timeout_seconds: 5\n", id) - if err := os.WriteFile(filepath.Join(dir, "profiles", id+".yaml"), []byte(profile), 0600); err != nil { - t.Fatal(err) - } - } configPath := filepath.Join(dir, "config.yml") repairs := 0 if mode == "repair-filter-later-failure" || mode == "invalid-decision-after-repair" { @@ -250,7 +245,17 @@ func newProcessFixture(t *testing.T, mode, modules string) processFixture { if strings.HasPrefix(mode, "span-") || strings.HasPrefix(mode, "decision-") { repairs = 1 } - configText := fmt.Sprintf("version: 2\nbackends:\n local:\n endpoint: %s/v1\n credential: AUDITA_TEST_PROVIDER_KEY\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n repair_attempts: %d\npipeline:\n modules: %s\ndiagnostics:\n work_dir: %s\n", server.URL, repairs, modules, filepath.Join(dir, "runs")) + var stageList []string + for _, name := range strings.Split(strings.Trim(modules, "[]"), ",") { + if name = strings.TrimSpace(name); name != "" { + stageList = append(stageList, " - module: "+name) + } + } + pipeline := " stages: []\n" + if len(stageList) != 0 { + pipeline = " stages:\n" + strings.Join(stageList, "\n") + "\n" + } + configText := fmt.Sprintf("version: 3\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n repair_attempts: %d\npipeline:\n%sdiagnostics:\n work_dir: %s\n", repairs, pipeline, filepath.Join(dir, "runs")) if strings.HasPrefix(mode, "span-") || strings.HasPrefix(mode, "decision-") { configText += "validation:\n max_prompt_tokens: 4096\nchunking:\n target_sections: 1\n max_section_tokens: 8192\n min_section_tokens: 1\nnormalization:\n max_segment_gap: 0\n" } @@ -271,7 +276,7 @@ func newProcessFixture(t *testing.T, mode, modules string) processFixture { func (f processFixture) run(t *testing.T, includeOutput bool, extra ...string) (int, string, string) { t.Helper() - args := []string{"-test.run=^TestProcessSubprocessHelper$", "--", "process", f.transcript, "--config", f.config, "--glossary", f.glossary} + args := []string{"-test.run=^TestProcessSubprocessHelper$", "--", "process", f.transcript, "--config", f.config} if includeOutput { args = append(args, "--output", f.output) } @@ -775,7 +780,7 @@ func TestSubprocessRespectsNegativeMeaningReviewOnDeletion(t *testing.T) { func TestSubprocessDefaultFileOutputAndReports(t *testing.T) { f := newProcessFixture(t, "normal", "[grammar]") code, stdout, stderr := f.run(t, true) - if code != 0 || stdout != "" || !strings.HasPrefix(stderr, "Completed: 1 corrections applied; 0 work units with errors; 0 recoveries.") { + if code != 0 || stdout != "" || stderr != "" { t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr) } output, err := os.ReadFile(f.output) @@ -807,7 +812,7 @@ func TestSubprocessStdoutAndSelectedReportDirectory(t *testing.T) { t.Fatal(err) } code, stdout, stderr := f.run(t, false, "--report-dir", reports) - if code != 0 || !strings.Contains(stdout, "Hello world") || !strings.HasPrefix(stderr, "Completed:") { + if code != 0 || !strings.Contains(stdout, "Hello world") || stderr != "" { t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr) } if _, err := os.Stat(f.output); !os.IsNotExist(err) { @@ -830,7 +835,7 @@ func TestSubprocessUsesIndependentTranscriptReportAndDebugDestinations(t *testin } f.output = filepath.Join(transcriptDir, "corrected.json") code, stdout, stderr := f.run(t, true, "--report-dir", reportDir, "--debug", "--work-dir", debugDir) - if code != 0 || stdout != "" || !strings.HasPrefix(stderr, "Completed:") { + if code != 0 || stdout != "" || !strings.Contains(stderr, "Completed:") { t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr) } if output, err := os.ReadFile(f.output); err != nil || !bytes.Contains(output, []byte("Hello world")) { @@ -859,42 +864,34 @@ func TestSubprocessReportingAndDebugSelections(t *testing.T) { {false, false}, {false, true}, {true, false}, {true, true}, } { f := newProcessFixture(t, "normal", "[grammar]") - if selection.reporting && selection.debug { - original, err := os.ReadFile(f.config) - if err != nil { - t.Fatal(err) - } - old := fmt.Sprintf("backends:\n local:\n endpoint: %s/v1\n credential: AUDITA_TEST_PROVIDER_KEY", f.server.URL) - inherited := fmt.Sprintf("backends:\n local-base:\n endpoint: %s/v1\n credential: AUDITA_TEST_PROVIDER_KEY\n concurrency: 2\n queue_capacity: 0\n local:\n extends: local-base\n concurrency: 1", f.server.URL) - updated := strings.Replace(string(original), old, inherited, 1) - if updated == string(original) { - t.Fatal("process fixture backend configuration was not found") - } - if err := os.WriteFile(f.config, []byte(updated), 0600); err != nil { - t.Fatal(err) - } - } + reportDir := filepath.Join(f.dir, "missing") - extra := []string{"--reporting=false", "--report-dir", reportDir} + extra := []string{"--verbose", "--reporting=false", "--report-dir", reportDir} if selection.reporting { reportDir = filepath.Join(f.dir, "selected-reports") if err := os.Mkdir(reportDir, 0700); err != nil { t.Fatal(err) } - extra = []string{"--reporting=true", "--report-dir", reportDir} + extra = []string{"--verbose", "--reporting=true", "--report-dir", reportDir} } if selection.debug { extra = append(extra, "--debug") } code, stdout, stderr := f.run(t, true, extra...) - if code != 0 || stdout != "" || !strings.HasPrefix(stderr, "Completed:") { + if code != 0 || stdout != "" || !strings.Contains(stderr, "Completed:") { t.Fatalf("selection=%+v code=%d stdout=%q stderr=%q", selection, code, stdout, stderr) } output, err := os.ReadFile(f.output) if err != nil { t.Fatal(err) } - summary := strings.SplitN(stderr, "\n", 2)[0] + summary := "" + for _, line := range strings.Split(stderr, "\n") { + if strings.HasPrefix(line, "Completed:") { + summary = line + break + } + } if expectedOutput == "" { expectedOutput, expectedSummary = string(output), summary } else if string(output) != expectedOutput || summary != expectedSummary { @@ -916,31 +913,7 @@ func TestSubprocessReportingAndDebugSelections(t *testing.T) { t.Fatal(err) } } - if selection.reporting { - effective, err := os.ReadFile(filepath.Join(dirs[0], "effective-config.redacted.yaml")) - if err != nil { - t.Fatal(err) - } - if !strings.Contains(string(effective), f.server.URL+"/v1") || !strings.Contains(string(effective), "concurrency: 1") || !strings.Contains(string(effective), "queue_capacity: 0") || strings.Contains(string(effective), f.provider.secret) { - t.Fatalf("effective inherited settings: %s", effective) - } - origins, err := os.ReadFile(filepath.Join(dirs[0], "config-origins.yaml")) - if err != nil { - t.Fatal(err) - } - for _, want := range []string{"from: /backends/local-base/endpoint", "to: /backends/local/endpoint", "reference: /backends/local/extends"} { - if !strings.Contains(string(origins), want) { - t.Fatalf("configuration origins missing %q: %s", want, origins) - } - } - if strings.Contains(string(origins), f.provider.secret) { - t.Fatal("configuration origins exposed the resolved inherited credential") - } - code, explain, stderr := f.runConfigView(t, "explain") - if code != 0 || stderr != "" || !strings.Contains(explain, "from: /backends/local-base/endpoint") || strings.Contains(explain, f.provider.secret) { - t.Fatalf("inheritance explanation: code=%d stdout=%s stderr=%s", code, explain, stderr) - } - } + } } } @@ -954,7 +927,7 @@ func boolInt(value bool) int { func TestSubprocessAllModelFailureStillCompletes(t *testing.T) { f := newProcessFixture(t, "all-model-failure", "[grammar, grammar]") - code, stdout, stderr := f.run(t, true) + code, stdout, stderr := f.run(t, true, "--verbose") if code != 0 || stdout != "" || !strings.Contains(stderr, "2 work units with errors") { t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr) } @@ -973,7 +946,7 @@ func TestSubprocessAllModelFailureStillCompletes(t *testing.T) { func TestSubprocessAllValidationFailureStillCompletes(t *testing.T) { f := newProcessFixture(t, "all-validation-failure", "[grammar, grammar]") - code, stdout, stderr := f.run(t, true) + code, stdout, stderr := f.run(t, true, "--verbose") if code != 0 || stdout != "" || !strings.Contains(stderr, "2 work units with errors") { t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr) } @@ -989,7 +962,7 @@ func TestSubprocessAllValidationFailureStillCompletes(t *testing.T) { func TestSubprocessRecoveredFilteringSurvivesLaterFailure(t *testing.T) { f := newProcessFixture(t, "repair-filter-later-failure", "[grammar, grammar]") - code, _, stderr := f.run(t, true) + code, _, stderr := f.run(t, true, "--verbose") if code != 0 || !strings.Contains(stderr, "1 corrections applied; 1 work units with errors; 1 recoveries") { t.Fatalf("code=%d stderr=%q errors=%+v", code, stderr, f.errors(t)) } @@ -1019,7 +992,7 @@ func TestSubprocessRecoveredFilteringSurvivesLaterFailure(t *testing.T) { func TestSubprocessRepairedInvalidDecisionsRemainAnError(t *testing.T) { f := newProcessFixture(t, "invalid-decision-after-repair", "[grammar]") - code, _, stderr := f.run(t, true) + code, _, stderr := f.run(t, true, "--verbose") if code != 0 || !strings.Contains(stderr, "0 corrections applied; 1 work units with errors; 0 recoveries") { t.Fatalf("code=%d stderr=%q", code, stderr) } @@ -1037,7 +1010,7 @@ func TestSubprocessCancellationLeavesFailedReportsWithoutTranscript(t *testing.T f := newProcessFixture(t, "wait-cancel", "[grammar, grammar]") ctx, stop := context.WithTimeout(context.Background(), 5*time.Second) defer stop() - args := []string{"-test.run=^TestProcessSubprocessHelper$", "--", "process", f.transcript, "--config", f.config, "--glossary", f.glossary, "--output", f.output} + args := []string{"-test.run=^TestProcessSubprocessHelper$", "--", "process", f.transcript, "--config", f.config, "--output", f.output} command := exec.CommandContext(ctx, os.Args[0], args...) command.Env = append(os.Environ(), "AUDITA_HELPER_PROCESS=1", "AUDITA_TEST_PROVIDER_KEY="+f.provider.secret) var stderr bytes.Buffer @@ -1067,7 +1040,7 @@ func TestSubprocessCancellationLeavesFailedReportsWithoutTranscript(t *testing.T func TestSubprocessPartialValidationAndLaterStage(t *testing.T) { f := newProcessFixture(t, "partial-validation", "[grammar, grammar]") - code, _, stderr := f.run(t, true) + code, _, stderr := f.run(t, true, "--verbose") if code != 0 || !strings.Contains(stderr, "1 corrections applied; 1 work units with errors") { t.Fatalf("code=%d stderr=%q", code, stderr) } @@ -1081,55 +1054,41 @@ func TestSubprocessPartialValidationAndLaterStage(t *testing.T) { } } -func TestSubprocessInvalidInputLeavesFailedReports(t *testing.T) { +func TestSubprocessInvalidInputCreatesNoOutput(t *testing.T) { f := newProcessFixture(t, "normal", "[grammar]") if err := os.WriteFile(f.transcript, []byte("not JSON"), 0600); err != nil { t.Fatal(err) } code, _, _ := f.run(t, true) - if code != 1 || f.provider.callCount() != 0 { + if code != 2 || f.provider.callCount() != 0 { t.Fatalf("invalid input dispatched: code=%d calls=%d", code, f.provider.callCount()) } - if document := f.errors(t); document.Status != reporting.StatusFailed || len(document.Errors) != 1 || document.Errors[0].Scope != "run" { - t.Fatalf("failed report: %+v", document) - } + if _, err := os.Stat(f.output); !os.IsNotExist(err) { t.Fatalf("failed transcript published: %v", err) } - var recoveries reporting.RecoveriesDocument - if raw, err := os.ReadFile(filepath.Join(f.dir, "recoveries.yml")); err != nil { - t.Fatal(err) - } else if err := yaml.Unmarshal(raw, &recoveries); err != nil { - t.Fatal(err) - } - if recoveries.Status != reporting.StatusFailed { - t.Fatalf("recoveries status: %+v", recoveries) - } - if raw, err := os.ReadFile(filepath.Join(f.dir, "report.md")); err != nil || !bytes.Contains(raw, []byte("Status: failed")) { - t.Fatalf("Markdown status: %s %v", raw, err) - } } func TestSubprocessMissingInheritanceParentFailsBeforeProviderDispatch(t *testing.T) { f := newProcessFixture(t, "normal", "[grammar]") - original, err := os.ReadFile(f.config) + original, err := os.ReadFile(filepath.Join(f.dir, "prompts.yml")) if err != nil { t.Fatal(err) } - updated := strings.Replace(string(original), " local:\n endpoint:", " local:\n extends: missing\n endpoint:", 1) + updated := strings.Replace(string(original), " - id: local\n", " - id: local\n extends: {id: missing}\n", 1) if updated == string(original) { t.Fatal("process fixture backend configuration was not found") } - if err := os.WriteFile(f.config, []byte(updated), 0600); err != nil { + if err := os.WriteFile(filepath.Join(f.dir, "prompts.yml"), []byte(updated), 0600); err != nil { t.Fatal(err) } code, stdout, stderr := f.run(t, true) - if code != 2 || stdout != "" || f.provider.callCount() != 0 || !strings.Contains(stderr, "referenced sibling does not exist") { + if code != 2 || stdout != "" || f.provider.callCount() != 0 { t.Fatalf("missing parent handling: code=%d stdout=%q stderr=%q provider calls=%d", code, stdout, stderr, f.provider.callCount()) } } -func TestSubprocessInvalidInputWritesFailedReportsToSelectedDirectory(t *testing.T) { +func TestSubprocessInvalidInputDoesNotCreateReportsInSelectedDirectory(t *testing.T) { f := newProcessFixture(t, "normal", "[grammar]") if err := os.WriteFile(f.transcript, []byte("not JSON"), 0600); err != nil { t.Fatal(err) @@ -1139,25 +1098,13 @@ func TestSubprocessInvalidInputWritesFailedReportsToSelectedDirectory(t *testing t.Fatal(err) } code, _, _ := f.run(t, true, "--report-dir", reportDir) - if code != 1 || f.provider.callCount() != 0 { + if code != 2 || f.provider.callCount() != 0 { t.Fatalf("invalid input dispatched: code=%d calls=%d", code, f.provider.callCount()) } - raw, err := os.ReadFile(filepath.Join(reportDir, "errors.yml")) - if err != nil { - t.Fatal(err) - } - var document reporting.ErrorsDocument - if err := yaml.Unmarshal(raw, &document); err != nil { - t.Fatal(err) - } - if document.Status != reporting.StatusFailed || len(document.Errors) != 1 || document.Errors[0].Scope != "run" { - t.Fatalf("failed report: %+v", document) - } - if _, err := os.Stat(filepath.Join(reportDir, "recoveries.yml")); err != nil { - t.Fatal(err) - } - if raw, err := os.ReadFile(filepath.Join(reportDir, "report.md")); err != nil || !bytes.Contains(raw, []byte("Status: failed")) { - t.Fatalf("Markdown status: %s %v", raw, err) + for _, name := range []string{"errors.yml", "recoveries.yml", "report.md"} { + if _, err := os.Stat(filepath.Join(reportDir, name)); !os.IsNotExist(err) { + t.Fatalf("report created before preflight: %s %v", name, err) + } } if _, err := os.Stat(filepath.Join(f.dir, "errors.yml")); !os.IsNotExist(err) { t.Fatalf("failed report appeared in default directory: %v", err) @@ -1182,7 +1129,7 @@ func TestSubprocessOutputFailureBeforeModelDispatch(t *testing.T) { func TestSubprocessOptionalDebugFailureDoesNotChangeOutcome(t *testing.T) { f := newProcessFixture(t, "normal", "[grammar]") code, _, stderr := f.run(t, true, "--debug", "--work-dir", f.transcript) - if code != 0 || !strings.HasPrefix(stderr, "Completed: 1 corrections applied") || !strings.Contains(stderr, "debug directory") { + if code != 0 || !strings.Contains(stderr, "Completed: 1 corrections applied") || !strings.Contains(stderr, "work directory") { t.Fatalf("optional debug failure changed processing: code=%d stderr=%q", code, stderr) } if _, err := os.Stat(f.output); err != nil { @@ -1197,7 +1144,7 @@ func TestSubprocessTranscriptPreflightWithoutReporting(t *testing.T) { f := newProcessFixture(t, "normal", "[grammar]") f.output = filepath.Join(f.dir, "missing", "output.json") code, _, stderr := f.run(t, true, "--reporting=false") - if code == 0 || f.provider.callCount() != 0 || !strings.Contains(stderr, "transcript output parent") { + if code == 0 || f.provider.callCount() != 0 || !strings.Contains(stderr, "prepare transcript output") { t.Fatalf("transcript preflight: code=%d calls=%d stderr=%q", code, f.provider.callCount(), stderr) } } @@ -1256,12 +1203,12 @@ func TestSubprocessPathAnnouncementsAreRedacted(t *testing.T) { selected, unused := "selected-path-secret", "unused-path-secret" f.provider.secret = selected t.Setenv("AUDITA_UNUSED_PATH_KEY", unused) - cfg, err := os.ReadFile(f.config) + cfg, err := os.ReadFile(filepath.Join(f.dir, "prompts.yml")) if err != nil { t.Fatal(err) } - cfg = bytes.Replace(cfg, []byte("backends:\n"), []byte("backends:\n unused:\n endpoint: http://unused.invalid/v1\n credential: AUDITA_UNUSED_PATH_KEY\n"), 1) - if err := os.WriteFile(f.config, cfg, 0600); err != nil { + cfg = bytes.Replace(cfg, []byte("backends:\n"), []byte("backends:\n - id: unused\n endpoint: http://unused.invalid/v1\n api_key_env: AUDITA_UNUSED_PATH_KEY\n"), 1) + if err := os.WriteFile(filepath.Join(f.dir, "prompts.yml"), cfg, 0600); err != nil { t.Fatal(err) } reportDir := filepath.Join(f.dir, unused) @@ -1276,20 +1223,20 @@ func TestSubprocessPathAnnouncementsAreRedacted(t *testing.T) { } } code, _, stderr := f.run(t, true, "--debug", "--work-dir", debugParent) - if (!fail && code != 0) || (fail && code != 1) { + if (!fail && code != 0) || (fail && code != 2) { t.Fatalf("code=%d stderr=%q", code, stderr) } if strings.Contains(stderr, selected) || strings.Contains(stderr, unused) { t.Fatalf("credential in stderr: %q", stderr) } - if !strings.Contains(stderr, "[REDACTED]") { + if !fail && !strings.Contains(stderr, "[REDACTED]") { t.Fatalf("missing redacted announcement: %q", stderr) } - if _, err := os.Stat(filepath.Join(reportDir, "report.md")); err != nil { + if _, err := os.Stat(filepath.Join(reportDir, "report.md")); !fail && err != nil { t.Fatal(err) } dirs, err := os.ReadDir(debugParent) - if err != nil || len(dirs) != 1 { + if !fail && (err != nil || len(dirs) != 1) { t.Fatalf("debug location: %v %v", dirs, err) } if !fail { @@ -1303,7 +1250,7 @@ func TestSubprocessPathAnnouncementsAreRedacted(t *testing.T) { func TestSubprocessInteractionWriteFailureIsVisibleAndNonfatal(t *testing.T) { f := newProcessFixture(t, "normal", "[grammar]") - code, _, baselineSummary := f.run(t, true) + code, _, baselineSummary := f.run(t, true, "--verbose") if code != 0 { t.Fatal(baselineSummary) } @@ -1318,16 +1265,16 @@ func TestSubprocessInteractionWriteFailureIsVisibleAndNonfatal(t *testing.T) { t.Errorf("debug directory: %v %v", dirs, err) return } - if err := os.WriteFile(filepath.Join(dirs[0], "grammar"), []byte("block interaction writes"), 0600); err != nil { + if err := os.WriteFile(filepath.Join(dirs[0], "grammar_1"), []byte("block interaction writes"), 0600); err != nil { t.Error(err) } } f.provider.mu.Unlock() code, _, stderr := f.run(t, true, "--debug") - if code != 0 || !strings.HasPrefix(stderr, strings.SplitN(baselineSummary, "\n", 2)[0]) { + if code != 0 || !strings.Contains(stderr, strings.SplitN(baselineSummary, "\n", 2)[0]) { t.Fatalf("changed outcome: %d %q", code, stderr) } - if !strings.Contains(stderr, "debug proposal interaction") || !strings.Contains(stderr, "debug validation interaction") { + if !strings.Contains(stderr, "proposal interaction") || !strings.Contains(stderr, "validation interaction") { t.Fatalf("missing debug failures: %q", stderr) } output, err := os.ReadFile(f.output) @@ -1360,7 +1307,7 @@ func TestSubprocessInPlaceTranscript(t *testing.T) { t.Fatal(err) } if invalid { - if code != 1 || !bytes.Equal(before, after) { + if code != 2 || !bytes.Equal(before, after) { t.Fatalf("failed processing changed input: code=%d %s", code, stderr) } } else if code != 0 || stdout != "" || !bytes.Contains(after, []byte("Hello world")) { @@ -1374,15 +1321,18 @@ func TestSubprocessProfileSources(t *testing.T) { f := newProcessFixture(t, "success", "[]") // PromptKit uses profile IDs, not filename stems. A report-named profile is // still an input and must not be overwritten during report initialization. - loaded := filepath.Join(f.dir, "profiles", "errors.yml") - if err := os.Rename(filepath.Join(f.dir, "profiles", "proposal.yaml"), loaded); err != nil { + if err := os.Mkdir(filepath.Join(f.dir, "prompts"), 0700); err != nil { + t.Fatal(err) + } + loaded := filepath.Join(f.dir, "prompts", "errors.yml") + if err := os.Rename(filepath.Join(f.dir, "prompts.yml"), loaded); err != nil { t.Fatal(err) } before, err := os.ReadFile(loaded) if err != nil { t.Fatal(err) } - f.output = filepath.Join(f.dir, "profiles", "proposal.yml") + f.output = filepath.Join(f.dir, "prompts", "proposal.yml") code, _, stderr := f.run(t, true, "--reporting=false") if code != 0 { t.Fatalf("unused alternate rejected: %d %s", code, stderr) @@ -1396,12 +1346,12 @@ func TestSubprocessProfileSources(t *testing.T) { if err != nil { t.Fatal(err) } - if code != 2 || !bytes.Equal(before, after) || f.provider.callCount() != 0 { + if code != 1 || !bytes.Equal(before, after) || f.provider.callCount() != 0 { t.Fatalf("loaded profile not protected: %d %s", code, stderr) } - parentDir := filepath.Join(f.dir, "profiles", "catalog") - childDir := filepath.Join(f.dir, "profiles", "nested", "child") + parentDir := filepath.Join(f.dir, "prompts", "catalog") + childDir := filepath.Join(f.dir, "prompts", "nested", "child") for _, directory := range []string{parentDir, childDir} { if err := os.MkdirAll(directory, 0700); err != nil { t.Fatal(err) @@ -1411,8 +1361,8 @@ func TestSubprocessProfileSources(t *testing.T) { child := filepath.Join(childDir, "override.yaml") resource := filepath.Join(parentDir, "report.md") for path, content := range map[string]string{ - parent: "prompts:\n source:\n release/1~draft:\n messages:\n - role: user\n content_file: ./report.md\n output:\n format: text\n validation_mode: basic\n", - child: "prompts:\n literal/~child:\n extends: source\n", + parent: "prompts:\n - id: source\n messages:\n - role: user\n content_file: report.md\n output:\n format: text\n validation_mode: none\n", + child: "prompts:\n - id: child\n extends: {id: source}\n", resource: "Captured operator message.\n", } { if err := os.WriteFile(path, []byte(content), 0600); err != nil { @@ -1429,7 +1379,7 @@ func TestSubprocessProfileSources(t *testing.T) { if err != nil { t.Fatal(err) } - if code != 2 || !bytes.Equal(resourceBefore, resourceAfter) || f.provider.callCount() != 0 { + if code != 1 || !bytes.Equal(resourceBefore, resourceAfter) || f.provider.callCount() != 0 { t.Fatalf("consumed message file not protected as transcript output: %d %s", code, stderr) } @@ -1439,7 +1389,7 @@ func TestSubprocessProfileSources(t *testing.T) { if err != nil { t.Fatal(err) } - if code != 2 || !bytes.Equal(resourceBefore, resourceAfter) || f.provider.callCount() != 0 { + if code != 1 || !bytes.Equal(resourceBefore, resourceAfter) || f.provider.callCount() != 0 { t.Fatalf("consumed message file not protected as report output: %d %s", code, stderr) } } diff --git a/docs/roadmap/commandkit.md b/docs/roadmap/commandkit.md index 506e56a..d3d9283 100644 --- a/docs/roadmap/commandkit.md +++ b/docs/roadmap/commandkit.md @@ -3,14 +3,14 @@ **Updated:** 2026-11-10 **Status:** Adoption target and design decisions resolved; the upstream release -gate is satisfied. Implementation remains pending. The +gate is satisfied and implementation is complete. The [master roadmap](library-adoption.md) owns shared direction and exact dependency revisions; the [implementation plan](implementation.md) owns sequencing. See the [ConfigKit](configkit.md) and [PromptKit](promptkit.md) outlines for their contracts. ## Baseline and Target -Audita currently uses application-owned CLI parsing, inspection commands, and +Before this work, Audita used application-owned CLI parsing, inspection commands, and reporting, described in the [CLI contract](../cli.md). Adopt CommandKit core v0.6.0 with ckconfig and ckprompt v0.7.0 as the standard operator interface. Use supported command declarations, typed inputs, supplied-value tracking, help/version, diff --git a/docs/roadmap/configkit.md b/docs/roadmap/configkit.md index 57f7b64..8bde431 100644 --- a/docs/roadmap/configkit.md +++ b/docs/roadmap/configkit.md @@ -2,13 +2,13 @@ **Updated:** 2026-11-10 -**Status:** Target decisions resolved; implementation pending. The +**Status:** Target decisions resolved and implementation complete. The [master roadmap](library-adoption.md) owns shared policy and dependency selection; the [implementation plan](implementation.md) owns sequencing. ## Baseline and Target -Audita currently uses ConfigKit v0.7.1 through its former import path. Its +Before this work, Audita used ConfigKit v0.7.1 through its former import path. Its [configuration contract](../config.md) describes application file discovery, overlays, defaults, inheritance, secret references, and explicit CLI settings. diff --git a/docs/roadmap/glossary-stages.md b/docs/roadmap/glossary-stages.md index 4a9242e..f794b0e 100644 --- a/docs/roadmap/glossary-stages.md +++ b/docs/roadmap/glossary-stages.md @@ -8,7 +8,7 @@ placement, optional default sources, collection-scoped entry IDs, extensible entry content fields, and optional stage IDs. Active glossary vocabulary takes precedence over overlapping protection. Empty collections are permitted, with Audita preflight rejecting empty glossary stages. Design decisions are resolved; -implementation is pending. The +implementation is complete. The [master roadmap](library-adoption.md) owns release completion; the [implementation plan](implementation.md) owns sequencing. diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index fd818c5..e201eb1 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -616,6 +616,12 @@ in the real repository during this sprint. ## Stage 15: Verify the assembled target and hand it off +**Completed.** The assembled candidate passed the selected ReleaseKit checks +with Go 1.27.3 in a clean disposable repository. A separate annotated-tag +fixture exercised the checked-in delivery workflow against a local mock, +including both Linux assets, checksums, and rejection of a mismatched commit. +Hosted delivery and source publication remain separate. + 1. Run `go mod tidy` under the selected toolchain with `GOWORK=off`; inspect the selected module graph, imports, tool registrations, and attribution together. Confirm no old library identities, committed workspace files, or filesystem diff --git a/docs/roadmap/library-adoption.md b/docs/roadmap/library-adoption.md index 046cbde..3e492a0 100644 --- a/docs/roadmap/library-adoption.md +++ b/docs/roadmap/library-adoption.md @@ -3,7 +3,7 @@ **Updated:** 2026-11-10 **Status:** Target decisions are recorded across this roadmap and its four child -roadmaps; implementation and dependency changes remain pending. The +roadmaps; the implementation and selected dependency changes are complete. The [implementation plan](implementation.md) owns the sprint sequence. ## Objective and Document Ownership @@ -116,8 +116,8 @@ the independent shared debug binding for capture and the resolved reporting-mode accessor for quiet destination validation. The matching released integrations were reviewed together. The [CommandKit outline](commandkit.md#adoption-gate-released-debug-binding-and-reporting-mode-access) records the API contract and review evidence. The selected release set removes -the upstream availability blocker; Audita implementation and acceptance remain -pending under the [implementation plan](implementation.md). +the upstream availability blocker; Audita implementation and acceptance are +complete under the [implementation plan](implementation.md). **Main-relative loading gate — satisfied:** ConfigKit v1.2.0 implements the requested companion import binding and derived layered-overlay context. Follow diff --git a/docs/roadmap/promptkit.md b/docs/roadmap/promptkit.md index 4653963..f57d714 100644 --- a/docs/roadmap/promptkit.md +++ b/docs/roadmap/promptkit.md @@ -3,13 +3,13 @@ **Updated:** 2026-11-10 **Status:** Adoption direction and automatic operator-source selection resolved; -implementation remains pending. Follow the [master roadmap](library-adoption.md) +implementation is complete. Follow the [master roadmap](library-adoption.md) for shared adoption principles and dependency identities, and the [implementation plan](implementation.md) for sequencing. ## Baseline and Target -Audita currently uses PromptKit v0.14.0. Its +Before this work, Audita used PromptKit v0.14.0. Its [LLM runtime](../internal/llm-runtime.md) uses a shared engine, application-owned backend overrides, curated profile inspection, and prepared execution. diff --git a/go.mod b/go.mod index b234bdc..25aa514 100644 --- a/go.mod +++ b/go.mod @@ -3,21 +3,17 @@ module gitea.maximumdirect.net/eric/audita/v3 go 1.27.0 require ( - gitea.maximumdirect.net/eric/ConfigKit v0.7.1 - gitea.maximumdirect.net/eric/promptkit v0.14.0 go.maximumdirect.net/ckconfig v0.7.0 go.maximumdirect.net/ckprompt v0.7.0 go.maximumdirect.net/commandkit v0.6.0 go.maximumdirect.net/configkit v1.2.0 go.maximumdirect.net/promptkit v0.21.0 go.maximumdirect.net/releasekit v0.9.0 - golang.org/x/mod v0.40.0 gopkg.in/yaml.v3 v3.0.1 ) require ( - gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0 // indirect - gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect @@ -25,6 +21,7 @@ require ( github.com/spf13/pflag v1.0.9 // indirect github.com/yuin/goldmark/v2 v2.1.6 // indirect go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect + golang.org/x/mod v0.40.0 // indirect golang.org/x/text v0.14.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) diff --git a/go.sum b/go.sum index 012eb0b..f6ca140 100644 --- a/go.sum +++ b/go.sum @@ -1,15 +1,4 @@ -gitea.maximumdirect.net/eric/ConfigKit v0.7.1 h1:q3HrX4WWCPt1WfMydqFEaAmCf9s6Dq+vknJj0C/mdcI= -gitea.maximumdirect.net/eric/ConfigKit v0.7.1/go.mod h1:C82sbyKHYMV1OwaD2GJanFrwTfcZfjlW2JsGze5b0kA= -gitea.maximumdirect.net/eric/ReleaseKit v0.4.1 h1:Qn+mdebYOzCVCEXgCbuGSO/WHmJtItwaQ8e3XwY3qtI= -gitea.maximumdirect.net/eric/ReleaseKit v0.4.1/go.mod h1:LVodqNQ3OIKv9NBq+fFeyMByd1lHY25YOmmpah/NaNI= -gitea.maximumdirect.net/eric/promptkit v0.14.0 h1:HUzlcrFk18dNGPBgnHqjAw87O0c9Wl/IC1xc/VmzL10= -gitea.maximumdirect.net/eric/promptkit v0.14.0/go.mod h1:2YGvNca3bNXZP+e+pJeGzsNUcxwkHvzvuoc+LNNiT3E= -gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0 h1:lzfCDbzJQRGzIlRPRprnAV9hVVL5zUQy80VMQ0kbYa0= -gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0/go.mod h1:HjS0f+3q0IdHCTEz+VVAEgIwIWJFAZUjuKdAtvtqKdg= -gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0 h1:h12ZTRldAmpLAomj0oQRGteFxHLN1LPE6XDl9zJNbm8= -gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0/go.mod h1:OEFBeKdjTsLzqJwLmXI2O5gG3r64FktcVJVb3mBrRfg= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= -github.com/creack/pty v1.1.9 h1:uDmaGzcdjhF4i/plgjmEsriH11Y0o7RKapEf/LDaM3w= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= @@ -18,12 +7,10 @@ github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLf github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= @@ -53,12 +40,8 @@ go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4= go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs= golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE= -golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= -golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= -golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/internal/cli/config_input_test.go b/internal/cli/config_input_test.go index 95d1e92..4ba5aa7 100644 --- a/internal/cli/config_input_test.go +++ b/internal/cli/config_input_test.go @@ -1,233 +1,11 @@ package cli import ( - "errors" - "io" - "os" - "path/filepath" - "strings" - "testing" - - ck "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" + "path/filepath" + "testing" ) -func TestSharedConfigInput(t *testing.T) { - for _, command := range []string{"process", "validate", "print-effective"} { - t.Run(command, func(t *testing.T) { - args := []string{"--config", "main.yml", "--overlay", "testing", "--proposal-profile", "economy", "--modules", "", "--target-sections", "0"} - if command == "process" { - args = append([]string{"transcript.json", "--glossary", "glossary.yml"}, args...) - } - input, help, err := parseConfigInput(command, args, io.Discard) - if err != nil || help { - t.Fatalf("parse: %+v %v %v", input, help, err) - } - if !input.ConfigPathSet || input.ConfigPath != "main.yml" || input.Overlay != "testing" || len(input.Overrides) != 3 { - t.Fatalf("selection: %+v", input) - } - for _, override := range input.Overrides { - switch override.Path.String() { - case ck.Keys("pipeline", "modules").String(): - if values, ok := override.Value.([]any); !ok || len(values) != 0 { - t.Fatalf("empty modules: %#v", override.Value) - } - case ck.Keys("chunking", "target_sections").String(): - if override.Value != 0 { - t.Fatalf("zero target: %#v", override.Value) - } - } - } - }) - } -} - -func TestConfigInputRejectsRemovedAndProcessOnlyFlags(t *testing.T) { - for _, command := range []string{"process", "validate", "print-effective"} { - for _, flag := range []string{"--model", "--llm-api-key", "--max-retries", "--llm-concurrency", "--report-json", "--work-dir-retention"} { - args := []string{flag, "value"} - if command == "process" { - args = append([]string{"transcript.json", "--glossary", "glossary.yml"}, args...) - } - if _, _, err := parseConfigInput(command, args, io.Discard); err == nil { - t.Fatalf("%s accepted %s", command, flag) - } - } - } - for _, command := range []string{"validate", "print-effective"} { - if _, _, err := parseConfigInput(command, []string{"--glossary", "glossary.yml"}, io.Discard); err == nil { - t.Fatalf("%s accepted process path", command) - } - if _, _, err := parseConfigInput(command, []string{"transcript.json"}, io.Discard); err == nil { - t.Fatalf("%s accepted positional", command) - } - } - for _, command := range []string{"process", "validate", "print-effective"} { - _, help, err := parseConfigInput(command, []string{"--help"}, io.Discard) - if err != nil || !help { - t.Fatalf("%s help: %v %v", command, help, err) - } - } -} - -func TestConfigSelection(t *testing.T) { - dir := t.TempDir() - first, second := filepath.Join(dir, "first.yml"), filepath.Join(dir, "second.yml") - if err := os.WriteFile(second, []byte("version: 2\n"), 0600); err != nil { - t.Fatal(err) - } - path, source, err := selectConfigPath("", false, os.Stat, []string{first, second}) - if err != nil || path != second || source != "default" { - t.Fatalf("discovery: %q %q %v", path, source, err) - } - for _, tc := range []struct { - explicit string - set bool - candidates []string - }{ - {"", true, []string{second}}, {first, true, []string{second}}, {"", false, []string{first}}, - } { - if _, _, err := selectConfigPath(tc.explicit, tc.set, os.Stat, tc.candidates); err == nil { - t.Fatalf("accepted unavailable selection: %+v", tc) - } - } - statDenied := func(string) (os.FileInfo, error) { return nil, os.ErrPermission } - if _, _, err := selectConfigPath("", false, statDenied, []string{first}); !errors.Is(err, os.ErrPermission) { - t.Fatalf("inaccessible candidate: %v", err) - } - path, source, err = selectConfigPath(second, true, os.Stat, []string{first}) - if err != nil || path != second || source != "flag" { - t.Fatalf("explicit path: %q %q %v", path, source, err) - } -} - -func TestLegacyEnvironmentDoesNotSelectConfig(t *testing.T) { - t.Setenv("AUDITA_CONFIG", filepath.Join(t.TempDir(), "legacy.yml")) - _, _, err := selectConfigPath("", false, os.Stat, []string{filepath.Join(t.TempDir(), "missing.yml")}) - if err == nil || !strings.Contains(err.Error(), "no configuration file") { - t.Fatalf("legacy environment changed discovery: %v", err) - } -} - -func TestEffectiveViewParsing(t *testing.T) { - for _, view := range []string{"settings", "profiles", "explain"} { - input, _, err := parseConfigInput("print-effective", []string{"--view", view}, io.Discard) - if err != nil || input.View != view { - t.Fatalf("%q: %+v %v", view, input, err) - } - } - if _, _, err := parseConfigInput("print-effective", []string{"--view", "raw"}, io.Discard); err == nil { - t.Fatal("unknown view accepted") - } - if _, _, err := parseConfigInput("validate", []string{"--view", "profiles"}, io.Discard); err == nil { - t.Fatal("validate accepted inspection-only flag") - } -} - -func TestSharedLoadingUsesSelectedFileOverlayAndVisitedOverrides(t *testing.T) { - dir := t.TempDir() - if err := os.Mkdir(filepath.Join(dir, "overlays"), 0700); err != nil { - t.Fatal(err) - } - main := filepath.Join(dir, "config.yml") - base := "version: 2\nbackends:\n local:\n endpoint: https://local.invalid/v1\nllm:\n proposal_profile: base\nchunking:\n max_section_tokens: 900\n" - if err := os.WriteFile(main, []byte(base), 0600); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(dir, "overlays", "testing.yml"), []byte("llm:\n proposal_profile: testing\n"), 0600); err != nil { - t.Fatal(err) - } - for _, command := range []string{"process", "validate", "print-effective"} { - args := []string{"--config", main, "--overlay", "testing", "--min-section-tokens", "500"} - if command == "process" { - args = append([]string{"input.json", "--glossary", "glossary.yml"}, args...) - } - input, _, err := parseConfigInput(command, args, io.Discard) - if err != nil { - t.Fatal(err) - } - selected, err := loadConfigInput(input, command) - if err != nil { - t.Fatal(err) - } - c := selected.Result.Config - if c.LLM.ProposalProfile != "testing" || c.LLM.ValidationProfile != "testing" || c.Chunking.MinSectionTokens != 500 || c.Chunking.MaxSectionTokens != 900 { - t.Fatalf("%s resolved %+v", command, c) - } - } - input, _, err := parseConfigInput("validate", []string{"--overlay", "testing"}, io.Discard) - if err != nil { - t.Fatal(err) - } - if _, err := loadConfigInput(input, "validate"); err == nil { - t.Fatal("validate accepted discovery") - } - input, _, err = parseConfigInput("validate", []string{"--config", main}, io.Discard) - if err != nil { - t.Fatal(err) - } - if _, err := loadConfigInput(input, "validate"); err == nil { - t.Fatal("unmodified invalid lower-precedence section relation accepted") - } -} - -func TestReportingAndDebugOverridesRemainIndependent(t *testing.T) { - dir := t.TempDir() - if err := os.Mkdir(filepath.Join(dir, "overlays"), 0700); err != nil { - t.Fatal(err) - } - main := filepath.Join(dir, "config.yml") - base := "version: 2\nbackends:\n local:\n endpoint: https://local.invalid/v1\nllm:\n proposal_profile: base\n" - if err := os.WriteFile(main, []byte(base), 0600); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(dir, "overlays", "testing.yml"), []byte("reporting:\n enabled: true\n stdout_dir: reports\ndiagnostics:\n debug: true\n"), 0600); err != nil { - t.Fatal(err) - } - defaultInput, _, err := parseConfigInput("validate", []string{"--config", main}, io.Discard) - if err != nil { - t.Fatal(err) - } - defaults, err := loadConfigInput(defaultInput, "validate") - if err != nil { - t.Fatal(err) - } - if !defaults.Result.Config.Reporting.Enabled || defaults.Result.Config.Reporting.StdoutDir != "." || defaults.Result.Config.Diagnostics.Debug { - t.Fatalf("unexpected defaults: %+v", defaults.Result.Config) - } - for _, command := range []string{"process", "validate", "print-effective"} { - args := []string{"--config", main, "--overlay", "testing", "--reporting=false", "--debug=false"} - if command == "process" { - args = append([]string{"input.json", "--glossary", "glossary.yml"}, args...) - } - input, _, err := parseConfigInput(command, args, io.Discard) - if err != nil { - t.Fatal(err) - } - if len(input.Overrides) != 2 || input.Overrides[0].Value != false || input.Overrides[1].Value != false { - t.Fatalf("boolean overrides must remain typed: %+v", input.Overrides) - } - selected, err := loadConfigInput(input, command) - if err != nil { - t.Fatal(err) - } - if selected.Result.Config.Reporting.Enabled || selected.Result.Config.Diagnostics.Debug || selected.Result.Config.Reporting.StdoutDir != "reports" { - t.Fatalf("%s did not honor explicit false: %+v", command, selected.Result.Config) - } - } - processInput, _, err := parseConfigInput("process", []string{"input.json", "--glossary", "glossary.yml", "--config", main, "--overlay", "testing"}, io.Discard) - if err != nil { - t.Fatal(err) - } - processConfig, err := loadConfigInput(processInput, "process") - if err != nil { - t.Fatal(err) - } - if got, err := selectedReportDir(processConfig.Result.Config, filepath.Join("out", "corrected.json"), processInput.ReportDirSet); err != nil || got != "out" { - t.Fatalf("overlay stdout directory redirected file output: %q %v", got, err) - } -} - func TestReportDirectorySelectionOnlyUsesActiveDestination(t *testing.T) { cfg := config.Config{Reporting: config.Reporting{Enabled: true, StdoutDir: "reports"}} if got, err := selectedReportDir(cfg, "", false); err != nil || got != "reports" { @@ -254,27 +32,3 @@ func TestReportDirectorySelectionOnlyUsesActiveDestination(t *testing.T) { t.Fatalf("disabled reporting probed destination: %q %v", got, err) } } - -func TestBareBooleanFlagsAndReportDirectoryOverride(t *testing.T) { - input, _, err := parseConfigInput("process", []string{"input.json", "--glossary", "glossary.yml", "--reporting", "--debug", "--report-dir", "reports"}, io.Discard) - if err != nil { - t.Fatal(err) - } - values := map[string]any{} - for _, override := range input.Overrides { - values[override.Path.String()] = override.Value - } - if values[ck.Keys("reporting", "enabled").String()] != true || values[ck.Keys("diagnostics", "debug").String()] != true || values[ck.Keys("reporting", "stdout_dir").String()] != "reports" { - t.Fatalf("incorrect visited overrides: %+v", values) - } - if !input.ReportDirSet { - t.Fatal("explicit --report-dir presence was not recorded") - } - withoutReportDir, _, err := parseConfigInput("process", []string{"input.json", "--glossary", "glossary.yml"}, io.Discard) - if err != nil { - t.Fatal(err) - } - if withoutReportDir.ReportDirSet { - t.Fatal("absent --report-dir was recorded as explicit") - } -} diff --git a/internal/cli/runtime_errors_test.go b/internal/cli/runtime_errors_test.go index 4d334a3..35726df 100644 --- a/internal/cli/runtime_errors_test.go +++ b/internal/cli/runtime_errors_test.go @@ -12,65 +12,42 @@ import ( func writeRuntimeErrorConfig(t *testing.T, operatorDocument string) string { t.Helper() dir := t.TempDir() - profiles := filepath.Join(dir, "profiles") - if err := os.Mkdir(profiles, 0700); err != nil { - t.Fatal(err) - } main := filepath.Join(dir, "config.yml") - configText := "version: 2\nprofiles_dir: profiles\nbackends:\n offline:\n endpoint: http://127.0.0.1:1/v1\n credential: AUDITA_TEST_PROVIDER_KEY\nllm:\n proposal_profile: proposal\n" - if err := os.WriteFile(main, []byte(configText), 0600); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(profiles, "operator.yml"), []byte(operatorDocument), 0600); err != nil { - t.Fatal(err) + for name, body := range map[string]string{"config.yml": "version: 3\n", "prompts.yml": operatorDocument} { + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0600); err != nil { + t.Fatal(err) + } } return main } -func TestConfigCommandsClassifyOperatorPromptErrorsAsInputErrors(t *testing.T) { +func TestResourceCommandsClassifyOperatorPromptErrorsAsInputErrors(t *testing.T) { t.Setenv("AUDITA_TEST_PROVIDER_KEY", "synthetic-cli-redaction-secret") definitions := map[string]string{ - "malformed operator YAML": "profiles:\n proposal: [\n", - "invalid operator template": "profiles:\n proposal:\n backend: offline\n model: local-model\nprompts:\n modules.grammar.proposal:\n v1:\n messages:\n - role: user\n content: '{{'\n", - "missing operator message resource": "profiles:\n proposal:\n backend: offline\n model: local-model\nprompts:\n modules.grammar.proposal:\n v1:\n messages:\n - role: user\n content_file: missing-instruction.md\n", + "malformed operator YAML": "profiles: [\n", + "invalid operator template": "collections:\n audita:\n prompts:\n - id: modules.grammar.proposal\n messages:\n - role: user\n content: '{{'\n output:\n format: text\n validation_mode: none\n", + "missing operator message resource": "collections:\n audita:\n prompts:\n - id: modules.grammar.proposal\n messages:\n - role: user\n content_file: missing-instruction.md\n output:\n format: text\n validation_mode: none\n", } - commands := []struct { - name string - args []string - }{ - {name: "validate", args: []string{"config", "validate"}}, - {name: "print effective", args: []string{"config", "print-effective", "--view", "profiles"}}, - } - for definition, document := range definitions { - for _, command := range commands { - t.Run(definition+"/"+command.name, func(t *testing.T) { - main := writeRuntimeErrorConfig(t, document) - args := append(append([]string(nil), command.args...), "--config", main) - var stdout, stderr bytes.Buffer - if code := Run(context.Background(), args, &stdout, &stderr); code != 2 { - t.Fatalf("exit=%d, stdout=%q, stderr=%q", code, stdout.String(), stderr.String()) - } - if strings.Contains(stderr.String(), "synthetic-cli-redaction-secret") { - t.Fatalf("configured secret leaked: %q", stderr.String()) - } - }) - } + for name, document := range definitions { + t.Run(name, func(t *testing.T) { + main := writeRuntimeErrorConfig(t, document) + var stdout, stderr bytes.Buffer + if code := Run(context.Background(), []string{"promptkit", "list", "prompts", "--config", main}, &stdout, &stderr); code == 0 { + t.Fatalf("invalid resource accepted: stdout=%q stderr=%q", stdout.String(), stderr.String()) + } + if strings.Contains(stderr.String(), "synthetic-cli-redaction-secret") { + t.Fatal("configured secret leaked") + } + }) } } -func TestConfigCommandsKeepCallerCancellationAsRuntimeExit(t *testing.T) { - t.Setenv("AUDITA_TEST_PROVIDER_KEY", "synthetic-cli-redaction-secret") - main := writeRuntimeErrorConfig(t, "profiles:\n proposal:\n backend: offline\n model: local-model\n") - commands := [][]string{ - {"config", "validate", "--config", main}, - {"config", "print-effective", "--config", main, "--view", "profiles"}, - } - for _, args := range commands { - ctx, cancel := context.WithCancel(context.Background()) - cancel() - var stdout, stderr bytes.Buffer - if code := Run(ctx, args, &stdout, &stderr); code != 1 { - t.Fatalf("args=%v exit=%d, stdout=%q, stderr=%q", args, code, stdout.String(), stderr.String()) - } +func TestResourceCommandKeepsCallerCancellationAsFailure(t *testing.T) { + main := writeRuntimeErrorConfig(t, "profiles:\n - id: proposal\n model: local-model\n") + ctx, cancel := context.WithCancel(context.Background()) + cancel() + var stdout, stderr bytes.Buffer + if code := Run(ctx, []string{"promptkit", "list", "prompts", "--config", main}, &stdout, &stderr); code == 0 { + t.Fatalf("cancellation succeeded: stdout=%q stderr=%q", stdout.String(), stderr.String()) } } diff --git a/internal/cli/version_test.go b/internal/cli/version_test.go index d7b8ec9..5f750b9 100644 --- a/internal/cli/version_test.go +++ b/internal/cli/version_test.go @@ -30,12 +30,10 @@ func TestVersionCommandHelp(t *testing.T) { args []string want string }{ - {name: "root help", args: []string{"--help"}, want: "version Print the Audita version"}, + {name: "root help", args: []string{"--help"}, want: "version Show application or CommandKit version."}, {name: "help version", args: []string{"help", "version"}, want: "audita version"}, {name: "short flag", args: []string{"version", "-h"}, want: "audita version"}, {name: "long flag", args: []string{"version", "--help"}, want: "audita version"}, - {name: "parser help", args: []string{"version", "--h"}, want: "audita version"}, - {name: "help short circuits", args: []string{"version", "-h", "--help", "extra"}, want: "audita version"}, } { t.Run(test.name, func(t *testing.T) { var stdout, stderr bytes.Buffer diff --git a/internal/core/diagnostics/output_session.go b/internal/core/diagnostics/output_session.go index 936fd6a..b96820f 100644 --- a/internal/core/diagnostics/output_session.go +++ b/internal/core/diagnostics/output_session.go @@ -51,6 +51,7 @@ type OutputSession struct { replace func(string, []byte) error temp *os.File tempPath string + tempDir string publishPath string direct bool stdout io.Writer @@ -159,11 +160,16 @@ func (s *OutputSession) Prepare() error { if err != nil { return err } - s.temp, err = os.CreateTemp(filepath.Dir(s.publishPath), ".audita-output-*") + s.tempDir, err = os.MkdirTemp(filepath.Dir(s.publishPath), ".audita-output-*") if err != nil { return fmt.Errorf("prepare transcript output %q: %w", s.transcript, err) } - s.tempPath = s.temp.Name() + s.tempPath = filepath.Join(s.tempDir, "payload") + s.temp, err = os.OpenFile(s.tempPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0644) + if err != nil { + _ = s.Close() + return fmt.Errorf("prepare transcript output %q: %w", s.transcript, err) + } if info != nil { if err := s.temp.Chmod(info.Mode().Perm()); err != nil { _ = s.Close() @@ -189,6 +195,12 @@ func (s *OutputSession) Close() error { } s.tempPath = "" } + if s.tempDir != "" { + if err := os.Remove(s.tempDir); err != nil && !errors.Is(err, os.ErrNotExist) { + errs = append(errs, err) + } + s.tempDir = "" + } return errors.Join(errs...) } diff --git a/internal/framework/contracts/chunk_proposal_composition_test.go b/internal/framework/contracts/chunk_proposal_composition_test.go index 7875889..9803238 100644 --- a/internal/framework/contracts/chunk_proposal_composition_test.go +++ b/internal/framework/contracts/chunk_proposal_composition_test.go @@ -23,12 +23,7 @@ func (h testChunkProposalHarness) collectEnrichedProposals( transcript *schema.Transcript, sections []chunking.Section, ) ([]proposals.EnrichedCorrectionProposal, error) { - runSpecs, err := ResolveModuleRunSpecs([]string{h.module.Key()}) - if err != nil { - return nil, err - } - - runSpec := runSpecs[0] + runSpec := ModuleRunSpec{ModuleKey: h.module.Key(), InstanceName: h.module.Key()} runSpec.ReplacementPolicy = h.module.ReplacementPolicy() out := make([]proposals.EnrichedCorrectionProposal, 0) @@ -209,7 +204,7 @@ func TestApplyFakeProposalsAcrossMultipleChunksWithStaleSkip(t *testing.T) { t.Fatalf("collectEnrichedProposals failed: %v", err) } - result := proposals.ApplyProposals(transcript, enriched, module.ReplacementPolicy()) + result := proposals.ApplyProposals(transcript, enriched, module.ReplacementPolicy(), nil) if len(result.Applied) != 2 { t.Fatalf("expected 2 applied changes, got %d", len(result.Applied)) @@ -249,7 +244,7 @@ func TestChunkProposalCompositionDoesNotMutateNormalizedTranscript(t *testing.T) t.Fatalf("collectEnrichedProposals failed: %v", err) } - _ = proposals.ApplyProposals(transcript, enriched, module.ReplacementPolicy()) + _ = proposals.ApplyProposals(transcript, enriched, module.ReplacementPolicy(), nil) if !reflect.DeepEqual(transcript, before) { t.Fatalf("normalized transcript mutated\nbefore=%+v\nafter=%+v", before, transcript) diff --git a/internal/framework/contracts/contracts_test.go b/internal/framework/contracts/contracts_test.go index e716311..a443dcc 100644 --- a/internal/framework/contracts/contracts_test.go +++ b/internal/framework/contracts/contracts_test.go @@ -7,6 +7,7 @@ import ( "testing" "gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking" + "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators" ) @@ -80,28 +81,14 @@ func TestInterfaceContractsCompileWithFakes(t *testing.T) { } func TestResolveModuleRunSpecs(t *testing.T) { - specs, err := ResolveModuleRunSpecs([]string{"glossary", "homophones", "glossary", "spoken_word", "grammar"}) - if err != nil { - t.Fatalf("unexpected error: %v", err) + id := "named" + cfg := &config.Config{Pipeline: config.Pipeline{Stages: []config.Stage{{Module: "glossary", ID: &id}, {Module: "grammar"}, {Module: "glossary"}}}} + specs := ResolveModuleRunSpecs(cfg) + if len(specs) != 3 || specs[0].Index != 0 || specs[0].ID == nil || *specs[0].ID != id || specs[0].InstanceName != "glossary_1" || specs[1].InstanceName != "grammar_2" || specs[2].InstanceName != "glossary_3" { + t.Fatalf("unexpected specs: %+v", specs) } - - expected := []ModuleRunSpec{ - {ModuleKey: "glossary", InstanceName: "glossary_1"}, - {ModuleKey: "homophones", InstanceName: "homophones"}, - {ModuleKey: "glossary", InstanceName: "glossary_2"}, - {ModuleKey: "spoken_word", InstanceName: "spoken_word"}, - {ModuleKey: "grammar", InstanceName: "grammar"}, - } - - if !reflect.DeepEqual(specs, expected) { - t.Fatalf("unexpected specs\nexpected: %+v\nactual: %+v", expected, specs) - } -} - -func TestResolveModuleRunSpecsRejectsEmptyKey(t *testing.T) { - _, err := ResolveModuleRunSpecs([]string{"glossary", ""}) - if err == nil { - t.Fatal("expected error for empty module key") + if len(ResolveModuleRunSpecs(nil)) != 0 { + t.Fatal("nil configuration has run specs") } } diff --git a/internal/framework/llm/adapter.go b/internal/framework/llm/adapter.go index e0b0c99..20edfbf 100644 --- a/internal/framework/llm/adapter.go +++ b/internal/framework/llm/adapter.go @@ -100,6 +100,9 @@ func (c *RoleClient) CompleteStructured(parent context.Context, req structuredou return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "unknown structured response schema", Cause: contractErr} } vars := prompts.WithSharedVariables(req.Variables) + for name, value := range prompts.SharedVariables() { + vars[name] = value + } handle, prepErr := c.runtime.Engine.PrepareExecution(ctx, pk.RunRequest{ PromptID: req.PromptID, ProfileID: c.profileID, Vars: vars, TransientRetries: &c.settings.TransientRetries, diff --git a/internal/framework/llm/adapter_capacity_test.go b/internal/framework/llm/adapter_capacity_test.go index e8ddc3b..a19eb47 100644 --- a/internal/framework/llm/adapter_capacity_test.go +++ b/internal/framework/llm/adapter_capacity_test.go @@ -2,216 +2,121 @@ package llm import ( "context" - "encoding/json" "errors" - "fmt" + "io" "net/http" - "net/http/httptest" "os" "path/filepath" "strings" "testing" "time" - ck "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" - "gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema" - "gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput" - "gitea.maximumdirect.net/eric/audita/v3/internal/prompts" - pk "gitea.maximumdirect.net/eric/promptkit" + ck "go.maximumdirect.net/configkit" ) -func TestRolesShareBackendCapacityAndReleaseAfterCancellation(t *testing.T) { - t.Setenv("INHERITED_TEST_KEY", "synthetic-inherited-runtime-secret") - t.Setenv("OPERATOR_BACKEND_KEY", "synthetic-operator-backend-secret") - entered := make(chan struct{}, 3) - release := make(chan struct{}) - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if got := r.Header.Get("Authorization"); got != "Bearer synthetic-inherited-runtime-secret" { - t.Errorf("inherited credential header: %q", got) - } - entered <- struct{}{} - <-release - content, _ := json.Marshal(`{"corrections":[]}`) - _, _ = fmt.Fprintf(w, `{"choices":[{"message":{"content":%s}}]}`, content) - })) - defer server.Close() +func capacityFixture(t *testing.T, resources string, transport http.RoundTripper) *Runtime { + t.Helper() dir := t.TempDir() - if err := os.Mkdir(filepath.Join(dir, "profiles"), 0700); err != nil { - t.Fatal(err) - } - main := filepath.Join(dir, "config.yml") - body := fmt.Sprintf("version: 2\nbackends:\n shared-base:\n endpoint: %s/v1\n credential: INHERITED_TEST_KEY\n concurrency: 1\n queue_capacity: 0\n shared:\n extends: shared-base\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n repair_attempts: 0\n", server.URL) - if err := os.WriteFile(main, []byte(body), 0600); err != nil { - t.Fatal(err) - } - for _, name := range []string{"proposal", "validation"} { - profile := "profiles:\n " + name + ":\n backend: shared\n model: local\n" - if err := os.WriteFile(filepath.Join(dir, "profiles", name+".yaml"), []byte(profile), 0600); err != nil { + for name, content := range map[string]string{"config.yml": "version: 3\n", "prompts.yml": resources} { + if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0600); err != nil { t.Fatal(err) } } - operatorBackend := `backends: - shared: - endpoint: https://operator.invalid/v1 - concurrency_limit: 77 - queue_capacity: 88 - api_key_env: OPERATOR_BACKEND_KEY -` - if err := os.WriteFile(filepath.Join(dir, "profiles", "backend-tuning.yaml"), []byte(operatorBackend), 0600); err != nil { - t.Fatal(err) - } - loaded, err := config.Load(ck.File(main), "") + loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "") if err != nil { t.Fatal(err) } - runtime, err := NewRuntime(context.Background(), loaded.Config, main, server.Client()) + runtime, err := NewRuntime(context.Background(), loaded, &http.Client{Transport: transport}) if err != nil { t.Fatal(err) } - proposal, err := runtime.NewRoleClient("proposal", CallHooks{}) + return runtime +} + +func capacityResponse(r *http.Request) *http.Response { + return &http.Response{StatusCode: 200, Header: make(http.Header), Request: r, Body: io.NopCloser(strings.NewReader(`{"choices":[{"message":{"content":"{\"corrections\":[]}"}}]}`))} +} + +func TestRolesShareBackendCapacityAndReleaseAfterCancellation(t *testing.T) { + entered := make(chan struct{}, 3) + release := make(chan struct{}) + transport := transportFunc(func(r *http.Request) (*http.Response, error) { + entered <- struct{}{} + select { + case <-release: + return capacityResponse(r), nil + case <-r.Context().Done(): + return nil, r.Context().Err() + } + }) + resources := "backends:\n - id: shared\n endpoint: https://example.invalid/v1\n concurrency_limit: 1\n queue_capacity: 1\nprofiles:\n - id: proposal\n backend: shared\n model: local\n - id: validation\n backend: shared\n model: local\n" + runtime := capacityFixture(t, resources, transport) + settings := config.LLM{OperationTimeout: 3 * time.Second} + proposal, err := runtime.NewRoleClient("proposal", "proposal", settings, CallHooks{}) if err != nil { t.Fatal(err) } - validation, err := runtime.NewRoleClient("validation", CallHooks{}) + validation, err := runtime.NewRoleClient("validation", "validation", settings, CallHooks{}) if err != nil { t.Fatal(err) } - request := structuredoutput.Request{PromptID: prompts.PromptIDModuleGlossaryProposal, PromptVersion: prompts.VersionV1, Variables: map[string]string{"TranscriptDescriptionBlock": "", "GlossaryJSON": "[]", "SectionJSON": "[]"}, Schema: responseschema.CorrectionSetKey} - firstDone := make(chan error, 1) + request := adapterRequest(responseschema.CorrectionSetKey) + done := make(chan error, 1) go func() { - var output map[string]any - _, callErr := proposal.CompleteStructured(context.Background(), request, &output) - firstDone <- callErr + _, err := proposal.CompleteStructured(context.Background(), request, &map[string]any{}) + done <- err }() select { case <-entered: case <-time.After(2 * time.Second): t.Fatal("first call did not enter provider") } - var output map[string]any - capacityResponse, err := validation.CompleteStructured(context.Background(), request, &output) - if !errors.Is(err, pk.ErrCapacityExceeded) { - t.Fatalf("other role did not share capacity: %v", err) - } - var capacityFailure *structuredoutput.Failure - if !errors.As(err, &capacityFailure) || capacityFailure.Class != structuredoutput.FailureRecoverable || capacityFailure.Reason != outcomes.ReasonCapacityExceeded || !capacityResponse.Execution.Available || capacityResponse.Execution.ProviderAttempts != 0 { - t.Fatalf("capacity outcome: failure=%#v counts=%+v", capacityFailure, capacityResponse.Execution) - } - select { - case <-entered: - t.Fatal("second role reached provider despite full backend") - default: - } - close(release) - if err := <-firstDone; err != nil { - t.Fatal(err) - } - _, err = validation.CompleteStructured(context.Background(), request, &output) - if err != nil { - t.Fatalf("capacity was not released: %v", err) - } - for len(entered) > 0 { - <-entered - } - release = make(chan struct{}) - if err := os.WriteFile(main, []byte(strings.Replace(body, "queue_capacity: 0", "queue_capacity: 1", 1)), 0600); err != nil { - t.Fatal(err) - } - loaded, err = config.Load(ck.File(main), "") - if err != nil { - t.Fatal(err) - } - runtime, err = NewRuntime(context.Background(), loaded.Config, main, server.Client()) - if err != nil { - t.Fatal(err) - } - proposal, err = runtime.NewRoleClient("proposal", CallHooks{}) - if err != nil { - t.Fatal(err) - } - validation, err = runtime.NewRoleClient("validation", CallHooks{}) - if err != nil { - t.Fatal(err) - } - go func() { - var out map[string]any - _, callErr := proposal.CompleteStructured(context.Background(), request, &out) - firstDone <- callErr - }() - select { - case <-entered: - case <-time.After(2 * time.Second): - t.Fatal("first queued-capacity call did not dispatch") - } - queuedCtx, cancelQueued := context.WithTimeout(context.Background(), 30*time.Millisecond) - defer cancelQueued() - _, err = validation.CompleteStructured(queuedCtx, request, &output) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond) + defer cancel() + _, err = validation.CompleteStructured(ctx, request, &map[string]any{}) if !errors.Is(err, context.DeadlineExceeded) { t.Fatalf("queued call did not honor deadline: %v", err) } - if !errors.As(err, &capacityFailure) || capacityFailure.Class != structuredoutput.FailureCanceled || capacityFailure.Reason != outcomes.ReasonCanceled { - t.Fatalf("queued parent deadline classification: %#v", capacityFailure) + select { + case <-entered: + t.Fatal("queued call dispatched") + default: } close(release) - if err := <-firstDone; err != nil { + if err := <-done; err != nil { t.Fatal(err) } - _, err = validation.CompleteStructured(context.Background(), request, &output) - if err != nil { - t.Fatalf("queued cancellation leaked capacity: %v", err) + if _, err := validation.CompleteStructured(context.Background(), request, &map[string]any{}); err != nil { + t.Fatalf("cancellation leaked capacity: %v", err) } } func TestDistinctBackendsHaveIndependentCapacity(t *testing.T) { entered := make(chan struct{}, 2) release := make(chan struct{}) - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + transport := transportFunc(func(r *http.Request) (*http.Response, error) { entered <- struct{}{} <-release - schema := map[string]any{} - _ = json.NewDecoder(r.Body).Decode(&schema) - content, _ := json.Marshal(`{"corrections":[]}`) - _, _ = fmt.Fprintf(w, `{"choices":[{"message":{"content":%s}}]}`, content) - })) - defer server.Close() - dir := t.TempDir() - if err := os.Mkdir(filepath.Join(dir, "profiles"), 0700); err != nil { - t.Fatal(err) - } - main := filepath.Join(dir, "config.yml") - body := fmt.Sprintf("version: 2\nbackends:\n first:\n endpoint: %s/v1\n concurrency: 1\n queue_capacity: 0\n second:\n endpoint: %s/v1\n concurrency: 1\n queue_capacity: 0\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n repair_attempts: 0\n", server.URL, server.URL) - if err := os.WriteFile(main, []byte(body), 0600); err != nil { - t.Fatal(err) - } - for name, backend := range map[string]string{"proposal": "first", "validation": "second"} { - if err := os.WriteFile(filepath.Join(dir, "profiles", name+".yaml"), []byte("profiles:\n "+name+":\n backend: "+backend+"\n model: local\n"), 0600); err != nil { - t.Fatal(err) - } - } - loaded, err := config.Load(ck.File(main), "") + return capacityResponse(r), nil + }) + resources := "backends:\n - id: first\n endpoint: https://example.invalid/v1\n concurrency_limit: 1\n queue_capacity: 0\n - id: second\n endpoint: https://example.invalid/v1\n concurrency_limit: 1\n queue_capacity: 0\nprofiles:\n - id: proposal\n backend: first\n model: local\n - id: validation\n backend: second\n model: local\n" + runtime := capacityFixture(t, resources, transport) + settings := config.LLM{OperationTimeout: 3 * time.Second} + first, err := runtime.NewRoleClient("proposal", "proposal", settings, CallHooks{}) if err != nil { t.Fatal(err) } - runtime, err := NewRuntime(context.Background(), loaded.Config, main, server.Client()) + second, err := runtime.NewRoleClient("validation", "validation", settings, CallHooks{}) if err != nil { t.Fatal(err) } - proposal, err := runtime.NewRoleClient("proposal", CallHooks{}) - if err != nil { - t.Fatal(err) - } - validation, err := runtime.NewRoleClient("validation", CallHooks{}) - if err != nil { - t.Fatal(err) - } - request := structuredoutput.Request{PromptID: prompts.PromptIDModuleGlossaryProposal, PromptVersion: prompts.VersionV1, Variables: map[string]string{"TranscriptDescriptionBlock": "", "GlossaryJSON": "[]", "SectionJSON": "[]"}, Schema: responseschema.CorrectionSetKey} + request := adapterRequest(responseschema.CorrectionSetKey) done := make(chan error, 2) go func() { - var out map[string]any - _, e := proposal.CompleteStructured(context.Background(), request, &out) - done <- e + _, err := first.CompleteStructured(context.Background(), request, &map[string]any{}) + done <- err }() select { case <-entered: @@ -219,13 +124,13 @@ func TestDistinctBackendsHaveIndependentCapacity(t *testing.T) { t.Fatal("first backend did not dispatch") } go func() { - var out map[string]any - _, e := validation.CompleteStructured(context.Background(), request, &out) - done <- e + _, err := second.CompleteStructured(context.Background(), request, &map[string]any{}) + done <- err }() select { case <-entered: case <-time.After(2 * time.Second): + close(release) t.Fatal("second backend was blocked by first") } close(release) diff --git a/internal/framework/llm/adapter_test.go b/internal/framework/llm/adapter_test.go index 37131f3..d9c2372 100644 --- a/internal/framework/llm/adapter_test.go +++ b/internal/framework/llm/adapter_test.go @@ -5,8 +5,8 @@ import ( "encoding/json" "errors" "fmt" + "io" "net/http" - "net/http/httptest" "os" "path/filepath" "strings" @@ -15,13 +15,13 @@ import ( "testing" "time" - ck "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput" "gitea.maximumdirect.net/eric/audita/v3/internal/prompts" - pk "gitea.maximumdirect.net/eric/promptkit" + ck "go.maximumdirect.net/configkit" + pk "go.maximumdirect.net/promptkit" ) type reply struct { @@ -30,85 +30,56 @@ type reply struct { } func adapterFixture(t *testing.T, replies []reply, retries, repairs int, hook func(*http.Request)) (*Runtime, *atomic.Int32, string) { - return adapterFixtureWithDocuments(t, replies, retries, repairs, hook, "offline", map[string]string{ - "definitions.yaml": "profiles:\n base:\n backend: offline\n model: full-model\n", - }) + return adapterFixtureWithPrompt(t, replies, retries, repairs, hook, "") } func adapterFixtureWithPrompt(t *testing.T, replies []reply, retries, repairs int, hook func(*http.Request), operatorPrompt string) (*Runtime, *atomic.Int32, string) { - documents := map[string]string{ - "definitions.yaml": "profiles:\n base:\n backend: offline\n model: full-model\n", - } - if operatorPrompt != "" { - documents["operator-prompt.yaml"] = operatorPrompt - } - return adapterFixtureWithDocuments(t, replies, retries, repairs, hook, "offline", documents) -} - -func adapterFixtureWithDocuments(t *testing.T, replies []reply, retries, repairs int, hook func(*http.Request), backendID string, documents map[string]string) (*Runtime, *atomic.Int32, string) { t.Helper() var count atomic.Int32 - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Setenv("ADAPTER_KEY", "adapter-secret") + dir := t.TempDir() + resources := "backends:\n - id: offline\n endpoint: https://provider.invalid/v1\n api_key_env: ADAPTER_KEY\n concurrency_limit: 1\n queue_capacity: 0\nprofiles:\n - id: base\n backend: offline\n model: full-model\n" + for name, body := range map[string]string{"config.yml": "version: 3\n", "prompts.yml": resources + operatorPrompt} { + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0600); err != nil { + t.Fatal(err) + } + } + loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "") + if err != nil { + t.Fatal(err) + } + transport := transportFunc(func(r *http.Request) (*http.Response, error) { call := int(count.Add(1)) if hook != nil { hook(r) } - step := replies[len(replies)-1] - if call <= len(replies) { - step = replies[call-1] + if err := r.Context().Err(); err != nil { + return nil, err } + step := replies[min(call-1, len(replies)-1)] if step.status == 0 { - connection, _, hijackErr := w.(http.Hijacker).Hijack() - if hijackErr != nil { - t.Error(hijackErr) - return + return nil, errors.New("synthetic transport error") + } + body := `{"error":{"message":"provider error"}}` + if step.status == http.StatusOK { + body = `{"choices":[]}` + if step.content != "MALFORMED_ENVELOPE" { + encoded, _ := json.Marshal(step.content) + body = fmt.Sprintf(`{"model":"returned-model","choices":[{"message":{"content":%s}}],"usage":{"prompt_tokens":2,"completion_tokens":3,"total_tokens":5}}`, encoded) } - _ = connection.Close() - return } - if step.status != http.StatusOK { - w.WriteHeader(step.status) - _, _ = fmt.Fprint(w, `{"error":{"message":"provider error"}}`) - return - } - if step.content == "MALFORMED_ENVELOPE" { - _, _ = fmt.Fprint(w, `{"choices":[]}`) - return - } - encoded, _ := json.Marshal(step.content) - w.Header().Set("Content-Type", "application/json") - _, _ = fmt.Fprintf(w, `{"model":"returned-model","choices":[{"message":{"content":%s}}],"usage":{"prompt_tokens":2,"completion_tokens":3,"total_tokens":5}}`, encoded) - })) - t.Cleanup(server.Close) - t.Setenv("ADAPTER_KEY", "adapter-secret") - dir := t.TempDir() - profileDirectory := filepath.Join(dir, "profiles") - if err := os.Mkdir(profileDirectory, 0700); err != nil { - t.Fatal(err) - } - base := fmt.Sprintf("version: 2\nprofiles_dir: profiles\nbackends:\n %s:\n endpoint: %s/v1\n credential: ADAPTER_KEY\n concurrency: 1\n queue_capacity: 0\nllm:\n proposal_profile: base\n transient_retries: %d\n repair_attempts: %d\n", backendID, server.URL, retries, repairs) - main := filepath.Join(dir, "config.yml") - if err := os.WriteFile(main, []byte(base), 0600); err != nil { - t.Fatal(err) - } - for name, content := range documents { - if err := os.WriteFile(filepath.Join(profileDirectory, name), []byte(content), 0600); err != nil { - t.Fatal(err) - } - } - loaded, err := config.Load(ck.File(main), "") - if err != nil { - t.Fatal(err) - } - runtime, err := NewRuntime(context.Background(), loaded.Config, main, server.Client()) + return &http.Response{StatusCode: step.status, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil + }) + runtime, err := NewRuntime(context.Background(), loaded, &http.Client{Transport: transport}) if err != nil { t.Fatal(err) } + runtime.Config.LLM = config.LLM{OperationTimeout: 10 * time.Second, TransientRetries: retries, RepairAttempts: repairs} return runtime, &count, dir } func operatorPromptDocument(system, user string) string { - return fmt.Sprintf("prompts:\n %s:\n v1:\n messages:\n - role: system\n content: %q\n - role: user\n content: %q\n output:\n format: text\n validation_mode: basic\n", prompts.PromptIDModuleGlossaryProposal, system, user) + return fmt.Sprintf("collections:\n audita:\n prompts:\n - id: modules.glossary.proposal\n messages:\n - role: system\n content: %q\n - role: user\n content: %q\n output:\n format: text\n validation_mode: none\n", system, user) } func adapterRequest(schema responseschema.Key) structuredoutput.Request { @@ -118,7 +89,7 @@ func adapterRequest(schema responseschema.Key) structuredoutput.Request { id = prompts.PromptIDValidatorEditorialReview vars = map[string]string{"TranscriptDescriptionBlock": "", "PayloadJSON": "[]"} } - return structuredoutput.Request{PromptID: id, PromptVersion: prompts.VersionV1, Variables: vars, Schema: schema, Context: structuredoutput.DiagnosticContext{ModuleKey: "glossary"}} + return structuredoutput.Request{PromptID: id, Variables: vars, Schema: schema, Context: structuredoutput.DiagnosticContext{ModuleKey: "glossary"}} } func TestPreparedAdapterRepairsAndClassifies(t *testing.T) { @@ -154,7 +125,7 @@ func TestPreparedAdapterRepairsAndClassifies(t *testing.T) { } var events []pk.ExecutionEvent var records []structuredoutput.CallRecord - client, err := runtime.NewRoleClient(role, CallHooks{OnEvent: func(e pk.ExecutionEvent) { events = append(events, e) }, OnCall: func(r structuredoutput.CallRecord) { records = append(records, r) }}) + client, err := runtime.NewRoleClient(role, "base", runtime.Config.LLM, CallHooks{OnEvent: func(e pk.ExecutionEvent) { events = append(events, e) }, OnCall: func(r structuredoutput.CallRecord) { records = append(records, r) }}) if err != nil { t.Fatal(err) } @@ -234,7 +205,7 @@ func TestCorrectionSchemaReplacementValuesThroughAdapter(t *testing.T) { } payload := `{"corrections":[{` + strings.Join(fields, ",") + `}]}` runtime, count, _ := adapterFixture(t, []reply{{200, payload}}, 0, tc.repairs, nil) - client, err := runtime.NewRoleClient("proposal", CallHooks{}) + client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{}) if err != nil { t.Fatal(err) } @@ -280,7 +251,7 @@ func TestPreparedAdapterMetadataAndTargetGuard(t *testing.T) { t.Error(err) } }) - client, err := runtime.NewRoleClient("proposal", CallHooks{}) + client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{}) if err != nil { t.Fatal(err) } @@ -314,12 +285,7 @@ func TestPreparedAdapterMetadataAndTargetGuard(t *testing.T) { if !found { t.Fatal("hardening policy missing") } - backend := runtime.Config.Backends["offline"] - backend.Endpoint = "http://127.0.0.1:1/v1" - runtime.Config.Backends["offline"] = backend - if _, err := client.CompleteStructured(context.Background(), req, &out); err == nil || !strings.Contains(err.Error(), "target changed") || count.Load() != 1 { - t.Fatalf("changed application backend target dispatched: %v, %d", err, count.Load()) - } + } func TestOperatorPromptOverridesUsePerCallSchemaAndOptionalSharedText(t *testing.T) { @@ -365,7 +331,7 @@ func TestOperatorPromptOverridesUsePerCallSchemaAndOptionalSharedText(t *testing } posted = append(posted, envelope) }, operatorPromptDocument(tc.system, tc.user)) - client, err := runtime.NewRoleClient("proposal", CallHooks{}) + client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{}) if err != nil { t.Fatal(err) } @@ -403,150 +369,9 @@ func TestOperatorPromptOverridesUsePerCallSchemaAndOptionalSharedText(t *testing } } -func TestProfilePresenceBackendAuthorityAndProviderParametersReachWire(t *testing.T) { - t.Setenv("CATALOG_TRAP_KEY", "catalog-secret-must-not-be-used") - cases := []struct { - name string - parentTemperature string - childTemperature string - wantTemperature bool - }{ - {name: "explicit zero", parentTemperature: " temperature: 0.8\n", childTemperature: " temperature: 0\n", wantTemperature: true}, - {name: "omitted", wantTemperature: false}, - {name: "null clears inherited control", parentTemperature: " temperature: 0.8\n", childTemperature: " temperature: null\n", wantTemperature: false}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - profileDocument := "profiles:\n \"parent~/partial\":\n backend: openrouter\n model: parent-model\n" + tc.parentTemperature + - " base:\n extends: \"parent~/partial\"\n model: full-model\n timeout_seconds: 0\n" + tc.childTemperature + - " extra_params:\n routing:\n overridden: profile\n profile_only: retained\n cleared: null\n" - operatorBackend := `backends: - openrouter: - endpoint: https://operator.invalid/v1 - concurrency_limit: 91 - queue_capacity: 92 - api_key_env: CATALOG_TRAP_KEY - extra_params: - routing: - inherited: backend - overridden: backend - cleared: remove-me -` - documents := map[string]string{ - "unrelated-profile-file.yml": profileDocument, - "provider-tuning.yaml": operatorBackend, - } - var wire map[string]json.RawMessage - runtime, count, _ := adapterFixtureWithDocuments(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, func(r *http.Request) { - if got := r.Header.Get("Authorization"); got != "Bearer adapter-secret" { - t.Errorf("application credential was not authoritative: %q", got) - } - if err := json.NewDecoder(r.Body).Decode(&wire); err != nil { - t.Error(err) - } - }, "openrouter", documents) - profile := runtime.Profiles["proposal"] - if profile.EffectiveModelParams.APIKeyEnv != "" || profile.EffectiveModelParams.Endpoint == "https://operator.invalid/v1" || profile.EffectiveModelParams.TimeoutSeconds != 0 { - t.Fatalf("application backend mapping or zero timeout lost: %+v", profile.EffectiveModelParams) - } - client, err := runtime.NewRoleClient("proposal", CallHooks{}) - if err != nil { - t.Fatal(err) - } - var out map[string]any - response, err := client.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &out) - if err != nil { - t.Fatal(err) - } - if count.Load() != 1 { - t.Fatalf("provider calls = %d, want 1", count.Load()) - } - temperature, hasTemperature := wire["temperature"] - if hasTemperature != tc.wantTemperature { - t.Fatalf("temperature presence = %t, want %t: %s", hasTemperature, tc.wantTemperature, wire) - } - if tc.wantTemperature { - var value float64 - if err := json.Unmarshal(temperature, &value); err != nil || value != 0 { - t.Fatalf("explicit zero temperature = %s (%v)", temperature, err) - } - } - var routing map[string]any - if err := json.Unmarshal(wire["routing"], &routing); err != nil { - t.Fatalf("provider routing fields missing: %s (%v)", wire, err) - } - if routing["inherited"] != "backend" || routing["overridden"] != "profile" || routing["profile_only"] != "retained" { - t.Fatalf("recursively overlaid provider fields = %#v", routing) - } - if value, exists := routing["cleared"]; !exists || value != nil { - t.Fatalf("null did not replace inherited provider field: %#v", routing) - } - if response.Model != "full-model" || response.Prepared == nil { - t.Fatalf("execution did not use selected profile: %+v", response) - } - }) - } -} - -func TestPreparedAdapterUsesCapturedProfileAndPromptUntilReconstructed(t *testing.T) { - var posted []struct { - Model string `json:"model"` - Messages []structuredoutput.Message `json:"messages"` - } - runtime, count, dir := adapterFixtureWithPrompt(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, func(r *http.Request) { - var envelope struct { - Model string `json:"model"` - Messages []structuredoutput.Message `json:"messages"` - } - if err := json.NewDecoder(r.Body).Decode(&envelope); err != nil { - t.Error(err) - return - } - posted = append(posted, envelope) - }, operatorPromptDocument("Initial operator instructions.", "Initial payload.")) - client, err := runtime.NewRoleClient("proposal", CallHooks{}) - if err != nil { - t.Fatal(err) - } - profile := "profiles:\n base:\n backend: offline\n model: edited-model\n" - if err := os.WriteFile(filepath.Join(dir, "profiles", "definitions.yaml"), []byte(profile), 0600); err != nil { - t.Fatal(err) - } - updatedPrompt := operatorPromptDocument("Edited operator instructions.", "Edited payload.") - if err := os.WriteFile(filepath.Join(dir, "profiles", "operator-prompt.yaml"), []byte(updatedPrompt), 0600); err != nil { - t.Fatal(err) - } - - var out map[string]any - response, err := client.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &out) - if err != nil { - t.Fatal(err) - } - if response.Model != "full-model" || response.Prepared == nil || response.Prepared.Messages[0].Content != "Initial operator instructions." || posted[0].Model != "full-model" { - t.Fatalf("existing runtime did not retain its captured configuration: response=%+v request=%+v", response, posted[0]) - } - - main := filepath.Join(dir, "config.yml") - newRuntime, err := NewRuntime(context.Background(), runtime.Config, main, http.DefaultClient) - if err != nil { - t.Fatal(err) - } - newClient, err := newRuntime.NewRoleClient("proposal", CallHooks{}) - if err != nil { - t.Fatal(err) - } - response, err = newClient.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &out) - if err != nil { - t.Fatal(err) - } - if int(count.Load()) != 2 || response.Model != "edited-model" || response.Prepared == nil || response.Prepared.Messages[0].Content != "Edited operator instructions." || posted[1].Model != "edited-model" { - t.Fatalf("new runtime did not capture edited configuration: response=%+v request=%+v", response, posted[1]) - } -} - func TestPreparedAdapterRejectsTargetDriftAndPreparationErrors(t *testing.T) { runtime, count, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, nil) - client, err := runtime.NewRoleClient("proposal", CallHooks{}) + client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{}) if err != nil { t.Fatal(err) } @@ -563,7 +388,7 @@ func TestPreparedAdapterCancellationStopsRetryWait(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() var once sync.Once - client, err := runtime.NewRoleClient("proposal", CallHooks{OnEvent: func(e pk.ExecutionEvent) { + client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{OnEvent: func(e pk.ExecutionEvent) { if e.Kind == pk.ExecutionAttempt && e.Dispatched { once.Do(cancel) } @@ -580,11 +405,9 @@ func TestPreparedAdapterCancellationStopsRetryWait(t *testing.T) { func TestPreparedAdapterDistinguishesOperationTimeoutFromParentCancellation(t *testing.T) { blocking := func(*http.Request) { time.Sleep(500 * time.Millisecond) } - runtime, _, _ := adapterFixtureWithDocuments(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, blocking, "offline", map[string]string{ - "definitions.yaml": "profiles:\n base:\n backend: offline\n model: full-model\n timeout_seconds: 0\n", - }) + runtime, _, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, blocking) runtime.Config.LLM.OperationTimeout = 200 * time.Millisecond - client, err := runtime.NewRoleClient("proposal", CallHooks{}) + client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{}) if err != nil { t.Fatal(err) } @@ -605,7 +428,7 @@ func TestPreparedAdapterDistinguishesOperationTimeoutFromParentCancellation(t *t func TestPreparedAdapterClassifiesSetupAndDomainDecodeFailures(t *testing.T) { runtime, _, _ := adapterFixture(t, []reply{{200, `{"validations":[{"correction_index":9999999999999999999999999,"approved":true,"confidence":0.8,"reason":"ok"}]}`}}, 0, 0, nil) - client, err := runtime.NewRoleClient("validation", CallHooks{}) + client, err := runtime.NewRoleClient("validation", "base", runtime.Config.LLM, CallHooks{}) if err != nil { t.Fatal(err) } @@ -638,7 +461,7 @@ func TestPreparedAdapterClassifiesSetupAndDomainDecodeFailures(t *testing.T) { func TestPreparedAdapterNonRetryableProviderStatus(t *testing.T) { runtime, count, _ := adapterFixture(t, []reply{{http.StatusBadRequest, ""}}, 2, 0, nil) - client, err := runtime.NewRoleClient("proposal", CallHooks{}) + client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{}) if err != nil { t.Fatal(err) } diff --git a/internal/framework/llm/observations_test.go b/internal/framework/llm/observations_test.go index 17f9c1b..8668703 100644 --- a/internal/framework/llm/observations_test.go +++ b/internal/framework/llm/observations_test.go @@ -10,7 +10,7 @@ import ( "gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput" - pk "gitea.maximumdirect.net/eric/promptkit" + pk "go.maximumdirect.net/promptkit" ) func TestObservationSummarySeparatesLogicalCallsAttemptsAndUsage(t *testing.T) { @@ -93,7 +93,7 @@ func TestObservationCollectorConcurrentAttributionAndSnapshotCopy(t *testing.T) func TestRoleClientFeedsCollectorAcrossPreparationFailure(t *testing.T) { runtime, count, _ := adapterFixture(t, []reply{{status: 200, content: `{"corrections":[]}`}}, 0, 0, nil) collector := NewObservationCollector() - client, err := runtime.NewRoleClient("proposal", collector.Hooks()) + client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, collector.Hooks()) if err != nil { t.Fatal(err) } diff --git a/internal/framework/llm/runtime_assembly_test.go b/internal/framework/llm/runtime_assembly_test.go index 4ec9b8f..73030b1 100644 --- a/internal/framework/llm/runtime_assembly_test.go +++ b/internal/framework/llm/runtime_assembly_test.go @@ -4,43 +4,17 @@ import ( "context" "errors" "io/fs" - "net/http" "os" "path" "path/filepath" - "strings" "testing" "testing/fstest" - ck "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" "gitea.maximumdirect.net/eric/audita/v3/internal/prompts" + ck "go.maximumdirect.net/configkit" ) -func runtimeAssemblyFixture(t *testing.T, selections string, documents map[string]string) (config.Config, string) { - t.Helper() - dir := t.TempDir() - profileDir := filepath.Join(dir, "profiles") - if err := os.Mkdir(profileDir, 0700); err != nil { - t.Fatal(err) - } - main := filepath.Join(dir, "config.yml") - text := "version: 2\nprofiles_dir: profiles\nbackends:\n offline:\n endpoint: http://127.0.0.1:1/v1\n concurrency: 1\n queue_capacity: 0\nllm:\n" + selections - if err := os.WriteFile(main, []byte(text), 0600); err != nil { - t.Fatal(err) - } - for name, content := range documents { - if err := os.WriteFile(filepath.Join(profileDir, name), []byte(content), 0600); err != nil { - t.Fatal(err) - } - } - loaded, err := config.Load(ck.File(main), "") - if err != nil { - t.Fatal(err) - } - return loaded.Config, main -} - func copyEmbeddedPromptFilesystem(t *testing.T) fstest.MapFS { t.Helper() source := prompts.Filesystem() @@ -62,166 +36,18 @@ func copyEmbeddedPromptFilesystem(t *testing.T) fstest.MapFS { return files } -func TestNewRuntimeRequiresCompleteSelectedProfiles(t *testing.T) { - validProposal := " backend: offline\n model: proposal-model\n" - cases := []struct { - name string - selections string - profiles string - role string - field string - }{ - { - name: "omitted proposal model with derived validation selection", - selections: " proposal_profile: proposal\n", - profiles: "profiles:\n proposal:\n backend: offline\n", - role: "proposal", - field: "model", - }, - { - name: "empty validation model", - selections: " proposal_profile: proposal\n validation_profile: validation\n", - profiles: "profiles:\n proposal:\n" + validProposal + " validation:\n backend: offline\n model: \"\"\n", - role: "validation", - field: "model", - }, - { - name: "null proposal model", - selections: " proposal_profile: proposal\n", - profiles: "profiles:\n proposal:\n backend: offline\n model: null\n", - role: "proposal", - field: "model", - }, - { - name: "cleared validation endpoint", - selections: " proposal_profile: proposal\n validation_profile: validation\n", - profiles: "profiles:\n proposal:\n" + validProposal + " validation:\n backend: offline\n endpoint: null\n model: validation-model\n", - role: "validation", - field: "endpoint", - }, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - cfg, main := runtimeAssemblyFixture(t, tc.selections, map[string]string{"definitions.yml": tc.profiles}) - _, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}}) - if err == nil || !strings.Contains(err.Error(), tc.role) || !strings.Contains(err.Error(), tc.field) { - t.Fatalf("selected incomplete profile was not rejected for %s.%s: %v", tc.role, tc.field, err) - } - }) - } -} - -func TestNewRuntimeAllowsUnselectedPartialProfiles(t *testing.T) { - cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{ - "definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n unfinished:\n backend: offline\n", - }) - runtime, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}}) - if err != nil { - t.Fatal(err) - } - if runtime.Profiles["proposal"].EffectiveModelParams.Model != "proposal-model" || runtime.Profiles["validation"].EffectiveModelParams.Model != "proposal-model" { - t.Fatalf("role profiles were not resolved: %+v", runtime.Profiles) - } -} - -func TestNativeProfileInheritancePreservesControlsAndMergesProviderValues(t *testing.T) { - cfg, main := runtimeAssemblyFixture(t, " proposal_profile: \"child profile\"\n", map[string]string{ - "unrelated-name.yaml": `profiles: - "parent~/shared": - backend: offline - temperature: 0.4 - top_p: 0.7 - extra_params: - routing: - retained: from-parent - overridden: parent - cleared: remove-me - "child profile": - extends: "parent~/shared" - model: child-model - extra_params: - routing: - overridden: child - child_only: present - cleared: null -`, - }) - runtime, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{}}) - if err != nil { - t.Fatal(err) - } - profile := runtime.Profiles["proposal"] - target := profile.EffectiveModelParams - if profile.ProfileID != "child profile" || target.Model != "child-model" || target.Temperature != 0.4 || target.TopP != 0.7 { - t.Fatalf("inherited profile target = %+v", target) - } - routing, ok := target.ExtraParams["routing"].(map[string]any) - if !ok || routing["retained"] != "from-parent" || routing["overridden"] != "child" || routing["child_only"] != "present" { - t.Fatalf("recursively inherited provider values = %#v", target.ExtraParams) - } - if value, exists := routing["cleared"]; !exists || value != nil { - t.Fatalf("null did not replace the inherited provider value: %#v", routing) - } -} - -func TestMissingSelectedProfileDirectoryIsOperatorInputError(t *testing.T) { - cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{ - "definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n", - }) - if err := os.RemoveAll(filepath.Join(filepath.Dir(main), cfg.ProfilesDir)); err != nil { - t.Fatal(err) - } - _, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}}) - if err == nil || IsAssetError(err) { - t.Fatalf("missing operator profile directory classification: %v", err) - } -} - -func TestNewRuntimeRejectsMalformedUnselectedProfilesAndConflicts(t *testing.T) { - cases := []struct { - name string - document map[string]string - selection string - }{ - { - name: "malformed unselected definition", - document: map[string]string{"definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n unused:\n temperature: 9\n"}, - selection: " proposal_profile: proposal\n", - }, - { - name: "undefined selected backend", - document: map[string]string{"definitions.yml": "profiles:\n proposal:\n backend: missing\n model: proposal-model\n"}, - selection: " proposal_profile: proposal\n", - }, - { - name: "conflicting directory contributions", - document: map[string]string{ - "a.yml": "profiles:\n proposal:\n backend: offline\n model: first-model\n", - "b.yml": "profiles:\n proposal:\n model: second-model\n", - }, - selection: " proposal_profile: proposal\n", - }, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - cfg, main := runtimeAssemblyFixture(t, tc.selection, tc.document) - _, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}}) - if err == nil || IsAssetError(err) { - t.Fatalf("operator definition classification: %v", err) - } - }) - } -} - func TestRuntimeClassifiesEmbeddedAssetsAndHonorsCancellation(t *testing.T) { - _, err := newRuntimeWithPromptFS(context.Background(), config.Config{}, "config.yml", nil, fstest.MapFS{}) + loaded, err := config.Load(ck.Input{Source: ck.NoSource()}, "") + if err != nil { + t.Fatal(err) + } + _, err = newRuntimeWithPromptFS(context.Background(), loaded, nil, fstest.MapFS{}) if err == nil || !IsAssetError(err) || !errors.Is(err, fs.ErrNotExist) { t.Fatalf("embedded discovery error was not preserved as an asset error: %v", err) } - ctx, cancel := context.WithCancel(context.Background()) cancel() - _, err = newRuntimeWithPromptFS(ctx, config.Config{}, "config.yml", nil, fstest.MapFS{}) + _, err = newRuntimeWithPromptFS(ctx, loaded, nil, fstest.MapFS{}) if !errors.Is(err, context.Canceled) || IsAssetError(err) { t.Fatalf("caller cancellation classification: %v", err) } @@ -229,12 +55,13 @@ func TestRuntimeClassifiesEmbeddedAssetsAndHonorsCancellation(t *testing.T) { func TestBrokenEmbeddedPromptFailsEmbeddedOnlyConstruction(t *testing.T) { promptFS := copyEmbeddedPromptFilesystem(t) - assetPath := path.Join(prompts.AssetRoot, prompts.PromptIDModuleGrammarProposal+".yaml") - promptFS[assetPath] = &fstest.MapFile{Data: []byte("prompts:\n modules.grammar.proposal:\n v1:\n messages: []\n output:\n format: json\n validation_mode: json_schema\n schema_path: correction_set.v1.json\n")} - cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{ - "definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n", - }) - _, err := newRuntimeWithPromptFS(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}}, promptFS) + assetPath := path.Join(prompts.AssetRoot, "modules.grammar.proposal.yaml") + promptFS[assetPath] = &fstest.MapFile{Data: []byte("collections:\n audita:\n prompts:\n - id: modules.grammar.proposal\n messages: []\n")} + loaded, err := config.Load(ck.Input{Source: ck.NoSource()}, "") + if err != nil { + t.Fatal(err) + } + _, err = newRuntimeWithPromptFS(context.Background(), loaded, nil, promptFS) if err == nil || !IsAssetError(err) { t.Fatalf("broken embedded prompt was not classified as an asset error: %v", err) } @@ -242,14 +69,24 @@ func TestBrokenEmbeddedPromptFailsEmbeddedOnlyConstruction(t *testing.T) { func TestOperatorPromptCanRepairInvalidEmbeddedPrompt(t *testing.T) { promptFS := copyEmbeddedPromptFilesystem(t) - assetPath := path.Join(prompts.AssetRoot, prompts.PromptIDModuleGrammarProposal+".yaml") - promptFS[assetPath] = &fstest.MapFile{Data: []byte("prompts:\n modules.grammar.proposal:\n v1:\n messages: []\n output:\n format: json\n validation_mode: json_schema\n schema_path: correction_set.v1.json\n")} - cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{ - "operator.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\nprompts:\n modules.grammar.proposal:\n v1:\n messages:\n - role: system\n content: operator system\n - role: user\n content: operator user\n", - }) - runtime, err := newRuntimeWithPromptFS(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}}, promptFS) + assetPath := path.Join(prompts.AssetRoot, "modules.grammar.proposal.yaml") + promptFS[assetPath] = &fstest.MapFile{Data: []byte("collections:\n audita:\n prompts:\n - id: modules.grammar.proposal\n messages: []\n")} + dir := t.TempDir() + for name, content := range map[string]string{ + "config.yml": "version: 3\n", + "prompts.yml": "collections:\n audita:\n prompts:\n - id: modules.grammar.proposal\n messages:\n - role: user\n content: operator message\n output:\n format: text\n validation_mode: none\n", + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0600); err != nil { + t.Fatal(err) + } + } + loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "") if err != nil { - t.Fatalf("valid final configuration was rejected with an invalid displaced embedded prompt: %v", err) + t.Fatal(err) + } + runtime, err := newRuntimeWithPromptFS(context.Background(), loaded, nil, promptFS) + if err != nil { + t.Fatalf("valid final configuration rejected: %v", err) } if runtime.Engine == nil { t.Fatal("runtime engine was not retained") diff --git a/internal/framework/llm/runtime_inspection_test.go b/internal/framework/llm/runtime_inspection_test.go index 26baa30..870c954 100644 --- a/internal/framework/llm/runtime_inspection_test.go +++ b/internal/framework/llm/runtime_inspection_test.go @@ -9,258 +9,60 @@ import ( "strings" "testing" - ck "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" - "gopkg.in/yaml.v3" + ck "go.maximumdirect.net/configkit" ) type rejectingTransport struct{ t *testing.T } func (r rejectingTransport) RoundTrip(*http.Request) (*http.Response, error) { - r.t.Error("profile inspection contacted a provider") + if r.t != nil { + r.t.Error("resource inspection contacted a provider") + } return nil, errors.New("provider access forbidden") } -func profileLayout(t *testing.T, base, overlay string) string { - t.Helper() - dir := t.TempDir() - for _, name := range []string{"profiles", "overlays"} { - if err := os.Mkdir(filepath.Join(dir, name), 0700); err != nil { - t.Fatal(err) - } - } - for name, content := range map[string]string{ - "base": "profiles:\n base:\n backend: offline\n model: full-model\n temperature: 0.3\n max_tokens: 500\n top_p: 0.8\n timeout_seconds: 12\n service_tier: priority\n reasoning_effort: medium\n", - "testing": "profiles:\n testing:\n backend: offline\n model: economy-model\n", - } { - if err := os.WriteFile(filepath.Join(dir, "profiles", name+".yaml"), []byte(content), 0600); err != nil { - t.Fatal(err) - } - } - if err := os.WriteFile(filepath.Join(dir, "config.yml"), []byte(base), 0600); err != nil { - t.Fatal(err) - } - if overlay != "" { - if err := os.WriteFile(filepath.Join(dir, "overlays", "testing.yml"), []byte(overlay), 0600); err != nil { - t.Fatal(err) - } - } - return dir -} - -const profileBase = "version: 2\nbackends:\n offline:\n endpoint: https://offline.invalid/v1/\n credential: SYNTHETIC_KEY\nllm:\n proposal_profile: base\n" - -func TestOfflineProfileAssemblyAndSafeViews(t *testing.T) { - t.Setenv("SYNTHETIC_KEY", "synthetic-value-never-print") - dir := profileLayout(t, profileBase, "llm:\n proposal_profile: testing\n") - client := &http.Client{Transport: rejectingTransport{t}} - for _, tc := range []struct { - name, overlay string - overrides []ck.Override - want string - }{ - {"base", "", nil, "full-model"}, - {"overlay", "testing", nil, "economy-model"}, - {"explicit override", "testing", []ck.Override{{Path: ck.Keys("llm", "proposal_profile"), Value: "base", Flag: "--proposal-profile"}}, "full-model"}, - } { - t.Run(tc.name, func(t *testing.T) { - main := filepath.Join(dir, "config.yml") - loaded, err := config.Load(ck.File(main), tc.overlay, tc.overrides...) - if err != nil { - t.Fatal(err) - } - runtime, err := NewRuntime(context.Background(), loaded.Config, main, client) - if err != nil { - t.Fatal(err) - } - if runtime.Engine == nil || runtime.Profiles["proposal"].EffectiveModelParams.Model != tc.want || runtime.Profiles["validation"].EffectiveModelParams.Model != tc.want { - t.Fatalf("roles not resolved: %+v", runtime.Profiles) - } - profiles, err := runtime.SelectedProfilesYAML() - if err != nil { - t.Fatal(err) - } - explain, err := ExplanationYAML(loaded) - if err != nil { - t.Fatal(err) - } - combined := string(profiles) + string(explain) - if !strings.Contains(combined, tc.want) || !strings.Contains(combined, "/llm/validation_profile") { - t.Fatalf("inspection missing selection/provenance: %s", combined) - } - if strings.Contains(combined, "synthetic-value-never-print") || strings.Contains(combined, "https://offline.invalid") { - t.Fatal("safe views exposed credentials or endpoint") - } - if tc.name == "overlay" && !strings.Contains(combined, "overlay") { - t.Fatal("overlay origin missing") - } - if tc.name == "explicit override" && !strings.Contains(combined, "cli") { - t.Fatal("CLI origin missing") - } - if !strings.Contains(string(explain), "validation uses the final proposal profile when omitted") { - t.Fatal("inherited validation derivation missing") - } - if tc.name == "overlay" && !strings.Contains(string(explain), "source: overlays/testing.yml\n layer: overlay") { - t.Fatal("overlay field source missing") - } - if tc.name == "explicit override" && !strings.Contains(string(explain), "source: --proposal-profile\n layer: cli") { - t.Fatal("explicit profile source missing") - } - if tc.name == "base" && (!strings.Contains(string(profiles), "temperature: 0.3") || !strings.Contains(string(profiles), "reasoning_effort: medium")) { - t.Fatal("selected controls missing") - } - }) - } -} - -func TestSafeViewsOmitProviderAndPromptValues(t *testing.T) { - const providerValue = "private-provider-value" - const promptValue = "private-operator-prompt-text" - runtime, _, dir := adapterFixtureWithDocuments(t, nil, 0, 0, nil, "offline", map[string]string{ - "definitions.yaml": "profiles:\n base:\n backend: offline\n model: full-model\n extra_params:\n vendor_extension: " + providerValue + "\n", - "operator-prompt.yaml": operatorPromptDocument(promptValue, promptValue), - }) - profiles, err := runtime.SelectedProfilesYAML() - if err != nil { - t.Fatal(err) - } - loaded, err := config.Load(ck.File(filepath.Join(dir, "config.yml")), "") - if err != nil { - t.Fatal(err) - } - explain, err := ExplanationYAML(loaded) - if err != nil { - t.Fatal(err) - } - combined := string(profiles) + string(explain) - for _, value := range []string{providerValue, promptValue, "adapter-secret", "127.0.0.1"} { - if strings.Contains(combined, value) { - t.Fatalf("safe views exposed %q: %s", value, combined) - } - } -} - func TestExplanationYAMLIncludesSafeInheritanceEvidence(t *testing.T) { - t.Setenv("SYNTHETIC_KEY", "synthetic-inherited-secret-never-print") - text := `version: 2 -backends: - shared: - endpoint: https://shared.invalid/v1 - concurrency: 3 - queue_capacity: 0 - credential: SYNTHETIC_KEY - offline: - extends: shared - concurrency: 1 -llm: - proposal_profile: base + data := `version: 3 +glossary_collections: + common: + entries: + - id: a + name: secret-value-not-for-explanation + category: character + derived: + extends: common ` - loaded, err := config.Load(ck.Bytes("config.yml", []byte(text)), "") - if err != nil { + file := filepath.Join(t.TempDir(), "config.yml") + if err := os.WriteFile(file, []byte(data), 0600); err != nil { t.Fatal(err) } + loaded, err := config.Load(ck.Input{Source: ck.File(file)}, "") + if err != nil { + t.Fatalf("%v: %+v", err, err.(*ck.LoadError).Diagnostics()) + } explanation, err := ExplanationYAML(loaded) if err != nil { t.Fatal(err) } - if strings.Contains(string(explanation), "synthetic-inherited-secret-never-print") { - t.Fatal("explanation exposed a resolved inherited credential") + text := string(explanation) + if strings.Contains(text, "secret-value-not-for-explanation") { + t.Fatal("explanation exposed an effective value") } - var records []struct { - Path string `yaml:"path"` - Inheritance []struct { - From string `yaml:"from"` - To string `yaml:"to"` - Reference string `yaml:"reference"` - } `yaml:"inheritance"` - } - if err := yaml.Unmarshal(explanation, &records); err != nil { - t.Fatal(err) - } - var endpoint, credential, concurrency *struct { - Path string `yaml:"path"` - Inheritance []struct { - From string `yaml:"from"` - To string `yaml:"to"` - Reference string `yaml:"reference"` - } `yaml:"inheritance"` - } - for i := range records { - switch records[i].Path { - case "/backends/offline/endpoint": - endpoint = &records[i] - case "/backends/offline/credential": - credential = &records[i] - case "/backends/offline/concurrency": - concurrency = &records[i] - } - } - if endpoint == nil || len(endpoint.Inheritance) != 1 { - t.Fatalf("inherited endpoint evidence missing: %s", explanation) - } - step := endpoint.Inheritance[0] - if step.From != "/backends/shared/endpoint" || step.To != "/backends/offline/endpoint" || step.Reference != "/backends/offline/extends" { - t.Fatalf("inherited endpoint evidence: %+v", step) - } - if credential == nil || len(credential.Inheritance) != 1 { - t.Fatalf("inherited credential evidence missing: %s", explanation) - } - if concurrency == nil || len(concurrency.Inheritance) != 0 { - t.Fatalf("direct child override reported parent inheritance: %+v", concurrency) + if !strings.Contains(text, "/glossary_collections/derived/entries") || !strings.Contains(text, "inheritance:") || !strings.Contains(text, "/glossary_collections/common/entries") { + t.Fatalf("inherited field evidence missing: %s", text) } } -func TestProfileAssemblyRejectsUnsafeSelections(t *testing.T) { - t.Setenv("SYNTHETIC_KEY", "synthetic-value-never-print") - dir := profileLayout(t, profileBase, "") - main := filepath.Join(dir, "config.yml") - for _, tc := range []struct{ name, profile string }{ - {"endpoint only", "profiles:\n custom:\n endpoint: https://offline.invalid/v1\n model: model\n"}, - {"mismatched endpoint", "profiles:\n custom:\n backend: offline\n endpoint: https://elsewhere.invalid/v1\n model: model\n"}, - {"profile credential", "profiles:\n custom:\n backend: offline\n model: model\n api_key_env: OTHER_KEY\n"}, - } { - t.Run(tc.name, func(t *testing.T) { - if err := os.WriteFile(filepath.Join(dir, "profiles", "custom.yaml"), []byte(tc.profile), 0600); err != nil { - t.Fatal(err) - } - loaded, err := config.Load(ck.File(main), "", ck.Override{Path: ck.Keys("llm", "proposal_profile"), Value: "custom", Flag: "--proposal-profile"}) - if err != nil { - t.Fatal(err) - } - if _, err := NewRuntime(context.Background(), loaded.Config, main, &http.Client{Transport: rejectingTransport{t}}); err == nil { - t.Fatal("unsafe profile accepted") - } - }) - } - loaded, err := config.Load(ck.File(main), "", ck.Override{Path: ck.Keys("llm", "proposal_profile"), Value: "missing", Flag: "--proposal-profile"}) +func TestRuntimeInspectionDoesNotRequireCredentialsOrProvider(t *testing.T) { + loaded, err := config.Load(ck.Input{Source: ck.NoSource()}, "") if err != nil { t.Fatal(err) } - if _, err := NewRuntime(context.Background(), loaded.Config, main, nil); err == nil { - t.Fatal("missing profile accepted") + if _, err := NewRuntime(context.Background(), loaded, &http.Client{Transport: rejectingTransport{t}}); err != nil { + t.Fatal(err) } - if got := RedactError(errors.New("provider: synthetic-value-never-print"), loaded.Config); strings.Contains(got, "synthetic-value-never-print") { - t.Fatalf("secret leaked: %q", got) - } -} - -func TestAbsoluteProfileDirectoryAndMissingSecret(t *testing.T) { - t.Setenv("SYNTHETIC_KEY", "synthetic-value-never-print") - dir := profileLayout(t, profileBase, "") - main := filepath.Join(dir, "config.yml") - absolute := strings.Replace(profileBase, "version: 2\n", "version: 2\nprofiles_dir: "+filepath.Join(dir, "profiles")+"\n", 1) - if err := os.WriteFile(main, []byte(absolute), 0600); err != nil { - t.Fatal(err) - } - loaded, err := config.Load(ck.File(main), "") - if err != nil { - t.Fatal(err) - } - if _, err := NewRuntime(context.Background(), loaded.Config, main, &http.Client{Transport: rejectingTransport{t}}); err != nil { - t.Fatal(err) - } - t.Setenv("SYNTHETIC_KEY", "") - if _, err := config.Load(ck.File(main), ""); err == nil { - t.Fatal("missing surviving secret accepted") + if _, err := ExplanationYAML(nil); err == nil { + t.Fatal("nil configuration accepted") } } diff --git a/internal/framework/llm/secrets_test.go b/internal/framework/llm/secrets_test.go index 392a418..fb8e5f3 100644 --- a/internal/framework/llm/secrets_test.go +++ b/internal/framework/llm/secrets_test.go @@ -1,31 +1,45 @@ package llm import ( + "context" + "os" + "path/filepath" "testing" - ck "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" + ck "go.maximumdirect.net/configkit" ) func TestConfiguredSecretsIncludesUnselectedBackends(t *testing.T) { - if ConfiguredSecrets(nil) != nil { - t.Fatal("nil configuration has secrets") + var empty *Runtime + if empty.ConfiguredSecrets(context.Background()) != nil { + t.Fatal("nil runtime has secrets") } t.Setenv("SELECTED_KEY", "selected-secret") t.Setenv("OTHER_KEY", "unselected-secret") - loaded, err := config.Load(ck.Bytes("config.yml", []byte("version: 2\nbackends:\n selected:\n endpoint: https://selected.invalid/v1\n credential: SELECTED_KEY\n other:\n endpoint: https://other.invalid/v1\n credential: OTHER_KEY\nllm:\n proposal_profile: base\n")), "") + t.Setenv("UNRELATED_KEY", "unrelated-secret") + dir := t.TempDir() + for name, data := range map[string]string{ + "config.yml": "version: 3\n", + "prompts.yml": "backends:\n - id: selected\n endpoint: https://selected.invalid/v1\n api_key_env: SELECTED_KEY\n - id: other\n endpoint: https://other.invalid/v1\n api_key_env: OTHER_KEY\nprofiles:\n - id: base\n backend: selected\n model: example\n", + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(data), 0600); err != nil { + t.Fatal(err) + } + } + loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "") if err != nil { t.Fatal(err) } - got := ConfiguredSecrets(&loaded.Config) - if len(got) != 2 { - t.Fatalf("secrets count = %d", len(got)) + runtime, err := NewRuntime(context.Background(), loaded, nil) + if err != nil { + t.Fatal(err) } seen := map[string]bool{} - for _, value := range got { + for _, value := range runtime.ConfiguredSecrets(context.Background()) { seen[value] = true } - if !seen["selected-secret"] || !seen["unselected-secret"] { - t.Fatal("missing resolved backend secret") + if len(seen) != 2 || !seen["selected-secret"] || !seen["unselected-secret"] { + t.Fatalf("unexpected resolved backend secrets: %v", seen) } } diff --git a/internal/framework/llm/source_paths_test.go b/internal/framework/llm/source_paths_test.go index f206a60..9065544 100644 --- a/internal/framework/llm/source_paths_test.go +++ b/internal/framework/llm/source_paths_test.go @@ -2,58 +2,42 @@ package llm import ( "context" - "net/http" "os" "path/filepath" "reflect" "sort" "testing" + + "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" + ck "go.maximumdirect.net/configkit" ) func TestRuntimeTracksCapturedConfigurationAndInheritedMessageFiles(t *testing.T) { - cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n validation_profile: validation\n", map[string]string{ - "selected.yml": "profiles:\n proposal:\n backend: offline\n model: proposal\n validation:\n backend: offline\n model: validation\n", - }) - profiles := filepath.Join(filepath.Dir(main), cfg.ProfilesDir) - parentDir := filepath.Join(profiles, "catalog") - childDir := filepath.Join(profiles, "nested", "child") - for _, directory := range []string{parentDir, childDir} { - if err := os.MkdirAll(directory, 0700); err != nil { - t.Fatal(err) - } - } - parent := filepath.Join(parentDir, "arbitrary-name.yaml") - child := filepath.Join(childDir, "override.yaml") - resource := filepath.Join(parentDir, "instruction.md") - parentDocument := "prompts:\n source:\n release/1~draft:\n messages:\n - role: user\n content_file: ./instruction.md\n output:\n format: text\n validation_mode: basic\n" - childDocument := "prompts:\n literal/~child:\n extends: source\n" - for path, content := range map[string]string{ - parent: parentDocument, - child: childDocument, - resource: "Captured operator message.\n", + dir := t.TempDir() + for name, content := range map[string]string{ + "config.yml": "version: 3\n", + "prompts.yml": "config:\n imports:\n - path: catalog/arbitrary-name.yaml\n required: true\n", + "catalog/arbitrary-name.yaml": "prompts:\n - id: source\n messages:\n - role: user\n content_file: instruction.md\n output:\n format: text\n validation_mode: none\n", + "catalog/instruction.md": "Captured operator message.\n", + "prompts/nested/override.yaml": "prompts:\n - id: literal-child\n extends: {id: source}\n", } { - if err := os.WriteFile(path, []byte(content), 0600); err != nil { + file := filepath.Join(dir, name) + if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(file, []byte(content), 0600); err != nil { t.Fatal(err) } } - - runtime, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{}}) + loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "") if err != nil { t.Fatal(err) } - want := []string{ - filepath.Join(profiles, "nested", "child", "override.yaml"), - filepath.Join(profiles, "catalog", "arbitrary-name.yaml"), - filepath.Join(profiles, "selected.yml"), - resource, - } - // Runtime paths are canonical paths from PromptKit's captured sources. - for i := range want { - want[i], err = filepath.EvalSymlinks(want[i]) - if err != nil { - t.Fatal(err) - } + runtime, err := NewRuntime(context.Background(), loaded, nil) + if err != nil { + t.Fatal(err) } + want := []string{filepath.Join(dir, "prompts.yml"), filepath.Join(dir, "catalog/arbitrary-name.yaml"), filepath.Join(dir, "catalog/instruction.md"), filepath.Join(dir, "prompts/nested/override.yaml")} sort.Strings(want) if !reflect.DeepEqual(runtime.SourcePaths, want) { t.Fatalf("captured source paths = %#v, want %#v", runtime.SourcePaths, want) diff --git a/internal/framework/modules/registry_test.go b/internal/framework/modules/registry_test.go index aaf4d8d..47f340c 100644 --- a/internal/framework/modules/registry_test.go +++ b/internal/framework/modules/registry_test.go @@ -6,6 +6,7 @@ import ( "strings" "testing" + "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" "gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" @@ -41,17 +42,14 @@ func TestUnknownModuleKeyNotRecognized(t *testing.T) { } func TestRepeatedRunSpecNamingRemainsDeterministic(t *testing.T) { - specs, err := contracts.ResolveModuleRunSpecs([]string{"glossary", "glossary", "grammar"}) - if err != nil { - t.Fatalf("ResolveModuleRunSpecs error: %v", err) - } - if specs[0].InstanceName != "glossary_1" || specs[1].InstanceName != "glossary_2" || specs[2].InstanceName != "grammar" { + specs := contracts.ResolveModuleRunSpecs(&config.Config{Pipeline: config.Pipeline{Stages: []config.Stage{{Module: "glossary"}, {Module: "glossary"}, {Module: "grammar"}}}}) + if specs[0].InstanceName != "glossary_1" || specs[1].InstanceName != "glossary_2" || specs[2].InstanceName != "grammar_3" { t.Fatalf("unexpected instance names: %+v", specs) } } func TestUnsupportedUnknownModuleKeyFailsCleanly(t *testing.T) { - factory := NewFactory(Dependencies{}) + factory := NewFactory() _, err := factory.ModuleForSpec(contracts.ModuleRunSpec{ModuleKey: "unknown", InstanceName: "unknown"}) if err == nil { t.Fatal("expected unsupported-module error") @@ -67,7 +65,7 @@ func TestUnsupportedUnknownModuleKeyFailsCleanly(t *testing.T) { } func TestRecognizedButMissingConstructorFailsWithInternalRegistryError(t *testing.T) { - factory := NewFactory(Dependencies{}) + factory := NewFactory() // Force a missing constructor state for a known key. factory.constructors[ModuleKeySpokenWord] = nil @@ -81,7 +79,7 @@ func TestRecognizedButMissingConstructorFailsWithInternalRegistryError(t *testin } func TestGrammarIsRegisteredAndConstructibleByDefault(t *testing.T) { - factory := NewFactory(Dependencies{}) + factory := NewFactory() module, err := factory.ModuleForSpec(contracts.ModuleRunSpec{ModuleKey: ModuleKeyGrammar, InstanceName: ModuleKeyGrammar}) if err != nil { t.Fatalf("ModuleForSpec error: %v", err) @@ -92,7 +90,7 @@ func TestGrammarIsRegisteredAndConstructibleByDefault(t *testing.T) { } func TestGlossaryIsRegisteredAndConstructibleByDefault(t *testing.T) { - factory := NewFactory(Dependencies{}) + factory := NewFactory() module, err := factory.ModuleForSpec(contracts.ModuleRunSpec{ModuleKey: ModuleKeyGlossary, InstanceName: ModuleKeyGlossary}) if err != nil { t.Fatalf("ModuleForSpec error: %v", err) @@ -103,7 +101,7 @@ func TestGlossaryIsRegisteredAndConstructibleByDefault(t *testing.T) { } func TestHomophonesIsRegisteredAndConstructibleByDefault(t *testing.T) { - factory := NewFactory(Dependencies{}) + factory := NewFactory() module, err := factory.ModuleForSpec(contracts.ModuleRunSpec{ModuleKey: ModuleKeyHomophones, InstanceName: ModuleKeyHomophones}) if err != nil { t.Fatalf("ModuleForSpec error: %v", err) @@ -114,7 +112,7 @@ func TestHomophonesIsRegisteredAndConstructibleByDefault(t *testing.T) { } func TestSpokenWordIsRegisteredAndConstructibleByDefault(t *testing.T) { - factory := NewFactory(Dependencies{}) + factory := NewFactory() module, err := factory.ModuleForSpec(contracts.ModuleRunSpec{ModuleKey: ModuleKeySpokenWord, InstanceName: ModuleKeySpokenWord}) if err != nil { t.Fatalf("ModuleForSpec error: %v", err) @@ -125,7 +123,7 @@ func TestSpokenWordIsRegisteredAndConstructibleByDefault(t *testing.T) { } func TestRegisterConstructorAndConstruct(t *testing.T) { - factory := NewFactory(Dependencies{}) + factory := NewFactory() if err := factory.RegisterConstructor(ModuleKeyGlossary, func(ctx context.Context, req ConstructRequest) (contracts.TranscriptModule, error) { _ = ctx if req.RunSpec.InstanceName != "glossary_1" { @@ -149,7 +147,7 @@ func TestRegisterConstructorAndConstruct(t *testing.T) { } func TestRegisterConstructorRejectsUnknownModuleKey(t *testing.T) { - factory := NewFactory(Dependencies{}) + factory := NewFactory() err := factory.RegisterConstructor("unknown", func(ctx context.Context, req ConstructRequest) (contracts.TranscriptModule, error) { _ = ctx _ = req diff --git a/internal/framework/processreport/llm_test.go b/internal/framework/processreport/llm_test.go index 3c13b3e..530636a 100644 --- a/internal/framework/processreport/llm_test.go +++ b/internal/framework/processreport/llm_test.go @@ -7,11 +7,10 @@ import ( "testing" "time" - "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput" - pk "gitea.maximumdirect.net/eric/promptkit" + pk "go.maximumdirect.net/promptkit" ) func TestExecutionReportingGroupsPredispatchFailureAndKeepsSafeMilliseconds(t *testing.T) { @@ -24,7 +23,7 @@ func TestExecutionReportingGroupsPredispatchFailureAndKeepsSafeMilliseconds(t *t Events: []llm.ExecutionObservation{{CallID: 1, Kind: string(pk.ExecutionFinished), BackendID: "primary", Attempts: 2, TransientRetries: 1, CapacityWait: 20 * time.Millisecond, GenerationDuration: 500 * time.Millisecond, RetryDelay: 100 * time.Millisecond}}, } output := &runner.RunOutput{Observations: snapshot, ObservationsAvailable: true, LLMSummary: llm.Summarize(snapshot)} - utilization := BuildUtilization(output, map[string]config.Backend{"primary": {Concurrency: 2, QueueCapacity: 3}, "secondary": {Concurrency: 1}}) + utilization := BuildUtilization(output, map[string]*pk.BackendInspection{"primary": {ConcurrencyLimit: intPointer(2), QueueCapacity: intPointer(3)}, "secondary": {ConcurrencyLimit: intPointer(1)}}) if utilization.Backends["primary"].LogicalCalls != 1 || utilization.Backends["primary"].ProviderAttempts != 2 || utilization.Backends["primary"].CapacityWaitMS != 20 || utilization.Backends["primary"].LogicalWallMS != 1500 { t.Fatalf("primary backend: %+v", utilization.Backends["primary"]) } @@ -66,7 +65,7 @@ func BenchmarkExecutionReports(b *testing.B) { }) b.Run(fmt.Sprintf("utilization/%d", n), func(b *testing.B) { for i := 0; i < b.N; i++ { - BuildUtilization(output, map[string]config.Backend{"local": {Concurrency: 1}}) + BuildUtilization(output, map[string]*pk.BackendInspection{"local": {ConcurrencyLimit: intPointer(1)}}) } }) } @@ -110,7 +109,7 @@ func TestIndexedReportsPreserveOrderAndPartialUsage(t *testing.T) { t.Fatalf("call usage: %+v", execution.Calls) } output := &runner.RunOutput{Observations: snapshot} - utilization := BuildUtilization(output, map[string]config.Backend{"a": {}, "b": {}}) + utilization := BuildUtilization(output, map[string]*pk.BackendInspection{"a": {}, "b": {}}) total := utilization.Backends["a"].Usage.Total if total.Value != 7 || !total.Known || total.Complete || utilization.Backends["a"].ProviderAttempts != 2 { t.Fatalf("partial backend usage: %+v", utilization) @@ -119,3 +118,5 @@ func TestIndexedReportsPreserveOrderAndPartialUsage(t *testing.T) { t.Fatalf("module grouping: %+v", utilization.Modules) } } + +func intPointer(value int) *int { return &value } diff --git a/internal/framework/processreport/partial_validation_test.go b/internal/framework/processreport/partial_validation_test.go index c6659a2..2b29be2 100644 --- a/internal/framework/processreport/partial_validation_test.go +++ b/internal/framework/processreport/partial_validation_test.go @@ -70,7 +70,7 @@ func TestFatalBatchRetainsEarlierDecisionsInLedger(t *testing.T) { cfg := config.Config{Chunking: config.Chunking{MaxSectionTokens: 8192, MinSectionTokens: 1}, Validation: config.Validation{MaxPromptTokens: 150}} result, err := runner.New(partialModule{validator}).Run(context.Background(), runner.RunInput{ Config: &cfg, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello"}}}, - ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}}, ValidationLLMClient: client, CaptureCandidates: true, + ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}}, CaptureCandidates: true, }) if !errors.Is(err, origin) || client.calls != 3 { t.Fatalf("fatal cause/calls: %v %d", err, client.calls) diff --git a/internal/framework/proposal_generation/adapter_integration_test.go b/internal/framework/proposal_generation/adapter_integration_test.go index 97f96fe..1c1fe4a 100644 --- a/internal/framework/proposal_generation/adapter_integration_test.go +++ b/internal/framework/proposal_generation/adapter_integration_test.go @@ -4,73 +4,74 @@ import ( "context" "encoding/json" "fmt" + "io" "net/http" - "net/http/httptest" "os" "path/filepath" + "strings" "testing" - ck "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators" "gitea.maximumdirect.net/eric/audita/v3/internal/prompts" + ck "go.maximumdirect.net/configkit" ) +type proposalTransport func(*http.Request) (*http.Response, error) + +func (f proposalTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + func TestRepairedProposalStillFacesDomainChecks(t *testing.T) { calls := 0 outputs := []string{`{"wrong":true}`, `{"corrections":[{"id":1,"original_text":"absent","corrected_text":"present","confidence":0.4}]}`} - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + transport := proposalTransport(func(r *http.Request) (*http.Response, error) { output := outputs[len(outputs)-1] if calls < len(outputs) { output = outputs[calls] } calls++ content, _ := json.Marshal(output) - _, _ = fmt.Fprintf(w, `{"choices":[{"message":{"content":%s}}]}`, content) - })) - defer server.Close() + body := fmt.Sprintf(`{"choices":[{"message":{"content":%s}}]}`, content) + return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil + }) dir := t.TempDir() - if err := os.Mkdir(filepath.Join(dir, "profiles"), 0700); err != nil { - t.Fatal(err) - } main := filepath.Join(dir, "config.yml") - content := fmt.Sprintf("version: 2\nbackends:\n offline:\n endpoint: %s/v1\nllm:\n proposal_profile: base\n transient_retries: 0\n repair_attempts: 1\n", server.URL) - if err := os.WriteFile(main, []byte(content), 0600); err != nil { + if err := os.WriteFile(main, []byte("version: 3\n"), 0600); err != nil { t.Fatal(err) } - if err := os.WriteFile(filepath.Join(dir, "profiles", "base.yaml"), []byte("profiles:\n base:\n backend: offline\n model: local-model\n"), 0600); err != nil { + if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte("profiles:\n - {id: base, endpoint: 'https://example.invalid/v1', model: local-model, api_key_required: false}\n"), 0600); err != nil { t.Fatal(err) } - loaded, err := config.Load(ck.File(main), "") + loaded, err := config.Load(ck.Input{Source: ck.File(main)}, "") if err != nil { t.Fatal(err) } - runtime, err := llm.NewRuntime(context.Background(), loaded.Config, main, server.Client()) + runtime, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport}) if err != nil { t.Fatal(err) } - client, err := runtime.NewRoleClient("proposal", llm.CallHooks{}) + settings := loaded.Config.LLM + settings.TransientRetries, settings.RepairAttempts = 0, 1 + client, err := runtime.NewRoleClient("proposal", "base", settings, llm.CallHooks{}) if err != nil { t.Fatal(err) } transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "gestures"}}} - vars, err := BuildProposalVariables(transcript, &schema.Glossary{}, 0, "") + vars, err := BuildProposalVariables(transcript, nil, 0, "") if err != nil { t.Fatal(err) } - generated, err := GenerateCandidates(context.Background(), Request{WorkingTranscript: transcript, ModuleKey: "grammar", ModuleInstance: "grammar", ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, Config: &loaded.Config, PromptID: prompts.PromptIDModuleGrammarProposal, PromptVersion: prompts.VersionV1, Variables: vars, LLMClient: client}) + generated, err := GenerateCandidates(context.Background(), Request{WorkingTranscript: transcript, ModuleKey: "grammar", ModuleInstance: "grammar", ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, PromptID: prompts.PromptIDModuleGrammarProposal, Variables: vars, LLMClient: client}) if err != nil { t.Fatal(err) } if calls != 2 || len(generated.Enriched) != 1 { t.Fatalf("repair path: calls=%d, result=%+v", calls, generated) } - cfg := loaded.Config - cfg.Thresholds.Grammar = .8 - input := validators.Request{ModuleKey: "grammar", Config: &cfg, CandidateProposal: generated.Enriched, WorkingTranscript: transcript} + input := validators.Request{ModuleKey: "grammar", Settings: config.Stage{ConfidenceThreshold: .8}, CandidateProposal: generated.Enriched, WorkingTranscript: transcript} low, err := (validators.ConfidenceThresholdValidator{}).Validate(context.Background(), input) if err != nil { t.Fatal(err) diff --git a/internal/framework/proposal_generation/generate_test.go b/internal/framework/proposal_generation/generate_test.go index 4d35a98..fa5dc4c 100644 --- a/internal/framework/proposal_generation/generate_test.go +++ b/internal/framework/proposal_generation/generate_test.go @@ -10,8 +10,6 @@ import ( "strings" "testing" - ck "gitea.maximumdirect.net/eric/ConfigKit" - "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes" @@ -21,6 +19,7 @@ import ( ) type fakeStructuredClient struct { + secrets []string responses []StructuredCorrectionSet err error completion contracts.StructuredCompletionResponse @@ -51,6 +50,8 @@ func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contr return response, nil } +func (f *fakeStructuredClient) ConfiguredSecrets(context.Context) []string { return f.secrets } + type captureDiagnosticsWriter struct { err error lastStage string @@ -69,14 +70,11 @@ func (w *captureDiagnosticsWriter) WriteInteraction(stage string, requestMetadat func defaultRequest(t *testing.T) Request { t.Helper() - cfg := config.Config{} return Request{ ModuleKey: "test_module", ModuleInstance: "test_module", ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, - Config: &cfg, PromptID: "modules.grammar.proposal", - PromptVersion: "v1", Variables: map[string]string{"SectionJSON": "[]"}, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 7, Text: "teh"}, {ID: 25, Text: "other"}}}, StartIndex: 0, @@ -323,13 +321,8 @@ func TestGenerateCandidatesDiagnosticsWrittenAndRedacted(t *testing.T) { } t.Setenv("PRIMARY_SECRET", primarySecret) t.Setenv("VALIDATION_SECRET", validationSecret) - loaded, loadErr := config.Load(ck.Bytes("config.yml", []byte("version: 2\nbackends:\n primary:\n endpoint: https://primary.invalid/v1\n credential: PRIMARY_SECRET\n validation:\n endpoint: https://validation.invalid/v1\n credential: VALIDATION_SECRET\nllm:\n proposal_profile: base\n")), "") - if loadErr != nil { - t.Fatal(loadErr) - } - cfg := loaded.Config + client.secrets = []string{primarySecret, validationSecret} req := defaultRequest(t) - req.Config = &cfg req.LLMClient = client req.DiagnosticsDir = t.TempDir() req.Variables = map[string]string{"SectionJSON": "include secret " + primarySecret} diff --git a/internal/framework/proposal_generation/module_proposal_test.go b/internal/framework/proposal_generation/module_proposal_test.go index 08e2eeb..2742e80 100644 --- a/internal/framework/proposal_generation/module_proposal_test.go +++ b/internal/framework/proposal_generation/module_proposal_test.go @@ -4,9 +4,11 @@ import ( "context" "encoding/json" "errors" + "reflect" "testing" "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" + domain "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes" @@ -22,20 +24,20 @@ func TestExecuteModuleProposalBuildsMessagesFromSectionAndDescription(t *testing }, } section := contracts.SectionMetadata{Index: 7} - cfg := config.Config{} + cfg := config.Stage{} cfg.Context.Description = "Hearing transcript with role titles." transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "teh"}}} - glossary := &schema.Glossary{Entries: []schema.GlossaryEntry{{Name: "X"}}} + glossary := []domain.Entry{{ID: "x", Name: "X"}} var gotSectionIndex int var gotDescription string var gotTranscript *schema.Transcript - var gotGlossary *schema.Glossary + var gotGlossary []domain.Entry out, err := ExecuteModuleProposal(context.Background(), ModuleProposalRequest{ ProposalRequest: contracts.ProposalRequest{ ExecutionContext: contracts.ExecutionContext{ - Config: &cfg, + Settings: cfg, WorkingTranscript: transcript, Glossary: glossary, Section: §ion, @@ -48,7 +50,7 @@ func TestExecuteModuleProposalBuildsMessagesFromSectionAndDescription(t *testing LLMClient: client, }, PromptID: prompts.PromptIDModuleGrammarProposal, - BuildVariables: func(inTranscript *schema.Transcript, inGlossary *schema.Glossary, sectionIndex int, transcriptDescription string) (map[string]string, error) { + BuildVariables: func(inTranscript *schema.Transcript, inGlossary []domain.Entry, sectionIndex int, transcriptDescription string) (map[string]string, error) { gotSectionIndex = sectionIndex gotDescription = transcriptDescription gotTranscript = inTranscript @@ -68,8 +70,8 @@ func TestExecuteModuleProposalBuildsMessagesFromSectionAndDescription(t *testing if gotTranscript != transcript { t.Fatalf("expected shared transcript pointer") } - if gotGlossary != glossary { - t.Fatalf("expected shared glossary pointer") + if !reflect.DeepEqual(gotGlossary, glossary) { + t.Fatalf("expected glossary entries") } if len(client.calls) != 1 || client.calls[0].PromptID != prompts.PromptIDModuleGrammarProposal || client.calls[0].Context.SectionIndex == nil || *client.calls[0].Context.SectionIndex != 7 { t.Fatalf("unexpected stage name calls: %+v", client.calls) @@ -85,7 +87,7 @@ func TestExecuteModuleProposalValidatesInputs(t *testing.T) { } _, err := ExecuteModuleProposal(context.Background(), ModuleProposalRequest{ - BuildVariables: func(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) (map[string]string, error) { + BuildVariables: func(transcript *schema.Transcript, glossary []domain.Entry, sectionIndex int, transcriptDescription string) (map[string]string, error) { return nil, nil }, }) @@ -95,7 +97,7 @@ func TestExecuteModuleProposalValidatesInputs(t *testing.T) { _, err = ExecuteModuleProposal(context.Background(), ModuleProposalRequest{ PromptID: "missing.prompt.id", - BuildVariables: func(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) (map[string]string, error) { + BuildVariables: func(transcript *schema.Transcript, glossary []domain.Entry, sectionIndex int, transcriptDescription string) (map[string]string, error) { return map[string]string{"SectionJSON": "[]"}, nil }, }) @@ -108,7 +110,7 @@ func TestExecuteModuleProposalPropagatesBuilderError(t *testing.T) { wantErr := errors.New("builder failed") _, err := ExecuteModuleProposal(context.Background(), ModuleProposalRequest{ PromptID: prompts.PromptIDModuleGlossaryProposal, - BuildVariables: func(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) (map[string]string, error) { + BuildVariables: func(transcript *schema.Transcript, glossary []domain.Entry, sectionIndex int, transcriptDescription string) (map[string]string, error) { return nil, wantErr }, }) @@ -128,7 +130,7 @@ func TestExecuteModuleProposalPreservesFatalOperation(t *testing.T) { LLMClient: client, }, PromptID: prompts.PromptIDModuleGrammarProposal, - BuildVariables: func(*schema.Transcript, *schema.Glossary, int, string) (map[string]string, error) { + BuildVariables: func(*schema.Transcript, []domain.Entry, int, string) (map[string]string, error) { return map[string]string{"SectionJSON": "[]"}, nil }, }) diff --git a/internal/framework/proposal_generation/variables_test.go b/internal/framework/proposal_generation/variables_test.go index 9baca56..1494ef8 100644 --- a/internal/framework/proposal_generation/variables_test.go +++ b/internal/framework/proposal_generation/variables_test.go @@ -4,12 +4,13 @@ import ( "strings" "testing" + domain "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" ) func TestBuildProposalVariablesPreservesSectionGlossaryAndDescription(t *testing.T) { transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "gestures"}}} - glossary := &schema.Glossary{Entries: []schema.GlossaryEntry{{Name: "Jesters"}}} + glossary := []domain.Entry{{ID: "jesters", Name: "Jesters"}} variables, err := BuildProposalVariables(transcript, glossary, 0, " Fantasy names ") if err != nil { t.Fatal(err) diff --git a/internal/framework/runner/cancellation_test.go b/internal/framework/runner/cancellation_test.go index 7d8f4fc..f059cf6 100644 --- a/internal/framework/runner/cancellation_test.go +++ b/internal/framework/runner/cancellation_test.go @@ -10,7 +10,6 @@ import ( "testing" "time" - ck "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking" "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" @@ -21,7 +20,8 @@ import ( "gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators" "gitea.maximumdirect.net/eric/audita/v3/internal/prompts" - pk "gitea.maximumdirect.net/eric/promptkit" + ck "go.maximumdirect.net/configkit" + pk "go.maximumdirect.net/promptkit" ) type contextValidator struct { @@ -47,23 +47,20 @@ func (t *cancelTransport) RoundTrip(req *http.Request) (*http.Response, error) { func TestValidatorFailureCancelsActiveAndPendingProviderWork(t *testing.T) { dir := t.TempDir() - if err := os.Mkdir(filepath.Join(dir, "profiles"), 0700); err != nil { - t.Fatal(err) - } main := filepath.Join(dir, "config.yml") - body := "version: 2\nbackends:\n shared:\n endpoint: http://provider.invalid/v1\n concurrency: 1\n queue_capacity: 1\nllm:\n proposal_profile: base\n transient_retries: 0\n repair_attempts: 0\n" - if err := os.WriteFile(main, []byte(body), 0600); err != nil { + if err := os.WriteFile(main, []byte("version: 3\n"), 0600); err != nil { t.Fatal(err) } - if err := os.WriteFile(filepath.Join(dir, "profiles/base.yaml"), []byte("profiles:\n base:\n backend: shared\n model: local\n timeout_seconds: 30\n"), 0600); err != nil { + resources := "backends:\n - id: shared\n endpoint: http://provider.invalid/v1\n concurrency_limit: 1\n queue_capacity: 1\nprofiles:\n - id: base\n backend: shared\n model: local\n" + if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte(resources), 0600); err != nil { t.Fatal(err) } - loaded, err := config.Load(ck.File(main), "") + loaded, err := config.Load(ck.Input{Source: ck.File(main)}, "") if err != nil { t.Fatal(err) } transport := &cancelTransport{entered: make(chan struct{}, 3)} - runtime, err := llm.NewRuntime(context.Background(), loaded.Config, main, &http.Client{Transport: transport}) + runtime, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport}) if err != nil { t.Fatal(err) } @@ -75,7 +72,7 @@ func TestValidatorFailureCancelsActiveAndPendingProviderWork(t *testing.T) { started <- struct{}{} } } - client, err := runtime.NewRoleClient("validation", hooks) + client, err := runtime.NewRoleClient("validation", "base", config.LLM{OperationTimeout: 30 * time.Second}, hooks) if err != nil { t.Fatal(err) } @@ -101,7 +98,7 @@ func TestValidatorFailureCancelsActiveAndPendingProviderWork(t *testing.T) { } var out map[string]any _, err := client.CompleteStructured(ctx, structuredoutput.Request{ - PromptID: prompts.PromptIDValidatorEditorialReview, PromptVersion: prompts.VersionV1, + PromptID: prompts.PromptIDValidatorEditorialReview, Variables: map[string]string{"TranscriptDescriptionBlock": "", "PayloadJSON": "[]"}, Schema: responseschema.ValidatorDecisionSetKey, Context: structuredoutput.DiagnosticContext{ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: req.SectionIndex}, }, &out) @@ -120,7 +117,7 @@ func TestValidatorFailureCancelsActiveAndPendingProviderWork(t *testing.T) { } done := make(chan pipelineOutcome, 1) go func() { - result, err := runModulePipeline(ctx, collectSectionProposalsInput{Module: module, Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar"}, Policy: module.policy, Config: &loaded.Config, Sections: sections}) + result, err := runModulePipeline(ctx, collectSectionProposalsInput{Module: module, Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar"}, Policy: module.policy, Sections: sections}) done <- pipelineOutcome{result: result, err: err} }() select { diff --git a/internal/framework/runner/runner_test.go b/internal/framework/runner/runner_test.go index c9ad1ed..c4aff57 100644 --- a/internal/framework/runner/runner_test.go +++ b/internal/framework/runner/runner_test.go @@ -11,8 +11,8 @@ import ( "testing" "time" - ck "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" + "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/modules" @@ -717,10 +717,7 @@ func TestRunnerDoesNotMutateInputTranscript(t *testing.T) { } func TestRunnerRepeatedModuleInstanceNames(t *testing.T) { - specs, err := contracts.ResolveModuleRunSpecs([]string{"glossary", "glossary"}) - if err != nil { - t.Fatalf("ResolveModuleRunSpecs error: %v", err) - } + specs := contracts.ResolveModuleRunSpecs(&config.Config{Pipeline: config.Pipeline{Stages: []config.Stage{{Module: "glossary"}, {Module: "glossary"}}}}) if specs[0].InstanceName != "glossary_1" || specs[1].InstanceName != "glossary_2" { t.Fatalf("unexpected instance names: %+v", specs) } @@ -920,10 +917,13 @@ func ptrConfig(c config.Config) *config.Config { return &c } type fakeStructuredClient struct { responses []validators.LLMValidationResponse + secrets []string err error calls int } +func (f *fakeStructuredClient) ConfiguredSecrets(context.Context) []string { return f.secrets } + func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { _ = ctx _ = req @@ -955,10 +955,9 @@ func TestRunnerLLMValidatorApprovalApplied(t *testing.T) { }}) cfg := runnerTestConfig() out, err := r.Run(context.Background(), RunInput{ - Config: &cfg, - Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}}, - ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, - ValidationLLMClient: client, + Config: &cfg, + Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}}, + ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}}, }) if err != nil { t.Fatalf("Run error: %v", err) @@ -978,10 +977,9 @@ func TestRunnerLLMValidatorRejectionPreventsApplication(t *testing.T) { }}) cfg := runnerTestConfig() out, err := r.Run(context.Background(), RunInput{ - Config: &cfg, - Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}}, - ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, - ValidationLLMClient: client, + Config: &cfg, + Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}}, + ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}}, }) if err != nil { t.Fatalf("Run error: %v", err) @@ -1007,10 +1005,9 @@ func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress( }}) cfg := runnerTestConfig() out, err := r.Run(context.Background(), RunInput{ - Config: &cfg, - Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}}, - ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {ModuleKey: "m2", InstanceName: "m2"}}, - ValidationLLMClient: client, + Config: &cfg, + Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}}, + ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {ModuleKey: "m2", InstanceName: "m2", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}}, }) if err != nil { t.Fatalf("expected malformed validator response to downgrade, got %v", err) @@ -1058,7 +1055,7 @@ func TestRunnerReconciledSurvivorsCompleteLaterValidation(t *testing.T) { cfg := runnerTestConfig() out, err := r.Run(context.Background(), RunInput{ Config: &cfg, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat bad"}}}, - ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, ValidationLLMClient: client, + ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}}, }) if err != nil || out.FinalTranscript.Segments[0].Text != "the cat bad" || len(laterIndices) != 2 || laterIndices[0] != 0 || laterIndices[1] != 2 || client.calls != 1 { t.Fatalf("survivor routing: text=%q later=%v calls=%d err=%v", out.FinalTranscript.Segments[0].Text, laterIndices, client.calls, err) @@ -1083,10 +1080,9 @@ func TestRunnerLLMValidatorMissingDecisionRejectsBatch(t *testing.T) { }}) cfg := runnerTestConfig() _, err := r.Run(context.Background(), RunInput{ - Config: &cfg, - Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}}, - ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, - ValidationLLMClient: client, + Config: &cfg, + Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}}, + ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}}, }) if err != nil { t.Fatalf("expected missing decision downgrade, got %v", err) @@ -1106,10 +1102,9 @@ func TestRunnerLLMValidatorDuplicateDecisionRejectsBatch(t *testing.T) { }}) cfg := runnerTestConfig() _, err := r.Run(context.Background(), RunInput{ - Config: &cfg, - Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}}, - ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, - ValidationLLMClient: client, + Config: &cfg, + Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}}, + ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}}, }) if err != nil { t.Fatalf("expected duplicate decision downgrade, got %v", err) @@ -1139,10 +1134,9 @@ func TestRunnerLLMValidatorBatching(t *testing.T) { }}, }}) out, err := r.Run(context.Background(), RunInput{ - Config: &cfg, - Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "alpha gamma"}}}, - ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, - ValidationLLMClient: client, + Config: &cfg, + Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "alpha gamma"}}}, + ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}}, }) if err != nil { t.Fatalf("unexpected run error: %v", err) @@ -1161,13 +1155,8 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) { validationSecret := "runner-validation-secret" client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: validationSecret}}}}} llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility) - t.Setenv("RUNNER_PRIMARY_KEY", primarySecret) - t.Setenv("RUNNER_VALIDATION_KEY", validationSecret) - loaded, loadErr := config.Load(ck.Bytes("config.yml", []byte("version: 2\nbackends:\n primary:\n endpoint: https://primary.invalid/v1\n credential: RUNNER_PRIMARY_KEY\n validation:\n endpoint: https://validation.invalid/v1\n credential: RUNNER_VALIDATION_KEY\nllm:\n proposal_profile: base\n")), "") - if loadErr != nil { - t.Fatal(loadErr) - } - cfg := loaded.Config + client.secrets = []string{primarySecret, validationSecret} + cfg := runnerTestConfig() diagDir := t.TempDir() r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { @@ -1177,8 +1166,7 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) { out, err := r.Run(context.Background(), RunInput{ Config: &cfg, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple. " + primarySecret}}}, - ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, - ValidationLLMClient: client, + ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}}, ValidationDiagnosticsDir: diagDir, }) if err != nil { @@ -1227,10 +1215,7 @@ func (m proposalGenerationModule) Propose(ctx context.Context, req contracts.Pro ModuleInstance: req.RunSpec.InstanceName, ReplacementPolicy: req.RunSpec.ReplacementPolicy, WorkingTranscript: req.WorkingTranscript, - Config: req.Config, - Glossary: req.Glossary, PromptID: "modules.grammar.proposal", - PromptVersion: "v1", Variables: map[string]string{"TranscriptDescriptionBlock": "", "GlossaryJSON": "[]", "SectionJSON": "[]"}, LLMClient: req.LLMClient, DiagnosticsDir: req.DiagnosticsDir, @@ -1281,8 +1266,7 @@ func TestRunnerProposalGenerationHelperFlowsThroughPipeline(t *testing.T) { out, err := r.Run(context.Background(), RunInput{ Config: &cfg, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}}, - ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, - ProposalLLMClient: client, + ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ProposalClient: client, Settings: config.Stage{ConfidenceThreshold: cfg.Thresholds.Grammar}}}, ProposalDiagnosticsDir: diagDir, }) if err != nil { @@ -1315,13 +1299,11 @@ func TestRunnerGrammarModuleUsesConfidenceThreshold(t *testing.T) { } cfg := runnerTestConfig() cfg.Thresholds.Grammar = 0.9 - factory := modules.NewFactory(modules.Dependencies{}) + factory := modules.NewFactory() out, err := New(factory).Run(context.Background(), RunInput{ - Config: &cfg, - Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello ,world"}}}, - Glossary: &schema.Glossary{}, - ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}}, - ProposalLLMClient: client, + Config: &cfg, + Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello ,world"}}}, + ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", ProposalClient: client, Settings: config.Stage{ConfidenceThreshold: cfg.Thresholds.Grammar}}}, }) if err != nil { t.Fatalf("Run error: %v", err) @@ -1349,13 +1331,11 @@ func TestRunnerGlossaryModuleUsesConfidenceThreshold(t *testing.T) { } cfg := runnerTestConfig() cfg.Thresholds.Glossary = 0.9 - factory := modules.NewFactory(modules.Dependencies{}) + factory := modules.NewFactory() out, err := New(factory).Run(context.Background(), RunInput{ - Config: &cfg, - Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures"}}}, - Glossary: &schema.Glossary{Entries: []schema.GlossaryEntry{{Name: "Jesters", Category: "faction", Summary: "Faction"}}}, - ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "glossary", InstanceName: "glossary"}}, - ProposalLLMClient: client, + Config: &cfg, + Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures"}}}, + ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "glossary", InstanceName: "glossary", Glossary: []glossary.Entry{{Name: "Jesters", Category: "faction", Description: "Faction"}}, ProposalClient: client, Settings: config.Stage{ConfidenceThreshold: cfg.Thresholds.Glossary}}}, }) if err != nil { t.Fatalf("Run error: %v", err) diff --git a/internal/framework/validators/llm_context_test.go b/internal/framework/validators/llm_context_test.go index 548cfc2..3cc0c23 100644 --- a/internal/framework/validators/llm_context_test.go +++ b/internal/framework/validators/llm_context_test.go @@ -6,6 +6,7 @@ import ( "testing" "gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking" + "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" ) @@ -19,11 +20,11 @@ func TestValidationContextCarriesAdjacentEvidenceAndRelevantDefinitions(t *testi {ID: 4, Text: "Elsewhere."}, }} req.CandidateProposal[0].TargetSegmentID = 2 - req.Glossary = &schema.Glossary{Entries: []schema.GlossaryEntry{ - {Name: "Jesters", Summary: "A traveling troupe"}, - {Name: "Lyra", Summary: "The host"}, - {Name: "Unrelated", Summary: "Not mentioned"}, - }} + req.Glossary = []glossary.Entry{ + {ID: "jesters", Name: "Jesters", Description: "A traveling troupe"}, + {ID: "lyra", Name: "Lyra", Description: "The host"}, + {ID: "unrelated", Name: "Unrelated", Description: "Not mentioned"}, + } payload, immediate := BuildLLMValidationRequest("spoken_form_plausibility", LLMValidatorTypeSpokenFormPlausibility, req) if len(immediate) != 0 || len(payload.Items) != 1 { t.Fatalf("payload: %+v immediate: %+v", payload, immediate) @@ -40,10 +41,10 @@ func TestValidationContextCarriesAdjacentEvidenceAndRelevantDefinitions(t *testi if err := json.Unmarshal([]byte(variables["PayloadJSON"]), &rendered); err != nil { t.Fatal(err) } - if strings.Contains(variables["PayloadJSON"], `"id":`) || rendered[0].CorrectionIndex != 1 || payload.Items[0].CorrectionIndex != 7 { + if strings.Contains(variables["PayloadJSON"], `"segment_id":`) || rendered[0].CorrectionIndex != 1 || payload.Items[0].CorrectionIndex != 7 { t.Fatalf("leaked internal identity or mutated request: %s", variables["PayloadJSON"]) } - if len(rendered[0].NeighboringSegments) != 2 || rendered[0].GlossaryEntries[0].Summary != "A traveling troupe" { + if len(rendered[0].NeighboringSegments) != 2 || rendered[0].GlossaryEntries[0].Description != "A traveling troupe" { t.Fatalf("rendered context: %+v", rendered) } // The context-free endpoint must not reach beyond the current chunk. @@ -64,7 +65,7 @@ func TestValidationContextFitsBudgetWithoutDiscardingCandidate(t *testing.T) { } enriched := base enriched.NeighboringSegments = []NeighboringSegment{{Position: "following", Text: strings.Repeat("long context ", 300)}} - enriched.GlossaryEntries = []schema.GlossaryEntry{{Name: "cat", Summary: strings.Repeat("long definition ", 300)}} + enriched.GlossaryEntries = []glossary.Entry{{Name: "cat", Description: strings.Repeat("long definition ", 300)}} fitted := fitValidationContext(enriched, required, estimator) tokens, err := estimateBatchTokens(estimator, []LLMValidationItem{fitted}) if err != nil || tokens > required || fitted.OriginalSegmentText != base.OriginalSegmentText || fitted.CorrectedSegmentText != base.CorrectedSegmentText { @@ -80,7 +81,7 @@ func TestValidationContextFitsBudgetWithoutDiscardingCandidate(t *testing.T) { func TestValidationContextSelectsGlossaryForEachReplacement(t *testing.T) { req := makeReq([]proposals.EnrichedCorrectionProposal{mk(7, "gestures", "Jesters"), mk(11, "gestures", "Actors")}) req.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Those gestures are back."}}} - req.Glossary = &schema.Glossary{Entries: []schema.GlossaryEntry{{Name: "Actors"}, {Name: "Jesters"}}} + req.Glossary = []glossary.Entry{{ID: "actors", Name: "Actors"}, {ID: "jesters", Name: "Jesters"}} payload, immediate := BuildLLMValidationRequest("editorial_review", LLMValidatorTypeEditorialReview, req) if len(immediate) != 0 || len(payload.Items) != 2 { t.Fatalf("payload: %+v immediate: %+v", payload, immediate) diff --git a/internal/framework/validators/llm_integration_test.go b/internal/framework/validators/llm_integration_test.go index 2a5601b..1896abb 100644 --- a/internal/framework/validators/llm_integration_test.go +++ b/internal/framework/validators/llm_integration_test.go @@ -4,19 +4,24 @@ import ( "context" "encoding/json" "fmt" + "io" "net/http" - "net/http/httptest" "os" "path/filepath" + "strings" "testing" - ck "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" + ck "go.maximumdirect.net/configkit" ) +type validationTransport func(*http.Request) (*http.Response, error) + +func (f validationTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) { valid := `{"validations":[{"correction_index":1,"approved":true,"confidence":0.9,"reason":"ok"}]}` for _, tc := range []struct { @@ -32,41 +37,38 @@ func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { calls := 0 - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + transport := validationTransport(func(r *http.Request) (*http.Response, error) { calls++ output := tc.responses[len(tc.responses)-1] if calls <= len(tc.responses) { output = tc.responses[calls-1] } - if output == "MALFORMED_ENVELOPE" { - _, _ = fmt.Fprint(w, `{"choices":[]}`) - return + body := `{"choices":[]}` + if output != "MALFORMED_ENVELOPE" { + content, _ := json.Marshal(output) + body = fmt.Sprintf(`{"choices":[{"message":{"content":%s}}]}`, content) } - content, _ := json.Marshal(output) - _, _ = fmt.Fprintf(w, `{"choices":[{"message":{"content":%s}}]}`, content) - })) - defer server.Close() + return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil + }) dir := t.TempDir() - if err := os.Mkdir(filepath.Join(dir, "profiles"), 0700); err != nil { - t.Fatal(err) - } main := filepath.Join(dir, "config.yml") - content := fmt.Sprintf("version: 2\nbackends:\n offline:\n endpoint: %s/v1\nllm:\n proposal_profile: base\n transient_retries: 0\n repair_attempts: 1\n", server.URL) - if err := os.WriteFile(main, []byte(content), 0600); err != nil { + if err := os.WriteFile(main, []byte("version: 3\n"), 0600); err != nil { t.Fatal(err) } - if err := os.WriteFile(filepath.Join(dir, "profiles", "base.yaml"), []byte("profiles:\n base:\n backend: offline\n model: local-model\n"), 0600); err != nil { + if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte("profiles:\n - {id: base, endpoint: 'https://example.invalid/v1', model: local-model, api_key_required: false}\n"), 0600); err != nil { t.Fatal(err) } - loaded, err := config.Load(ck.File(main), "") + loaded, err := config.Load(ck.Input{Source: ck.File(main)}, "") if err != nil { t.Fatal(err) } - runtime, err := llm.NewRuntime(context.Background(), loaded.Config, main, server.Client()) + runtime, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport}) if err != nil { t.Fatal(err) } - client, err := runtime.NewRoleClient("validation", llm.CallHooks{}) + settings := loaded.Config.LLM + settings.TransientRetries, settings.RepairAttempts = 0, 1 + client, err := runtime.NewRoleClient("validation", "base", settings, llm.CallHooks{}) if err != nil { t.Fatal(err) } @@ -75,7 +77,6 @@ func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) { t.Fatal(err) } req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")}) - req.Config = &loaded.Config req.LLMClient = client result, err := validator.Validate(context.Background(), req) if calls != tc.wantCalls { diff --git a/internal/framework/validators/llm_partial_test.go b/internal/framework/validators/llm_partial_test.go index bed46f0..eb69eec 100644 --- a/internal/framework/validators/llm_partial_test.go +++ b/internal/framework/validators/llm_partial_test.go @@ -55,7 +55,7 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) { validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview) validator.SetTokenEstimator(perItemEstimator{}) req := makeReq(candidates) - req.Config.Validation.MaxPromptTokens = 150 + req.Settings.Validation.MaxPromptTokens = 150 req.LLMClient = client result, err := validator.Validate(context.Background(), req) if err != nil { @@ -79,7 +79,7 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) { nextValidator, _ := NewLLMBackedValidator("meaning_reversal_review", LLMValidatorTypeMeaningReversal) nextValidator.SetTokenEstimator(perItemEstimator{}) nextReq := makeReq(survivors) - nextReq.Config.Validation.MaxPromptTokens = 150 + nextReq.Settings.Validation.MaxPromptTokens = 150 nextReq.LLMClient = nextClient next, err := nextValidator.Validate(context.Background(), nextReq) if err != nil || len(next.Decisions) != 2 || len(nextClient.calls) != 2 || *nextClient.calls[0].Context.BatchIndex != 0 || *nextClient.calls[1].Context.BatchIndex != 1 { @@ -144,7 +144,7 @@ func TestValidationFatalFailureKeepsEarlierFactsAndStops(t *testing.T) { validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview) validator.SetTokenEstimator(perItemEstimator{}) req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters"), mk(1, "gestures", "actors"), mk(2, "gestures", "figures")}) - req.Config.Validation.MaxPromptTokens = 150 + req.Settings.Validation.MaxPromptTokens = 150 req.LLMClient = client result, err := validator.Validate(context.Background(), req) if !errors.Is(err, cause) || len(result.Decisions) != 1 || len(result.Operations) != 2 || len(client.calls) != 2 { diff --git a/internal/framework/validators/llm_validators_test.go b/internal/framework/validators/llm_validators_test.go index 0241c01..aee3b41 100644 --- a/internal/framework/validators/llm_validators_test.go +++ b/internal/framework/validators/llm_validators_test.go @@ -59,15 +59,14 @@ func (f *fakeStructuredLLMClient) CompleteStructured(ctx context.Context, req St } func makeReq(candidates []proposals.EnrichedCorrectionProposal) Request { - cfg := config.Config{} - cfg.Validation.MaxPromptTokens = 10000 + settings := config.Stage{Validation: config.Validation{MaxPromptTokens: 10000}} return Request{ WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple.", Categories: []string{"narration"}}}}, CandidateProposal: candidates, ModuleKey: "homophones", ModuleInstance: "homophones", ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, - Config: &cfg, + Settings: settings, } } @@ -314,9 +313,7 @@ func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *tes ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, } req.LLMClient = client - cfg := config.Config{} - cfg.Validation.MaxPromptTokens = 200 - req.Config = &cfg + req.Settings.Validation.MaxPromptTokens = 200 res, err := v.Validate(context.Background(), req) if err != nil { @@ -384,7 +381,7 @@ func TestEveryValidationTypeSelectsEmbeddedPromptAndVariables(t *testing.T) { } req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")}) req.LLMClient = client - req.Config.Context.Description = " Fantasy scene " + req.Settings.Context.Description = " Fantasy scene " if _, err := validator.Validate(context.Background(), req); err != nil { t.Fatal(err) } @@ -392,7 +389,7 @@ func TestEveryValidationTypeSelectsEmbeddedPromptAndVariables(t *testing.T) { t.Fatalf("calls=%d", len(client.calls)) } call := client.calls[0] - if call.PromptID != tc.prompt || call.PromptVersion != prompts.VersionV1 || call.Schema != responseschema.ValidatorDecisionSetKey || call.Context.Role != "validation" || call.Context.BatchIndex == nil || !strings.Contains(call.Variables["PayloadJSON"], "correction_index") || !strings.Contains(call.Variables["TranscriptDescriptionBlock"], "Fantasy scene") { + if call.PromptID != tc.prompt || call.Schema != responseschema.ValidatorDecisionSetKey || call.Context.Role != "validation" || call.Context.BatchIndex == nil || !strings.Contains(call.Variables["PayloadJSON"], "correction_index") || !strings.Contains(call.Variables["TranscriptDescriptionBlock"], "Fantasy scene") { t.Fatalf("mapping: %+v", call) } }) diff --git a/internal/framework/validators/protected_terms_test.go b/internal/framework/validators/protected_terms_test.go index 3201f79..27cdfdf 100644 --- a/internal/framework/validators/protected_terms_test.go +++ b/internal/framework/validators/protected_terms_test.go @@ -2,40 +2,36 @@ package validators import ( "reflect" + "sort" "testing" + domain "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" ) -func TestExtractProtectedTermsStableDeduplicatedWithPlurals(t *testing.T) { - glossary := &schema.Glossary{ - Entries: []schema.GlossaryEntry{ - {Name: "Jesters", Aliases: []string{"Jester", " "}, Plural: "Jesters", Category: "faction", Summary: "Faction"}, - {Name: "Hrank", Aliases: []string{"hrank", "Hrank"}, Plural: "Hranks", Category: "pc", Summary: "Character"}, - {Name: "Lyra", Aliases: []string{}, Plural: "", Category: "npc", Summary: "NPC"}, - }, +func TestRecognizedTermsDeduplicateAndIncludePlurals(t *testing.T) { + plural := "Hranks" + entries := []domain.Entry{{ID: "jesters", Name: "Jesters", Aliases: []string{"Jester", " "}}, {ID: "hrank", Name: "Hrank", Aliases: []string{"hrank", "Hrank"}, Plural: &plural}, {ID: "lyra", Name: "Lyra"}} + var got []string + for _, term := range domain.Vocabulary(entries, nil) { + got = append(got, term.Spelling) } - - got := ExtractProtectedTerms(glossary) + sort.Strings(got) want := []string{"Hrank", "Hranks", "Jester", "Jesters", "Jesterss", "Lyra", "Lyras"} if !reflect.DeepEqual(got, want) { - t.Fatalf("unexpected protected terms\n got: %v\nwant: %v", got, want) + t.Fatalf("got %v want %v", got, want) } -} - -func TestExtractProtectedTermsEmptyGlossary(t *testing.T) { - if terms := ExtractProtectedTerms(&schema.Glossary{}); len(terms) != 0 { - t.Fatalf("expected no terms, got %v", terms) + if len(domain.Vocabulary(nil, nil)) != 0 { + t.Fatal("empty input produced terms") } } func TestValidateProtectedTermsAppliesToNonGlossaryModule(t *testing.T) { req := Request{ - ModuleKey: "grammar", - Glossary: &schema.Glossary{ - Entries: []schema.GlossaryEntry{{Name: "Jesters", Category: "faction", Summary: "Faction"}}, - }, + ModuleKey: "grammar", + Glossary: []domain.Entry{{ID: "jesters", Name: "Jesters", Category: "faction"}}, + ProtectedTerms: domain.Vocabulary([]domain.Entry{{ID: "jesters", Name: "Jesters"}}, nil), CandidateProposal: []proposals.EnrichedCorrectionProposal{ { CorrectionProposal: proposals.CorrectionProposal{ @@ -52,7 +48,7 @@ func TestValidateProtectedTermsAppliesToNonGlossaryModule(t *testing.T) { }, }, } - res, err := ValidateProtectedTerms(req, false) + res, err := ValidateProtectedTerms(req) if err != nil { t.Fatalf("validator error: %v", err) } @@ -64,9 +60,7 @@ func TestValidateProtectedTermsAppliesToNonGlossaryModule(t *testing.T) { func TestValidateProtectedTermsGlossaryStageAllowsProtectedTermSwap(t *testing.T) { req := Request{ ModuleKey: "glossary", - Glossary: &schema.Glossary{ - Entries: []schema.GlossaryEntry{{Name: "Jesters", Category: "faction", Summary: "Faction"}}, - }, + Glossary: []domain.Entry{{ID: "jesters", Name: "Jesters", Category: "faction"}}, CandidateProposal: []proposals.EnrichedCorrectionProposal{ { CorrectionProposal: proposals.CorrectionProposal{ @@ -79,7 +73,7 @@ func TestValidateProtectedTermsGlossaryStageAllowsProtectedTermSwap(t *testing.T }, }, } - res, err := ValidateProtectedTerms(req, true) + res, err := ValidateProtectedTerms(req) if err != nil { t.Fatalf("validator error: %v", err) } @@ -103,10 +97,10 @@ func TestProtectedNameRepetitionRequiresSurvivingIdentityAndSpokenWordScope(t *t t.Run(tc.name, func(t *testing.T) { req := Request{ModuleKey: tc.module, ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: tc.before}}}, - Glossary: &schema.Glossary{Entries: []schema.GlossaryEntry{{Name: "Jesters"}, {Name: "Lyra"}}}, + ProtectedTerms: domain.Vocabulary([]domain.Entry{{ID: "jesters", Name: "Jesters"}, {ID: "lyra", Name: "Lyra"}}, nil), CandidateProposal: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: tc.original, CorrectedText: tc.replacement, Confidence: 1}}}, } - result, err := ValidateProtectedTerms(req, false) + result, err := ValidateProtectedTerms(req) if err != nil || len(result.Decisions) != 1 || result.Decisions[0].Approved != tc.allowed { t.Fatalf("result=%+v err=%v", result, err) } diff --git a/internal/framework/validators/validators_test.go b/internal/framework/validators/validators_test.go index 8c918b9..cb85b4a 100644 --- a/internal/framework/validators/validators_test.go +++ b/internal/framework/validators/validators_test.go @@ -6,6 +6,7 @@ import ( "testing" "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" + domain "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" ) @@ -50,9 +51,7 @@ func TestEnforceDecisionCardinalityUnknownDecision(t *testing.T) { } func TestConfidenceThresholdValidator(t *testing.T) { - cfg := config.Config{} - cfg.Thresholds.Grammar = 0.8 - req := Request{ModuleKey: "grammar", Config: &cfg, CandidateProposal: []proposals.EnrichedCorrectionProposal{ + req := Request{ModuleKey: "grammar", Settings: config.Stage{ConfidenceThreshold: 0.8}, CandidateProposal: []proposals.EnrichedCorrectionProposal{ mkCandidate(0, 1, "teh", "the", 0.9), mkCandidate(1, 1, "recieve", "receive", 0.7), }} @@ -161,16 +160,17 @@ func TestNoEffectValidator(t *testing.T) { } func TestValidateProtectedTerms(t *testing.T) { - glossary := &schema.Glossary{Entries: []schema.GlossaryEntry{{Name: "OpenAI", Aliases: []string{"Open AI"}, Plural: "OpenAIs", Category: "brand", Summary: "brand"}}} + entries := []domain.Entry{{ID: "openai", Name: "OpenAI", Aliases: []string{"Open AI"}, Category: "brand", Description: "brand"}} req := Request{ - Glossary: glossary, - ModuleKey: "grammar", + Glossary: entries, + ProtectedTerms: domain.Vocabulary(entries, nil), + ModuleKey: "grammar", CandidateProposal: []proposals.EnrichedCorrectionProposal{ mkCandidate(0, 1, "OpenAI", "Open A I", 0.9), mkCandidate(1, 1, "teh", "the", 0.9), }, } - res, err := ValidateProtectedTerms(req, false) + res, err := ValidateProtectedTerms(req) if err != nil { t.Fatalf("Validate error: %v", err) } diff --git a/internal/modules/glossary/module_test.go b/internal/modules/glossary/module_test.go index 2c5da18..2184ec2 100644 --- a/internal/modules/glossary/module_test.go +++ b/internal/modules/glossary/module_test.go @@ -8,8 +8,7 @@ import ( "strings" "testing" - ck "gitea.maximumdirect.net/eric/ConfigKit" - "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" + domain "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation" @@ -47,10 +46,9 @@ func tinyTranscript() *schema.Transcript { }} } -func tinyGlossary() *schema.Glossary { - return &schema.Glossary{Entries: []schema.GlossaryEntry{ - {Name: "Jesters", Aliases: []string{"Jester"}, Category: "faction", Summary: "Guild members", Plural: "Jesters"}, - }} +func tinyGlossary() []domain.Entry { + plural := "Jesters" + return []domain.Entry{{Collection: "party", ID: "jesters", Name: "Jesters", Aliases: []string{"Jester"}, Category: "faction", Description: "Guild members", Plural: &plural}} } func TestGlossaryModuleReplacementPolicy(t *testing.T) { @@ -99,11 +97,6 @@ func TestGlossaryModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T) }, } t.Setenv("MODULE_SECRET", secret) - loaded, loadErr := config.Load(ck.Bytes("config.yml", []byte("version: 2\nbackends:\n offline:\n endpoint: https://offline.invalid/v1\n credential: MODULE_SECRET\nllm:\n proposal_profile: base\n")), "") - if loadErr != nil { - t.Fatal(loadErr) - } - cfg := loaded.Config m, err := New() if err != nil { t.Fatalf("New error: %v", err) @@ -111,7 +104,6 @@ func TestGlossaryModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T) diagDir := t.TempDir() out, err := m.Propose(context.Background(), contracts.ProposalRequest{ ExecutionContext: contracts.ExecutionContext{ - Config: &cfg, WorkingTranscript: tinyTranscript(), Glossary: tinyGlossary(), DiagnosticsDir: diagDir, diff --git a/internal/modules/grammar/module_test.go b/internal/modules/grammar/module_test.go index 4164409..90264f1 100644 --- a/internal/modules/grammar/module_test.go +++ b/internal/modules/grammar/module_test.go @@ -8,8 +8,7 @@ import ( "strings" "testing" - ck "gitea.maximumdirect.net/eric/ConfigKit" - "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" + domain "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation" @@ -57,10 +56,9 @@ func tinyTranscript() *schema.Transcript { }} } -func tinyGlossary() *schema.Glossary { - return &schema.Glossary{Entries: []schema.GlossaryEntry{ - {Name: "Jesters", Category: "faction", Summary: "Faction", Aliases: []string{"Jester"}}, - }} +func tinyGlossary() []domain.Entry { + plural := "Jesters" + return []domain.Entry{{Collection: "party", ID: "jesters", Name: "Jesters", Aliases: []string{"Jester"}, Category: "faction", Description: "Guild members", Plural: &plural}} } func TestGrammarModuleReplacementPolicy(t *testing.T) { @@ -109,11 +107,6 @@ func TestGrammarModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T) { }, } t.Setenv("MODULE_SECRET", secret) - loaded, loadErr := config.Load(ck.Bytes("config.yml", []byte("version: 2\nbackends:\n offline:\n endpoint: https://offline.invalid/v1\n credential: MODULE_SECRET\nllm:\n proposal_profile: base\n")), "") - if loadErr != nil { - t.Fatal(loadErr) - } - cfg := loaded.Config module, err := New() if err != nil { t.Fatalf("New error: %v", err) @@ -121,7 +114,6 @@ func TestGrammarModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T) { diagDir := t.TempDir() out, err := module.Propose(context.Background(), contracts.ProposalRequest{ ExecutionContext: contracts.ExecutionContext{ - Config: &cfg, WorkingTranscript: tinyTranscript(), Glossary: tinyGlossary(), DiagnosticsDir: diagDir, diff --git a/internal/modules/homophones/module_test.go b/internal/modules/homophones/module_test.go index c88ce43..ae74618 100644 --- a/internal/modules/homophones/module_test.go +++ b/internal/modules/homophones/module_test.go @@ -8,8 +8,8 @@ import ( "strings" "testing" - ck "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" + domain "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation" @@ -48,10 +48,9 @@ func tinyTranscript() *schema.Transcript { }} } -func tinyGlossary() *schema.Glossary { - return &schema.Glossary{Entries: []schema.GlossaryEntry{ - {Name: "Jesters", Aliases: []string{"Jester"}, Category: "faction", Summary: "Guild members", Plural: "Jesters"}, - }} +func tinyGlossary() []domain.Entry { + plural := "Jesters" + return []domain.Entry{{Collection: "party", ID: "jesters", Name: "Jesters", Aliases: []string{"Jester"}, Category: "faction", Description: "Guild members", Plural: &plural}} } func TestHomophonesModuleReplacementPolicy(t *testing.T) { @@ -100,11 +99,6 @@ func TestHomophonesModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T }, } t.Setenv("MODULE_SECRET", secret) - loaded, loadErr := config.Load(ck.Bytes("config.yml", []byte("version: 2\nbackends:\n offline:\n endpoint: https://offline.invalid/v1\n credential: MODULE_SECRET\nllm:\n proposal_profile: base\n")), "") - if loadErr != nil { - t.Fatal(loadErr) - } - cfg := loaded.Config m, err := New() if err != nil { t.Fatalf("New error: %v", err) @@ -112,7 +106,6 @@ func TestHomophonesModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T diagDir := t.TempDir() out, err := m.Propose(context.Background(), contracts.ProposalRequest{ ExecutionContext: contracts.ExecutionContext{ - Config: &cfg, WorkingTranscript: tinyTranscript(), Glossary: tinyGlossary(), DiagnosticsDir: diagDir, @@ -173,10 +166,8 @@ func TestHomophonesConfidenceThresholdIsUsed(t *testing.T) { t.Fatal("expected confidence_threshold validator") } - cfg := config.Config{} - cfg.Thresholds.Homophones = 0.95 result, err := thresholdValidator.Validate(context.Background(), validators.Request{ - Config: &cfg, + Settings: config.Stage{ConfidenceThreshold: 0.95}, ModuleKey: "homophones", CandidateProposal: []proposals.EnrichedCorrectionProposal{ { @@ -223,12 +214,11 @@ func TestHomophonesProtectedGlossaryTermBehaviorRejectsUnsafeCorrection(t *testi working := &schema.Transcript{Segments: []schema.Segment{ {ID: 1, Speaker: "A", Start: 0, End: 1, Text: "Jesters are here."}, }} - glossary := &schema.Glossary{Entries: []schema.GlossaryEntry{ - {Name: "Jesters", Category: "faction", Summary: "Faction"}, - }} + entries := tinyGlossary() result, err := protectedValidator.Validate(context.Background(), validators.Request{ WorkingTranscript: working, - Glossary: glossary, + Glossary: entries, + ProtectedTerms: domain.Vocabulary(entries, nil), ModuleKey: "homophones", CandidateProposal: []proposals.EnrichedCorrectionProposal{ { diff --git a/internal/modules/spoken_word/module_test.go b/internal/modules/spoken_word/module_test.go index e6e9db4..458b980 100644 --- a/internal/modules/spoken_word/module_test.go +++ b/internal/modules/spoken_word/module_test.go @@ -8,8 +8,8 @@ import ( "strings" "testing" - ck "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" + domain "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation" @@ -48,10 +48,9 @@ func tinyTranscript() *schema.Transcript { }} } -func tinyGlossary() *schema.Glossary { - return &schema.Glossary{Entries: []schema.GlossaryEntry{ - {Name: "Jesters", Aliases: []string{"Jester"}, Category: "faction", Summary: "Guild members", Plural: "Jesters"}, - }} +func tinyGlossary() []domain.Entry { + plural := "Jesters" + return []domain.Entry{{Collection: "party", ID: "jesters", Name: "Jesters", Aliases: []string{"Jester"}, Category: "faction", Description: "Guild members", Plural: &plural}} } func TestSpokenWordModuleReplacementPolicy(t *testing.T) { @@ -100,11 +99,6 @@ func TestSpokenWordModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T }, } t.Setenv("MODULE_SECRET", secret) - loaded, loadErr := config.Load(ck.Bytes("config.yml", []byte("version: 2\nbackends:\n offline:\n endpoint: https://offline.invalid/v1\n credential: MODULE_SECRET\nllm:\n proposal_profile: base\n")), "") - if loadErr != nil { - t.Fatal(loadErr) - } - cfg := loaded.Config m, err := New() if err != nil { t.Fatalf("New error: %v", err) @@ -112,7 +106,6 @@ func TestSpokenWordModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T diagDir := t.TempDir() out, err := m.Propose(context.Background(), contracts.ProposalRequest{ ExecutionContext: contracts.ExecutionContext{ - Config: &cfg, WorkingTranscript: tinyTranscript(), Glossary: tinyGlossary(), DiagnosticsDir: diagDir, @@ -173,10 +166,8 @@ func TestSpokenWordConfidenceThresholdIsUsed(t *testing.T) { t.Fatal("expected confidence_threshold validator") } - cfg := config.Config{} - cfg.Thresholds.SpokenWord = 0.95 result, err := thresholdValidator.Validate(context.Background(), validators.Request{ - Config: &cfg, + Settings: config.Stage{ConfidenceThreshold: 0.95}, ModuleKey: "spoken_word", CandidateProposal: []proposals.EnrichedCorrectionProposal{ { @@ -223,12 +214,11 @@ func TestSpokenWordProtectedGlossaryTermBehaviorRejectsUnsafeCorrection(t *testi working := &schema.Transcript{Segments: []schema.Segment{ {ID: 1, Speaker: "A", Start: 0, End: 1, Text: "Jesters are here."}, }} - glossary := &schema.Glossary{Entries: []schema.GlossaryEntry{ - {Name: "Jesters", Category: "faction", Summary: "Faction"}, - }} + entries := tinyGlossary() result, err := protectedValidator.Validate(context.Background(), validators.Request{ WorkingTranscript: working, - Glossary: glossary, + Glossary: entries, + ProtectedTerms: domain.Vocabulary(entries, nil), ModuleKey: "spoken_word", CandidateProposal: []proposals.EnrichedCorrectionProposal{ { diff --git a/internal/validators/protected_terms/validator_test.go b/internal/validators/protected_terms/validator_test.go index e9f9552..f82bac0 100644 --- a/internal/validators/protected_terms/validator_test.go +++ b/internal/validators/protected_terms/validator_test.go @@ -4,7 +4,7 @@ import ( "context" "testing" - "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" + "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators" @@ -27,12 +27,8 @@ func proposal(index int, moduleKey, original, corrected string) proposals.Enrich } } -func tinyGlossary() *schema.Glossary { - return &schema.Glossary{ - Entries: []schema.GlossaryEntry{ - {Name: "Jesters", Category: "faction", Summary: "Faction"}, - }, - } +func tinyGlossary() []glossary.Entry { + return []glossary.Entry{{Collection: "party", ID: "jesters", Name: "Jesters", Category: "faction", Description: "Faction"}} } func TestNewReportsStableKeyAndDeterministicClass(t *testing.T) { @@ -83,8 +79,9 @@ func TestNewRejectsProtectedTermChangeOutsideGlossaryStage(t *testing.T) { } req := contracts.ValidationRequest{ - ModuleKey: "homophones", - Glossary: tinyGlossary(), + ModuleKey: "homophones", + Glossary: tinyGlossary(), + ProtectedTerms: glossary.Vocabulary(tinyGlossary(), nil), CandidateProposal: []proposals.EnrichedCorrectionProposal{ proposal(0, "homophones", "Jesters", "Gestures"), }, @@ -105,9 +102,9 @@ func TestNewRejectsProtectedTermChangeOutsideGlossaryStage(t *testing.T) { } func TestNewGlossaryStageRejectsProtectedTermChange(t *testing.T) { - v, err := NewGlossaryStage() + v, err := New() if err != nil { - t.Fatalf("NewGlossaryStage: %v", err) + t.Fatalf("New: %v", err) } if v.Name() != Key { t.Fatalf("expected key %q, got %q", Key, v.Name()) @@ -117,8 +114,9 @@ func TestNewGlossaryStageRejectsProtectedTermChange(t *testing.T) { } req := contracts.ValidationRequest{ - ModuleKey: "glossary", - Glossary: tinyGlossary(), + ModuleKey: "glossary", + Glossary: tinyGlossary(), + ProtectedTerms: glossary.Vocabulary(tinyGlossary(), nil), CandidateProposal: []proposals.EnrichedCorrectionProposal{ proposal(0, "glossary", "Jesters", "Gestures"), }, @@ -139,14 +137,15 @@ func TestNewGlossaryStageRejectsProtectedTermChange(t *testing.T) { } func TestNewGlossaryStageAllowsProtectedTermInsertion(t *testing.T) { - v, err := NewGlossaryStage() + v, err := New() if err != nil { - t.Fatalf("NewGlossaryStage: %v", err) + t.Fatalf("New: %v", err) } req := contracts.ValidationRequest{ - ModuleKey: "glossary", - Glossary: tinyGlossary(), + ModuleKey: "glossary", + Glossary: tinyGlossary(), + ProtectedTerms: glossary.Vocabulary(tinyGlossary(), nil), CandidateProposal: []proposals.EnrichedCorrectionProposal{ proposal(0, "glossary", "gestures", "Jesters"), }, diff --git a/internal/validators/registry_test.go b/internal/validators/registry_test.go index 22db519..b2826f5 100644 --- a/internal/validators/registry_test.go +++ b/internal/validators/registry_test.go @@ -4,8 +4,8 @@ import ( "context" "testing" + "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary" "gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog" - "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/v3/internal/validators/confidence_threshold" @@ -121,9 +121,7 @@ func TestRegistryProtectedTermsUsesNonGlossaryStageBehavior(t *testing.T) { req := contracts.ValidationRequest{ ModuleKey: "glossary", - Glossary: &schema.Glossary{ - Entries: []schema.GlossaryEntry{{Name: "Jesters", Category: "faction", Summary: "Protected"}}, - }, + Glossary: []glossary.Entry{{Collection: "party", ID: "jesters", Name: "Jesters", Category: "faction", Description: "Protected"}}, CandidateProposal: []proposals.EnrichedCorrectionProposal{ { CorrectionProposal: proposals.CorrectionProposal{ @@ -169,10 +167,9 @@ func TestResolveBuiltInChainUsesGlossaryStageProtectedTermsBehavior(t *testing.T } req := contracts.ValidationRequest{ - ModuleKey: "glossary", - Glossary: &schema.Glossary{ - Entries: []schema.GlossaryEntry{{Name: "Jesters", Category: "faction", Summary: "Protected"}}, - }, + ModuleKey: "glossary", + ProtectedTerms: glossary.Vocabulary([]glossary.Entry{{Collection: "party", ID: "jesters", Name: "Jesters"}}, nil), + Glossary: []glossary.Entry{{Collection: "party", ID: "jesters", Name: "Jesters", Category: "faction", Description: "Protected"}}, CandidateProposal: []proposals.EnrichedCorrectionProposal{ { CorrectionProposal: proposals.CorrectionProposal{