diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 1503f919..e8926235 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1780,6 +1780,36 @@ them; live OpenRouter/local services are not default validation requirements. ## Stage 37: Verify Assembled Recovery And Native Command Workflows +**Status:** Complete. Added two focused native CLI recovery contracts using the +maintained ordered D&D composition, existing provider fake, real domain modules, +typed generated references, filesystem cache, and capture. They include semantic +normalization and the real combat-semantics LLM validator on one limited backend +with one shared worker. A final-turn corrected scene is stored under the initial +prepared message hash; a downstream failure preserves accepted predecessors and +the chunk plan, and fixed reruns execute only the previously failed work. Full +hits retain capture without native executions or historical accounting. Native +failure causes, session/model/backend selections, unknown usage, terminal group +coverage, domain payloads, and source identity remain intact. + +A failed permissive NPC refresh emits one qualified fallback, forces actual +generated-reference consumers to execute without reuse, leaves unrelated lanes +cached, and preserves earlier accepted files. Restoring ordinary mode reuses those +files without provider work. Operator-schema tests now execute the same prepared +handle and verify the custom provider constraint and native validation for inline, +source-relative, and external-root schemas; missing replacement roots do not fall +back to embedded schemas. No compatibility/configuration translation was added. + +Reviewed and retained the existing focused producer/assessment/cache/capture, +worker/admission, command/preflight, and report/subprocess coverage for structural +failure with nil error, result-less/undelivered responses, full ordered guidance, +final-turn assessment, unchanged candidates, native overrides, cancellation, +stream/delivery outcomes, and no duplicate debug preparation. Explicit test repair +budgets use complete native output contracts, without pinning upstream defaults. +All four prescribed package suites and vet pass, as do repeated focused CLI and +shared observer/coordinator/backend race checks and `git diff --check`. No real +credentials, external inference, or model-quality claim is involved. Stage 46 +still owns final whole-roadmap verification after documentation updates. + **Goal:** prove the cross-boundary behavior most likely to regress in this cutover. **Inspect/change:** CLI assembled/command/cache contracts, module integration diff --git a/internal/cli/native_schema_load_test.go b/internal/cli/native_schema_load_test.go index 8e66aa47..435cdeaf 100644 --- a/internal/cli/native_schema_load_test.go +++ b/internal/cli/native_schema_load_test.go @@ -2,6 +2,7 @@ package cli import ( "context" + "errors" "path/filepath" "testing" @@ -14,7 +15,7 @@ func TestNativeLoadPreparesOperatorSchemaSelections(t *testing.T) { name, schema, root string missing bool }{ - {"inline", "schema: {type: object}", "", false}, + {"inline", `schema: {type: object, required: [operator_value], properties: {operator_value: {const: selected}}, additionalProperties: false}`, "", false}, {"source file", "schema_file: response.json", "", false}, {"external root", "schema_path: response.json", "schema_root: external-schemas", false}, {"replaced root missing embedded schema", "schema_path: dnd/scenes/schemas/dnd_scenes_llm.v1.json", "schema_root: external-schemas", true}, @@ -23,10 +24,18 @@ func TestNativeLoadPreparesOperatorSchemaSelections(t *testing.T) { dir := t.TempDir() main := filepath.Join(dir, "config.yml") writeConfigKitFile(t, main, "version: 6\n") - writeConfigKitFile(t, filepath.Join(dir, "response.json"), `{"type":"object"}`) - writeConfigKitFile(t, filepath.Join(dir, "external-schemas", "response.json"), `{"type":"object"}`) + schema := `{"type":"object","required":["operator_value"],"properties":{"operator_value":{"const":"selected"}},"additionalProperties":false}` + writeConfigKitFile(t, filepath.Join(dir, "response.json"), schema) + writeConfigKitFile(t, filepath.Join(dir, "external-schemas", "response.json"), schema) writeConfigKitFile(t, filepath.Join(dir, "prompts.yml"), "collections:\n notarius:\n "+test.root+"\n prompts:\n - id: dnd.scenes\n default_profile: notarius/dnd-extraction\n messages: [{role: user, content: operator-message}]\n output:\n format: json\n validation_mode: json_schema\n "+test.schema+"\n") - loaders := &nativeLoaders{} + calls := 0 + loaders := &nativeLoaders{engineOptions: []promptkit.Option{promptkit.WithLLMClient(captureClient(func(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + calls++ + if !productionSchemaHasProperty(request, "operator_value") { + return nil, errors.New("operator schema did not reach provider preparation") + } + return &promptkit.GenerateResponse{Content: `{"operator_value":"selected"}`}, nil + }))}} engine, err := loaders.engine(context.Background(), nativeSelection(main)) if err != nil { t.Fatal(err) @@ -48,6 +57,10 @@ func TestNativeLoadPreparesOperatorSchemaSelections(t *testing.T) { if len(details.Messages) != 1 || details.Messages[0].Content != "operator-message" || details.StructuredOutput == nil { t.Fatalf("replacement contract = %#v", details) } + result, err := engine.RunPrepared(context.Background(), prepared) + if err != nil || result == nil || !result.Validation.IsValid || calls != 1 { + t.Fatalf("operator contract execution=%#v error=%v calls=%d", result, err, calls) + } // Replacing one identified prompt must not discard unrelated embedded prompts. if _, err := engine.InspectPrompt(context.Background(), "notarius/dnd.spells"); err != nil { t.Fatal(err) diff --git a/internal/cli/recovery_contract_test.go b/internal/cli/recovery_contract_test.go new file mode 100644 index 00000000..311ae184 --- /dev/null +++ b/internal/cli/recovery_contract_test.go @@ -0,0 +1,322 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "strings" + "sync" + "testing" + "time" + + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + "go.maximumdirect.net/commandkit" + "go.maximumdirect.net/promptkit" +) + +// Use the maintained ordered domain composition and existing provider fake. +// Only external failures and one rejected scene candidate are controlled here. +type recoveryClient struct { + base enemyEventLLMClient + mu sync.Mutex + requests []promptkit.GenerateRequest + failEvents, failNPC, correctScene bool + failure error +} + +func (client *recoveryClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + role := enemyEventRequestRole(request) + if productionSchemaHasProperty(request, "classification") { + role = "classification" + } + client.mu.Lock() + client.requests = append(client.requests, request) + failEvents, failNPC := client.failEvents, client.failNPC + transcript, _ := productionJSONInput(request, "units") + combat := strings.Contains(string(transcript), "Roll initiative") + incorrectScene := role == "kind" && combat && client.correctScene + if incorrectScene { + client.correctScene = false + } + client.mu.Unlock() + if role == "events" && failEvents || role == "npcs" && combat && failNPC { + return nil, client.failure + } + if incorrectScene { + return &promptkit.GenerateResponse{Content: `{"kind":"narrative","title":"Arrival","summary":"An initially missed combat scene."}`}, nil + } + if role == "classification" { + classification := "non_combat" + if combat { + classification = "combat" + } + return &promptkit.GenerateResponse{Content: fmt.Sprintf(`{"classification":%q,"explanation":"Classification follows the supplied scene transcript."}`, classification)}, nil + } + // Empty generated registries remain valid current-run context. Do not invent + // identities or events merely to fill the dependent lanes in the fallback run. + if role == "npc-occurrences" || role == "combat_turns" || role == "events" { + if content, err := productionJSONInput(request, "npcs"); err == nil { + var registry struct { + NPCs []json.RawMessage `json:"npcs"` + } + if err := json.Unmarshal(content, ®istry); err != nil { + return nil, err + } + if len(registry.NPCs) == 0 { + property := "occurrences" + if role == "combat_turns" { + property = "combat_turns" + } + if role == "events" { + property = "events" + } + return &promptkit.GenerateResponse{Content: fmt.Sprintf(`{%q:[]}`, property)}, nil + } + } + } + return client.base.Generate(ctx, request) +} + +func (client *recoveryClient) snapshot() (map[string]int, []promptkit.GenerateRequest) { + client.mu.Lock() + defer client.mu.Unlock() + counts := map[string]int{} + for _, request := range client.requests { + role := enemyEventRequestRole(request) + if productionSchemaHasProperty(request, "classification") { + role = "classification" + } + counts[role]++ + } + return counts, append([]promptkit.GenerateRequest(nil), client.requests...) +} + +type recoveryFixture struct { + config, cache, output, debug string + client *recoveryClient + options Options +} + +func newRecoveryFixture(t *testing.T) *recoveryFixture { + t.Helper() + root := t.TempDir() + for _, name := range []string{"prompts.yml", "transcript.json", "party.txt", "glossary.txt", "spell-catalog.json", "pipelines/dnd-session.yml", "pipelines/dnd-session.describe-session.yml"} { + content, err := os.ReadFile(repositoryPath("examples", "dnd-complete", name)) + if err != nil { + t.Fatal(err) + } + writeConfigKitFile(t, filepath.Join(root, name), string(content)) + } + description := filepath.Join(root, "pipelines", "dnd-session.describe-session.yml") + content, err := os.ReadFile(description) + if err != nil { + t.Fatal(err) + } + configured := replaceRequiredOnce(t, string(content), " extract:\n module: dnd/scene-descriptions\n", ` extract: + module: dnd/scene-descriptions + validators: + - module: generic/valid_json + - module: extract/dnd/scene-descriptions/shape + - module: extract/dnd/scene-descriptions/source_refs + - module: generic/valid_json_schema + - module: extract/dnd/scene-descriptions/source_relatedness + - module: extract/dnd/scene-descriptions/combat_semantics +`) + writeConfigKitFile(t, description, configured) + fixture := &recoveryFixture{config: filepath.Join(root, "config.yml"), cache: filepath.Join(root, "cache"), output: filepath.Join(root, "output"), debug: filepath.Join(root, "debug"), client: &recoveryClient{failure: errors.New("controlled downstream provider failure")}} + writeConfigKitFile(t, fixture.config, fmt.Sprintf(`version: 6 +config: + imports: + - path: pipelines/dnd-session.yml + required: true +output: {directory: %q} +cache_directory: %q +debug: {directory: %q} +concurrency: {workers: 1} +promptkit: + backends: + - id: fixture + endpoint: https://example.invalid/v1 + concurrency_limit: 1 + collections: + notarius: + profiles: + - id: dnd-extraction + backend: fixture + model: recovery-model + api_key_required: false + timeout_seconds: 2 + - id: dnd-extraction-example + extends: {id: dnd-extraction} +`, fixture.output, fixture.cache, fixture.debug)) + fixture.options = productionRunOptions(t, fixture.client) + run := 0 + fixture.options.RunIDGenerator = func(time.Time) (string, error) { run++; return fmt.Sprintf("recovery-%d", run), nil } + return fixture +} + +func (fixture *recoveryFixture) run(t *testing.T) (commandkit.Outcome, runResult, string) { + t.Helper() + args := []string{"run", "dnd-session", "--config", fixture.config, "--input-file", filepath.Join(filepath.Dir(fixture.config), "transcript.json"), "--debug", "--progress=false", "--format=json", "--llm-session-id=recovery-session", "--llm-transient-retries=0", + "--set", "/pipelines/dnd-session/max_additional_turns=1", + } + app, err := NewApplication("development", fixture.options) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + var stdout, stderr bytes.Buffer + outcome := app.Execute(ctx, args, commandkit.Streams{Out: &stdout, Err: &stderr}) + var report runResult + if outcome.Status == 0 { + if err := json.Unmarshal(stdout.Bytes(), &report); err != nil { + t.Fatalf("completion=%q: %v", stdout.String(), err) + } + } else if stdout.Len() != 0 { + t.Fatalf("failed operation emitted success output: %q", stdout.String()) + } + return outcome, report, stderr.String() +} + +func TestAssembledRecoveryReusesCorrectedPredecessorsAndChunkPlan(t *testing.T) { + fixture := newRecoveryFixture(t) + fixture.client.failEvents, fixture.client.correctScene = true, true + outcome, _, diagnostics := fixture.run(t) + if outcome.Status != 1 || !errors.Is(outcome.Cause, fixture.client.failure) { + t.Fatalf("downstream failure=%#v diagnostics=%q", outcome, diagnostics) + } + assertAbsent(t, fixture.output) + firstDebug := filepath.Join(fixture.debug, "recovery-1") + assertFile(t, filepath.Join(firstDebug, "outcome.json")) + counts, requests := fixture.client.snapshot() + if counts["scenes"] != 1 || counts["kind"] != 3 || counts["classification"] != 3 || counts["duplicate_groups"] == 0 || counts["events"] != 1 { + t.Fatalf("ordered native roles/correction=%v", counts) + } + for _, request := range requests { + if request.Prompt.SessionID != "recovery-session" || request.Target.Model != "recovery-model" || request.Target.BackendID != "fixture" { + t.Fatalf("lost shared native selection: %#v", request) + } + } + summary := readProductionJSON[promptkit.ExecutionGroupSummary](t, filepath.Join(firstDebug, "executions.json")) + if len(summary.Members) != len(requests) || summary.Attempts != uint64(len(requests)) || !summary.TerminalCoverageComplete || summary.Usage.TotalTokens.Known { + t.Fatalf("failed-run native facts lost or duplicated: %#v", summary) + } + // Locate only the initial scene request's native hash in available capture; + // assert domain bytes at that key, not a persisted execution ledger. + var correctedKey string + for name, content := range readTree(t, firstDebug) { + if !strings.HasSuffix(name, "/prepared.json") { + continue + } + var details promptkit.PreparedRun + if json.Unmarshal(content, &details) != nil || details.PromptID != "notarius/dnd.scene_descriptions" { + continue + } + for _, message := range details.Messages { + if strings.Contains(message.Content, "Roll initiative") { + correctedKey = details.MessageContentHash + } + } + } + if correctedKey == "" { + t.Fatal("no prepared initial combat scene evidence") + } + corrected := readProductionJSON[dnd.SceneDescriptionList](t, filepath.Join(fixture.cache, correctedKey+".json")) + if len(corrected.Scenes) != 1 || corrected.Scenes[0].Kind != dnd.SceneKindCombat { + t.Fatalf("cache did not retain the corrected domain candidate: %#v", corrected) + } + fixture.client.failEvents = false + outcome, report, diagnostics := fixture.run(t) + if outcome.Status != 0 || report.FallbackCount != 0 || report.ArtifactCount != 12 { + t.Fatalf("recovery=%#v report=%#v diagnostics=%q", outcome, report, diagnostics) + } + after, _ := fixture.client.snapshot() + for role, count := range counts { + want := count + if role == "events" { + want++ + } + if after[role] != want { + t.Fatalf("recovery reran accepted %s: %v -> %v", role, counts, after) + } + } + artifact := readProductionJSON[dnd.EnemyEventList](t, filepath.Join(report.OutputDirectory, "lanes", "enemy-events.json")) + if len(artifact.Events) != 1 || artifact.Events[0].SourceRefs[0].SourceID != "session-ravenfall" { + t.Fatalf("recovered domain evidence=%#v", artifact) + } + stable := readTree(t, fixture.cache) + outcome, hitReport, diagnostics := fixture.run(t) + if outcome.Status != 0 || hitReport.ArtifactCount != 12 { + t.Fatalf("cached run=%#v diagnostics=%q", outcome, diagnostics) + } + last, _ := fixture.client.snapshot() + if !reflect.DeepEqual(last, after) || !sameFiles(stable, readTree(t, fixture.cache)) { + t.Fatalf("full hit executed or rewrote acceptance: %v -> %v", after, last) + } + for name := range readTree(t, hitReport.DebugDirectory) { + if strings.Contains(name, "/native/") && strings.HasSuffix(name, "/result.json") { + t.Fatalf("full hit fabricated native execution: %s", name) + } + } + hitSummary := readProductionJSON[promptkit.ExecutionGroupSummary](t, filepath.Join(hitReport.DebugDirectory, "executions.json")) + if len(hitSummary.Members) != 0 || !hitSummary.TerminalCoverageComplete { + t.Fatalf("full hit fabricated historical execution totals: %#v", hitSummary) + } +} + +func TestAssembledFallbackSkipsDependentReuseAndPreservesFailedRefresh(t *testing.T) { + fixture := newRecoveryFixture(t) + outcome, _, diagnostics := fixture.run(t) + if outcome.Status != 0 { + t.Fatalf("initial run=%#v diagnostics=%q", outcome, diagnostics) + } + before, _ := fixture.client.snapshot() + accepted := readTree(t, fixture.cache) + fixture.client.failNPC = true + description := filepath.Join(filepath.Dir(fixture.config), "pipelines", "dnd-session.describe-session.yml") + content, err := os.ReadFile(description) + if err != nil { + t.Fatal(err) + } + writeConfigKitFile(t, description, replaceRequiredOnce(t, string(content), " extract:\n module: dnd/npc-registry\n", " extract:\n module: dnd/npc-registry\n cache: refresh\n mode: permissive\n")) + outcome, report, diagnostics := fixture.run(t) + if outcome.Status != 0 || report.FallbackCount != 1 { + t.Fatalf("fallback=%#v report=%#v diagnostics=%q", outcome, report, diagnostics) + } + after, _ := fixture.client.snapshot() + for _, role := range []string{"scenes", "kind", "classification", "locations", "items", "location-occurrences", "duplicate_groups"} { + if after[role] != before[role] { + t.Fatalf("unaffected %s reran: %v -> %v", role, before, after) + } + } + for _, role := range []string{"npcs", "spell_casts", "npc-occurrences", "combat_turns", "events"} { + if after[role] <= before[role] { + t.Fatalf("affected %s reused earlier acceptance: %v -> %v", role, before, after) + } + } + if !sameFiles(accepted, readTree(t, fixture.cache)) { + t.Fatal("failed refresh/fallback replaced accepted cache data") + } + registry := readProductionJSON[dnd.NPCRegistry](t, filepath.Join(report.OutputDirectory, "lanes", "npc-registry.json")) + if registry.NPCs == nil || len(registry.NPCs) != 0 { + t.Fatalf("fallback registry=%#v", registry) + } + fixture.client.failNPC = false + writeConfigKitFile(t, description, string(content)) + outcome, report, diagnostics = fixture.run(t) + if outcome.Status != 0 || report.FallbackCount != 0 { + t.Fatalf("restored acceptance=%#v report=%#v diagnostics=%q", outcome, report, diagnostics) + } + last, _ := fixture.client.snapshot() + if !reflect.DeepEqual(last, after) { + t.Fatalf("failed refresh lost earlier reusable results: %v -> %v", after, last) + } +} + +var _ promptkit.LLMClient = (*recoveryClient)(nil)