From 2c1d727232d24a088c61c520efd5dcb477bf2404 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 22 Nov 2026 01:03:06 +0000 Subject: [PATCH] Migrate NPC and location occurrences to native correction and explicit empty fallbacks --- .../prompts/correction.md | 7 + .../location-occurrences/prompts/prompt.yaml | 20 +- .../dnd/npc-occurrences/prompts/correction.md | 7 + .../dnd/npc-occurrences/prompts/prompt.yaml | 20 +- assets/occurrence_extraction_test.go | 140 +++++++++++++ docs/roadmap/implementation.md | 22 ++ .../extract/locationoccurrences/extractor.go | 129 +++++------- .../locationoccurrences/extractor_test.go | 63 ++---- .../native_execution_test.go | 162 +++++++++++++++ .../locationoccurrences/prompt_assets.go | 75 ------- .../locationoccurrences/prompt_assets_test.go | 53 ----- .../dnd/extract/locationoccurrences/schema.go | 25 --- .../locationoccurrences/schema_test.go | 97 --------- .../locationoccurrences/test_helpers_test.go | 116 ++++++++--- .../dnd/extract/npcoccurrences/extractor.go | 152 +++++--------- .../extract/npcoccurrences/extractor_test.go | 195 +++--------------- .../npcoccurrences/native_execution_test.go | 161 +++++++++++++++ .../extract/npcoccurrences/prompt_assets.go | 75 ------- .../npcoccurrences/prompt_assets_test.go | 70 ------- .../dnd/extract/npcoccurrences/schema.go | 25 --- .../dnd/extract/npcoccurrences/schema_test.go | 127 ------------ .../npcoccurrences/test_helpers_test.go | 184 +++++++++++++++++ .../shape/validator_test.go | 2 +- 23 files changed, 948 insertions(+), 979 deletions(-) create mode 100644 assets/dnd/location-occurrences/prompts/correction.md create mode 100644 assets/dnd/npc-occurrences/prompts/correction.md create mode 100644 assets/occurrence_extraction_test.go create mode 100644 internal/modules/dnd/extract/locationoccurrences/native_execution_test.go delete mode 100644 internal/modules/dnd/extract/locationoccurrences/prompt_assets.go delete mode 100644 internal/modules/dnd/extract/locationoccurrences/prompt_assets_test.go delete mode 100644 internal/modules/dnd/extract/locationoccurrences/schema.go delete mode 100644 internal/modules/dnd/extract/locationoccurrences/schema_test.go create mode 100644 internal/modules/dnd/extract/npcoccurrences/native_execution_test.go delete mode 100644 internal/modules/dnd/extract/npcoccurrences/prompt_assets.go delete mode 100644 internal/modules/dnd/extract/npcoccurrences/prompt_assets_test.go delete mode 100644 internal/modules/dnd/extract/npcoccurrences/schema.go delete mode 100644 internal/modules/dnd/extract/npcoccurrences/schema_test.go create mode 100644 internal/modules/dnd/extract/npcoccurrences/test_helpers_test.go diff --git a/assets/dnd/location-occurrences/prompts/correction.md b/assets/dnd/location-occurrences/prompts/correction.md new file mode 100644 index 00000000..95cc48fd --- /dev/null +++ b/assets/dnd/location-occurrences/prompts/correction.md @@ -0,0 +1,7 @@ +Revise the previous location occurrence list using these findings: + +{{ input "guidance" }} + +Return a complete replacement occurrence list using the original transcript and +registry context. Keep identity-grounding context separate from current-transcript +evidence, and return only JSON matching the original response contract. diff --git a/assets/dnd/location-occurrences/prompts/prompt.yaml b/assets/dnd/location-occurrences/prompts/prompt.yaml index b7476aff..a4d85f81 100644 --- a/assets/dnd/location-occurrences/prompts/prompt.yaml +++ b/assets/dnd/location-occurrences/prompts/prompt.yaml @@ -20,7 +20,7 @@ prompts: messages: - role: system content_file: ../../shared/prompts/common-dnd-system.md - - role: user + - role: system content_file: ../../shared/prompts/common-dnd-identity.md - role: user content_file: ../../shared/prompts/common-dnd-references.md @@ -30,11 +30,11 @@ prompts: content_file: ../../shared/prompts/common-dnd-transcript-chunk.md cache_control: type: ephemeral - - role: user + - role: system content_file: ../../shared/prompts/common-dnd-extraction-evidence.md - role: user content_file: ./location-registry.md - - role: user + - role: system content_file: ./instructions.md cache_control: type: ephemeral @@ -42,3 +42,17 @@ prompts: format: json validation_mode: json_schema schema_path: dnd/location-occurrences/schemas/dnd_location_occurrences_llm.v1.json + + - id: dnd.location_occurrences.repair + default_profile: notarius/dnd-extraction + inputs: + - name: guidance + required: true + content_type: text/plain + messages: + - role: user + content_file: ./correction.md + output: + format: json + validation_mode: json_schema + schema_path: dnd/location-occurrences/schemas/dnd_location_occurrences_llm.v1.json diff --git a/assets/dnd/npc-occurrences/prompts/correction.md b/assets/dnd/npc-occurrences/prompts/correction.md new file mode 100644 index 00000000..d18ad373 --- /dev/null +++ b/assets/dnd/npc-occurrences/prompts/correction.md @@ -0,0 +1,7 @@ +Revise the previous NPC occurrence list using these findings: + +{{ input "guidance" }} + +Return a complete replacement occurrence list using the original transcript and +registry context. Keep identity-grounding context separate from current-transcript +evidence, and return only JSON matching the original response contract. diff --git a/assets/dnd/npc-occurrences/prompts/prompt.yaml b/assets/dnd/npc-occurrences/prompts/prompt.yaml index 27695658..8beaf21d 100644 --- a/assets/dnd/npc-occurrences/prompts/prompt.yaml +++ b/assets/dnd/npc-occurrences/prompts/prompt.yaml @@ -20,7 +20,7 @@ prompts: messages: - role: system content_file: ../../shared/prompts/common-dnd-system.md - - role: user + - role: system content_file: ../../shared/prompts/common-dnd-identity.md - role: user content_file: ../../shared/prompts/common-dnd-references.md @@ -30,11 +30,11 @@ prompts: content_file: ../../shared/prompts/common-dnd-transcript-chunk.md cache_control: type: ephemeral - - role: user + - role: system content_file: ../../shared/prompts/common-dnd-extraction-evidence.md - role: user content_file: ../../shared/prompts/common-dnd-npc-registry.md - - role: user + - role: system content_file: ./instructions.md cache_control: type: ephemeral @@ -42,3 +42,17 @@ prompts: format: json validation_mode: json_schema schema_path: dnd/npc-occurrences/schemas/dnd_npc_occurrences_llm.v1.json + + - id: dnd.npc_occurrences.repair + default_profile: notarius/dnd-extraction + inputs: + - name: guidance + required: true + content_type: text/plain + messages: + - role: user + content_file: ./correction.md + output: + format: json + validation_mode: json_schema + schema_path: dnd/npc-occurrences/schemas/dnd_npc_occurrences_llm.v1.json diff --git a/assets/occurrence_extraction_test.go b/assets/occurrence_extraction_test.go new file mode 100644 index 00000000..e22b511c --- /dev/null +++ b/assets/occurrence_extraction_test.go @@ -0,0 +1,140 @@ +package assets_test + +import ( + "context" + "encoding/json" + "io/fs" + "reflect" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/assets" + "go.maximumdirect.net/promptkit" +) + +func TestOccurrencePromptPairsPreserveRegistryAndTranscriptBoundaries(t *testing.T) { + _, engine := loadNativeAssets(t) + for _, test := range []struct { + family, registry string + rules []string + }{ + {"npc", `{"npcs":[{"name":"occurrence-registry-name"}]}`, []string{"Use the supplied canonical NPC", "noncombat_presence", "combat_opponent"}}, + {"location", `{"locations":[{"name":"occurrence-registry-name","registry_refs":[{"start_unit_id":7,"end_unit_id":7}],"context":[{"unit_id":7,"text":"registry-context"}]}]}`, + []string{"context supports that coreference", "must not create a registry location", "provenance must never replace current-chunk evidence", "complete ordered"}}, + } { + t.Run(test.family, func(t *testing.T) { + id := "notarius/dnd." + test.family + "_occurrences" + initial, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{ + PromptID: id, APIKey: "test-only", + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline(`{"units":[{"sentinel":"occurrence-transcript"}]}`), + test.family + "_registry": promptkit.Inline(test.registry), + "players": promptkit.Inline("occurrence-players"), "party": promptkit.Inline("occurrence-party"), "glossary": promptkit.Inline("occurrence-glossary"), + }, + }) + if err != nil { + t.Fatal(err) + } + defer initial.Discard() + correction, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{ + PromptID: id + ".repair", APIKey: "test-only", + Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline("Correct the unsupported occurrence.")}, + }) + if err != nil { + t.Fatal(err) + } + defer correction.Discard() + first, next := initial.Details(), correction.Details() + if first.SelectedProfileID != "notarius/dnd-extraction" || first.SelectedProfileID != next.SelectedProfileID || + first.StructuredOutput == nil || next.StructuredOutput == nil || !reflect.DeepEqual(first.OutputContract, next.OutputContract) { + t.Fatal("occurrence native pair lost schema/defaults") + } + var system, all string + for _, message := range first.Messages { + all += message.Content + if message.Role == "system" { + system += message.Content + } + } + for _, rule := range append(test.rules, "Transcript units are the only evidence") { + if !strings.Contains(strings.Join(strings.Fields(system), " "), rule) { + t.Fatalf("missing enduring rule %q", rule) + } + } + for _, content := range []string{"occurrence-registry-name", "occurrence-transcript", "occurrence-players", "occurrence-party", "occurrence-glossary"} { + if strings.Count(all, content) != 1 { + t.Fatalf("input not rendered once: %q", content) + } + } + if len(next.Messages) != 1 || next.Messages[0].Role != "user" || !strings.Contains(next.Messages[0].Content, "Correct the unsupported occurrence.") || + strings.Contains(next.Messages[0].Content, "occurrence-registry-name") { + t.Fatal("correction repeated fixed inputs") + } + }) + } +} + +func TestOccurrenceSchemasRetainSupportedKindsAndPrivateSelectors(t *testing.T) { + for _, test := range []struct { + family string + kinds []string + }{ + {"npc", []string{"mentioned", "noncombat_presence", "dialogue", "combat_ally", "combat_opponent", "other"}}, + {"location", []string{"visited", "planned", "recalled", "mentioned"}}, + } { + t.Run(test.family, func(t *testing.T) { + schema, err := fs.ReadFile(assets.FS(), "dnd/"+test.family+"-occurrences/schemas/dnd_"+test.family+"_occurrences_llm.v1.json") + if err != nil { + t.Fatal(err) + } + record := func() map[string]any { + value := map[string]any{"name": "Canonical Name", "kind": test.kinds[0], "source_refs": []any{map[string]any{"start_unit_id": 1, "end_unit_id": 2}}} + if test.family == "location" { + value["registry_refs"] = []any{map[string]any{"start_unit_id": 1, "end_unit_id": 2}} + } + return value + } + check := func(value map[string]any, valid bool) { + t.Helper() + content, err := json.Marshal(map[string]any{"occurrences": []any{value}}) + if err != nil { + t.Fatal(err) + } + if err := validateJSONSchema(content, schema); (err == nil) != valid { + t.Fatalf("valid=%t response=%s error=%v", valid, content, err) + } + } + for _, kind := range test.kinds { + value := record() + value["kind"] = kind + check(value, true) + } + semantic := record() + semantic["name"] = "" + semantic["source_refs"] = []any{map[string]any{"start_unit_id": 0, "end_unit_id": -1}} + check(semantic, true) + for _, mutate := range []func(map[string]any){ + func(value map[string]any) { delete(value, "name") }, + func(value map[string]any) { value["kind"] = 1 }, + func(value map[string]any) { value["kind"] = "unsupported" }, + func(value map[string]any) { value[test.family+"_id"] = "assigned-later" }, + func(value map[string]any) { value["unexpected"] = true }, + func(value map[string]any) { + value["source_refs"].([]any)[0].(map[string]any)["source_id"] = "assigned-later" + }, + } { + value := record() + mutate(value) + check(value, false) + } + if test.family == "location" { + value := record() + delete(value, "registry_refs") + check(value, false) + value = record() + value["registry_refs"].([]any)[0].(map[string]any)["start_unit_id"] = 0 + check(value, false) + } + }) + } +} diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index f36aafb7..41c7c3af 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -858,6 +858,28 @@ from Stage 12. Retain meaningful per-family grounding and identity regressions. ## Stage 19: Migrate NPC And Location Occurrence Extraction +**Status:** Complete. Both occurrence producers now use fixed transcript/registry +materials, native initial/correction requests, and the shared typed producer +helper. Added same-schema/default-profile correction pairs and static prompt +dependencies; removed private completion, schema/asset registration, and +manifest/fingerprint metadata. Names-only NPC resolution and complete ordered +contextual location selectors retain deterministic durable identity, supported +kinds, source-order canonicalization, and current-transcript evidence. Unknown +or ambiguous selections supply actionable interpretation guidance before typed +assessment rather than guessing an identity or returning partial output. +Permissive fallbacks are codec-valid nonnil empty collections, explicitly +qualified and reuse-ineligible; accepted work inherits incoming ineligibility. +Retained domain regressions with native provider fakes and added fixed resolution/ +history, source correction, run override, strict/permissive, failed continuation, +structural-failure, and cancellation cases. Prompt-pair/schema tests run in the +content package. Asset and shape tests/vet/race checks, independent NPC response +model tests/race checks, focused pipeline tests, and `git diff --check` pass. +Updated the location-occurrence shape spec comparison for slice-bearing native +dependencies. Both full extractor packages remain blocked at setup by shared +legacy helpers importing old ConfigKit's removed YAML `plugin/limit`; their +native runtime tests are not claimed passing. Scheduled dependency cleanup and +final whole-tree verification must run those tests. + **Goal:** registry-consuming occurrence lanes retain contextual resolution. **Inspect/change:** `internal/modules/dnd/extract/{npcoccurrences,locationoccurrences}`, diff --git a/internal/modules/dnd/extract/locationoccurrences/extractor.go b/internal/modules/dnd/extract/locationoccurrences/extractor.go index 2f6afd1b..456c77a0 100644 --- a/internal/modules/dnd/extract/locationoccurrences/extractor.go +++ b/internal/modules/dnd/extract/locationoccurrences/extractor.go @@ -3,8 +3,12 @@ package locationoccurrences import ( "context" + "encoding/json" "fmt" "sort" + "strings" + + "go.maximumdirect.net/promptkit" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" @@ -13,10 +17,9 @@ import ( "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" ) -const ( - Key = "dnd/location-occurrences" - mappingPolicy = "dnd.location_occurrences.extract_mapping.v3" -) +const Key = "dnd/location-occurrences" +const PromptID = "notarius/dnd.location_occurrences" +const CorrectionPromptID = "notarius/dnd.location_occurrences.repair" const ( LocationRegistryReferenceSlot = locationregistry.ReferenceSlot @@ -48,21 +51,17 @@ func referenceSlots() []contracts.ReferenceSlot { } var _ contracts.Extractor[dnd.LocationOccurrenceList] = (*Extractor)(nil) -var _ contracts.ManifestMetadataProvider = (*Extractor)(nil) -var _ pipeline.CheckpointFingerprintProvider = (*Extractor)(nil) type Options struct{} type Extractor struct { - llm contracts.StructuredLLMClient - locationResolver *locationregistry.Resolver - promptSHA string - responseSchemaSHA string + engine *promptkit.Engine + locationResolver *locationregistry.Resolver } -func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) { - if llmClient == nil { - return nil, extractorErrorf("LLM client must not be nil") +func New(engine *promptkit.Engine, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) { + if engine == nil { + return nil, extractorErrorf("native engine must not be nil") } if len(references) > 1 { return nil, extractorErrorf("at most one reference set may be supplied") @@ -75,62 +74,19 @@ func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contr if err != nil { return nil, extractorErrorf("prepare location registry prompt input: %w", err) } - promptSHA, err := promptAssetMetadata() - if err != nil { - return nil, extractorErrorf("load prompt metadata: %w", err) - } - responseSchema, err := loadResponseSchema() - if err != nil { - return nil, extractorErrorf("load response schema: %w", err) - } - return &Extractor{llm: llmClient, locationResolver: locationResolver, promptSHA: promptSHA, responseSchemaSHA: responseSchema.SHA256}, nil + return &Extractor{engine: engine, locationResolver: locationResolver}, nil } func (e *Extractor) Key() string { return Key } func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot { return referenceSlots() } -func (e *Extractor) ManifestMetadata() map[string]any { - if e == nil { - return nil - } - metadata := map[string]any{ - "prompt_id": PromptID, - "prompt_version": SchemaVersion, - "prompt_sha256": e.promptSHA, - "mapping_policy": mappingPolicy, - "response_schema_key": string(ResponseSchemaKey), - "response_schema_id": ResponseSchemaID, - "response_schema_name": ResponseSchemaName, - "response_schema_version": SchemaVersion, - "response_schema_sha256": e.responseSchemaSHA, - } - seeded := e.locationResolver.Seeded() - if seeded.Bound() { - metadata["location_registry_digest"] = seeded.Digest() - metadata["location_count"] = seeded.Count() - } - return metadata -} - -func (e *Extractor) CheckpointFingerprints() []pipeline.CheckpointFingerprint { - if e == nil { - return nil - } - return []pipeline.CheckpointFingerprint{ - {Name: "prompt", Value: e.promptSHA}, - {Name: "response_schema", Value: e.responseSchemaSHA}, - {Name: "mapping_policy", Value: mappingPolicy}, - {Name: "location_registry", Value: e.locationResolver.Seeded().IdentityDigest()}, - } -} - -func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.LocationOccurrenceList], error) { +func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest[dnd.LocationOccurrenceList]) (contracts.TypedExtractionResult[dnd.LocationOccurrenceList], error) { if e == nil { return contracts.TypedExtractionResult[dnd.LocationOccurrenceList]{}, extractorErrorf("extractor must not be nil") } - if e.llm == nil { - return contracts.TypedExtractionResult[dnd.LocationOccurrenceList]{}, extractorErrorf("LLM client must not be nil") + if e.engine == nil { + return contracts.TypedExtractionResult[dnd.LocationOccurrenceList]{}, extractorErrorf("native engine must not be nil") } sourceInput, err := shared.PrepareChunkExtraction(ctx, req) if err != nil { @@ -148,33 +104,46 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRe return contracts.TypedExtractionResult[dnd.LocationOccurrenceList]{}, extractorErrorf("prepare location grounding: %w", err) } - var response extractionResponse - inputs := shared.PromptInputs(sourceInput, req.References) - inputs[LocationRegistryReferenceSlot] = grounding.PromptInput() - completion, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{ - StageName: Key, PromptID: PromptID, PromptVersion: SchemaVersion, - ProfileID: req.LLMProfile, SessionID: req.SessionID, StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts, - Correction: req.Correction, Inputs: inputs, - }, &response) - if err != nil { - return contracts.TypedExtractionResult[dnd.LocationOccurrenceList]{}, extractorErrorf("complete structured output: %w", err) + order := shared.NewSourceRefOrder(req.Source) + + sourceID := req.Source.ID + materials := shared.PromptInputs(sourceInput, req.References) + materials[LocationRegistryReferenceSlot] = grounding.PromptInput() + inputs := make(map[string]promptkit.ArtifactRef, len(materials)) + for name, material := range materials { + inputs[name] = promptkit.InlineWithURI(material.OriginURI, string(material.Content)) } - candidate, err := shared.ModelCandidateFromResponse(completion) + initial := promptkit.RunRequest{PromptID: PromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID, Inputs: inputs} + value, outcome, err := pipeline.RunProducer(ctx, e.engine, req.Operation, initial, + func(_ context.Context, result *promptkit.RunResult) (*dnd.LocationOccurrenceList, []string, error) { + var response extractionResponse + if err := json.Unmarshal([]byte(result.RawOutput), &response); err != nil { + return nil, nil, extractorErrorf("decode native output: %w", err) + } + value, err := canonicalOccurrenceList(response, order, sourceID, grounding) + if err != nil { + return nil, []string{"Resolve every occurrence to exactly one supplied location selector. Copy its canonical name and complete ordered registry_refs array; use an empty registry_refs array for a unique name. Omit unsupported or ambiguous selections. Keep occurrence evidence in source_refs separate from registry context, and return the complete replacement occurrence list."}, nil + } + return &value, nil, nil + }, + func(_ context.Context, guidance []string) (promptkit.RunRequest, error) { + return promptkit.RunRequest{PromptID: CorrectionPromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID, + Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline(strings.Join(guidance, "\n\n"))}}, nil + }, + func(context.Context, *dnd.LocationOccurrenceList) (dnd.LocationOccurrenceList, error) { + return dnd.LocationOccurrenceList{Occurrences: []dnd.LocationOccurrence{}}, nil + }) if err != nil { - return contracts.TypedExtractionResult[dnd.LocationOccurrenceList]{}, extractorErrorf("capture model candidate: %w", err) + return contracts.TypedExtractionResult[dnd.LocationOccurrenceList]{}, extractorErrorf("produce occurrences: %w", err) } - occurrences, err := canonicalOccurrenceList(response, shared.NewSourceRefOrder(req.Source), req.Source.ID, grounding) - if err != nil { - return contracts.TypedExtractionResult[dnd.LocationOccurrenceList]{}, extractorErrorf("resolve location grounding: %w", err) - } - return contracts.TypedExtractionResult[dnd.LocationOccurrenceList]{Value: occurrences, ModelCandidate: candidate}, nil + return contracts.TypedExtractionResult[dnd.LocationOccurrenceList]{Value: value, Outcome: outcome}, nil } func ModuleSpec() pipeline.ModuleSpec { return pipeline.ModuleSpec{ Key: Key, Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, - CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1, - Requires: append([]string(nil), requiredCapabilities...), Provides: append([]string(nil), providedCapabilities...), + Prompts: []string{PromptID, CorrectionPromptID}, + Requires: append([]string(nil), requiredCapabilities...), Provides: append([]string(nil), providedCapabilities...), ArtifactKind: dnd.LocationOccurrenceListKind, ReferenceSlots: referenceSlots(), } } @@ -185,7 +154,7 @@ func Register(registry *pipeline.ExtractorRegistry) error { if err != nil { return nil, err } - return New(request.Dependencies.LLM, options, request.References) + return New(request.Dependencies.Engine, options, request.References) }) } diff --git a/internal/modules/dnd/extract/locationoccurrences/extractor_test.go b/internal/modules/dnd/extract/locationoccurrences/extractor_test.go index c700e685..7fefab3e 100644 --- a/internal/modules/dnd/extract/locationoccurrences/extractor_test.go +++ b/internal/modules/dnd/extract/locationoccurrences/extractor_test.go @@ -2,7 +2,6 @@ package locationoccurrences import ( "context" - "encoding/json" "errors" "reflect" "strings" @@ -70,22 +69,19 @@ func TestExtractResolvesContextualSelectorsAndUsesCurrentTranscriptEvidenceOnly( if occurrence := result.Value.Occurrences[0]; occurrence.LocationID != second.ID || occurrence.Name != second.Name || occurrence.SourceRefs[0].SourceID != req.Source.ID { t.Fatalf("occurrence = %#v", occurrence) } - input := client.requests[0].Inputs[LocationRegistryReferenceSlot] - if input.Name != LocationRegistryReferenceSlot || !strings.Contains(string(input.Content), `"registry_refs":[{"start_unit_id":20,"end_unit_id":20}]`) { - t.Fatalf("location prompt input = %#v", input) + input := renderedInput(t, client.requests[0], LocationRegistryReferenceSlot) + if !strings.Contains(string(input), `"registry_refs":[{"start_unit_id":20,"end_unit_id":20}]`) { + t.Fatalf("contextual registry=%s", input) } for _, forbidden := range []string{"source_refs", "source_id", first.ID, second.ID} { - if strings.Contains(string(input.Content), forbidden) { - t.Fatalf("location prompt leaked %q: %s", forbidden, input.Content) + if strings.Contains(string(input), forbidden) { + t.Fatalf("location projection exposed %q", forbidden) } } - if strings.Contains(string(client.requests[0].Inputs["transcript"].Content), "other-session") { - t.Fatal("transcript input contains registry evidence") - } - metadata, err := json.Marshal(newExtractor(t, &fakeOccurrencesLLMClient{}, references).ManifestMetadata()) - if err != nil || strings.Contains(string(metadata), "other-session") || strings.Contains(string(metadata), first.ID) { - t.Fatalf("manifest metadata = %s, %v", metadata, err) + if strings.Contains(string(renderedInput(t, client.requests[0], "transcript")), "other-session") { + t.Fatal("registry provenance became occurrence evidence") } + } func TestExtractRejectsUnknownMalformedOrMismatchedSelectorsAtomically(t *testing.T) { @@ -101,7 +97,7 @@ func TestExtractRejectsUnknownMalformedOrMismatchedSelectorsAtomically(t *testin {{Name: known.Name, RegistryRefs: []locationregistry.RegistryRef{}, Kind: "mentioned", SourceRefs: occurrenceRefs(10, 10)}, {Name: "Unknown", RegistryRefs: []locationregistry.RegistryRef{}, Kind: "mentioned", SourceRefs: occurrenceRefs(20, 20)}}, } { client := &fakeOccurrencesLLMClient{response: extractionResponse{Occurrences: occurrences}} - if result, err := newExtractor(t, client, references).Extract(context.Background(), req); err == nil || result.Value.Occurrences != nil || !strings.Contains(err.Error(), "location selector") { + if result, err := newExtractor(t, client, references).Extract(context.Background(), req); err == nil || result.Value.Occurrences != nil { t.Fatalf("Extract() = %#v, %v", result, err) } } @@ -125,7 +121,7 @@ func TestExtractRejectsUnknownMalformedOrMismatchedSelectorsAtomically(t *testin client := &fakeOccurrencesLLMClient{response: extractionResponse{Occurrences: []occurrenceResponse{{Name: selector.name, RegistryRefs: selector.refs, Kind: "mentioned", SourceRefs: occurrenceRefs(10, 10)}}}} duplicateRequest := extractionRequest() duplicateRequest.References = duplicateReferences - if _, err := newExtractor(t, client, duplicateReferences).Extract(context.Background(), duplicateRequest); err == nil || !strings.Contains(err.Error(), "location selector") { + if result, err := newExtractor(t, client, duplicateReferences).Extract(context.Background(), duplicateRequest); err == nil || result.Outcome.Disposition != "" || len(result.Value.Occurrences) != 0 { t.Fatalf("Extract(%#v) error = %v", selector, err) } } @@ -163,28 +159,25 @@ func TestExtractResolvesGeneratedRegistryAtOperationTimeAndDoesNotMutateResponse if err != nil || result.Value.Occurrences[0].Name != "The Mill" { t.Fatalf("Extract() = %#v, %v", result, err) } - if input := client.requests[0].Inputs[LocationRegistryReferenceSlot]; strings.Contains(string(input.Content), location.ID) || input.OriginURI != "" { - t.Fatalf("generated registry prompt input = %#v", input) - } - if _, ok := extractor.ManifestMetadata()["location_registry_digest"]; ok { - t.Fatalf("operation registry leaked into static metadata: %#v", extractor.ManifestMetadata()) + if strings.Contains(string(renderedInput(t, client.requests[0], LocationRegistryReferenceSlot)), location.ID) { + t.Fatal("generated registry exposed a durable ID") } if client.response.Occurrences[0].SourceRefs[0].StartUnitID != 30 { t.Fatalf("model response mutated: %#v", client.response) } } -func TestExtractorContractsMetadataAndFailures(t *testing.T) { - if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "LLM client") { +func TestExtractorContractsDependenciesAndFailures(t *testing.T) { + if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "native engine") { t.Fatalf("New(nil) error = %v", err) } - if _, err := New(&fakeOccurrencesLLMClient{}, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one reference set") { + if _, err := New(newExtractor(t, &fakeOccurrencesLLMClient{}).engine, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one reference set") { t.Fatalf("New() error = %v", err) } malformed := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{LocationRegistryReferenceSlot: { Items: []contracts.ReferenceItem{{SlotName: LocationRegistryReferenceSlot, MediaType: "application/json", Content: []byte(`{"secret":"registry evidence"}`)}}, }}} - if _, err := New(&fakeOccurrencesLLMClient{}, Options{}, malformed); err == nil || !strings.Contains(err.Error(), "prepare location registry") || strings.Contains(err.Error(), "registry evidence") { + if _, err := New(newExtractor(t, &fakeOccurrencesLLMClient{}).engine, Options{}, malformed); err == nil || !strings.Contains(err.Error(), "prepare location registry") || strings.Contains(err.Error(), "registry evidence") { t.Fatalf("New() error = %v", err) } @@ -197,11 +190,11 @@ func TestExtractorContractsMetadataAndFailures(t *testing.T) { for _, test := range []struct { name string extractor *Extractor - req contracts.TypedExtractionRequest + req contracts.TypedExtractionRequest[dnd.LocationOccurrenceList] want string }{ {"nil extractor", nilExtractor, req, "extractor"}, - {"nil client", &Extractor{}, req, "LLM client"}, + {"nil client", &Extractor{}, req, "native engine"}, {"invalid request", extractor, mismatchedSourceInputRequest(req), "must match chunk"}, } { t.Run(test.name, func(t *testing.T) { @@ -215,7 +208,7 @@ func TestExtractorContractsMetadataAndFailures(t *testing.T) { } spec := ModuleSpec() - if spec.Key != Key || spec.Stage != pipeline.StageExtract || spec.ExecutionClass != contracts.ExecutionClassLLMBacked || spec.CorrectionProtocol != contracts.CorrectionProtocolSingleResponseV1 || spec.ArtifactKind != dnd.LocationOccurrenceListKind { + if spec.Key != Key || spec.Stage != pipeline.StageExtract || spec.ExecutionClass != contracts.ExecutionClassLLMBacked || !reflect.DeepEqual(spec.Prompts, []string{PromptID, CorrectionPromptID}) || spec.ArtifactKind != dnd.LocationOccurrenceListKind { t.Fatalf("ModuleSpec() = %#v", spec) } var slot contracts.ReferenceSlot @@ -237,24 +230,6 @@ func TestExtractorContractsMetadataAndFailures(t *testing.T) { if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil { t.Fatal("DecodeOptions() accepted unknown options") } - metadata := newExtractor(t, &fakeOccurrencesLLMClient{}, references).ManifestMetadata() - for _, key := range []string{"prompt_sha256", "response_schema_sha256", "location_registry_digest"} { - if value, ok := metadata[key].(string); !ok || !strings.HasPrefix(value, "sha256:") { - t.Fatalf("metadata[%q] = %#v", key, metadata[key]) - } - } - if got := newExtractor(t, &fakeOccurrencesLLMClient{}, references).CheckpointFingerprints(); len(got) != 4 || got[3].Name != "location_registry" || got[3].Value != locationRegistryIdentityDigest(t, references) { - t.Fatalf("fingerprints = %#v", got) - } -} - -func locationRegistryIdentityDigest(t *testing.T, references contracts.ReferenceSet) string { - t.Helper() - resolver, err := locationregistry.NewResolver(references) - if err != nil { - t.Fatal(err) - } - return resolver.Seeded().IdentityDigest() } func locationRegistry(t *testing.T, names ...string) dnd.LocationRegistry { diff --git a/internal/modules/dnd/extract/locationoccurrences/native_execution_test.go b/internal/modules/dnd/extract/locationoccurrences/native_execution_test.go new file mode 100644 index 00000000..3ae182e5 --- /dev/null +++ b/internal/modules/dnd/extract/locationoccurrences/native_execution_test.go @@ -0,0 +1,162 @@ +package locationoccurrences + +import ( + "context" + "errors" + "reflect" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + occurrencecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/locationoccurrences" + refs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/locationoccurrences/source_refs" + + "go.maximumdirect.net/promptkit" +) + +const unresolvedOccurrence = `{"occurrences":[{"name":"The Tavern","registry_refs":[],"kind":"visited","source_refs":[{"start_unit_id":10,"end_unit_id":10}]}]}` +const knownOccurrence = `{"occurrences":[{"name":"The Tavern","registry_refs":[{"start_unit_id":20,"end_unit_id":20}],"kind":"visited","source_refs":[{"start_unit_id":10,"end_unit_id":10}]}]}` +const invalidCitation = `{"occurrences":[{"name":"The Tavern","registry_refs":[{"start_unit_id":20,"end_unit_id":20}],"kind":"visited","source_refs":[{"start_unit_id":999,"end_unit_id":999}]}]}` + +func operationReferences(t *testing.T) (contracts.ReferenceSet, string) { + locations := locationRegistry(t, "The Tavern", "The Tavern") + return registryReferences(t, locations), locations.Locations[1].ID +} + +func sourceAssessment(ctx context.Context, request contracts.TypedExtractionRequest[dnd.LocationOccurrenceList], value dnd.LocationOccurrenceList) (contracts.CandidateAssessment, error) { + validator := refs.New(refs.Options{}) + result, err := validator.Validate(ctx, contracts.TypedValidationRequest[dnd.LocationOccurrenceList]{Stage: "extract", Source: request.Source, Chunk: request.Chunk, Value: value}) + if err != nil { + return contracts.CandidateAssessment{}, err + } + decision := contracts.AssessmentRejected + if result.Approved { + decision = contracts.AssessmentApproved + } + return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{ValidatorName: validator.Name(), Decision: decision, Result: result}}}, nil +} + +func TestNativeCorrectionKeepsRegistryResolutionAndInputsFixed(t *testing.T) { + for _, first := range []string{unresolvedOccurrence, invalidCitation} { + request := extractionRequest() + references, expectedID := operationReferences(t) + request.References = references + request.Operation.Controls.MaxAdditionalTurns = 1 + // Accepted work consuming an ineligible generated reference stays ineligible. + request.Operation.ReuseEligible = false + request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "occurrence-override"} + var prepared []promptkit.PreparedRun + request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = append(prepared, value) } + assessments := 0 + request.Operation.Assess = func(ctx context.Context, value dnd.LocationOccurrenceList) (contracts.CandidateAssessment, error) { + assessments++ + return sourceAssessment(ctx, request, value) + } + client := &fakeOccurrencesLLMClient{responses: []string{first, knownOccurrence}} + client.onGenerate = func(context.Context, promptkit.GenerateRequest) { + if len(client.requests) == 1 { + request.References.Slots[LocationRegistryReferenceSlot].Items[0].Content[0] = '!' + } + } + result, err := newExtractor(t, client, request.References).Extract(context.Background(), request) + expectedAssessments := 1 + if first == invalidCitation { + expectedAssessments = 2 + } + if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || result.Outcome.ReuseEligible || + len(client.requests) != 2 || len(prepared) != 2 || assessments != expectedAssessments || len(result.Value.Occurrences) != 1 { + t.Fatalf("corrected occurrences=%#v %v, assessments=%d native=%d/%d", result, err, assessments, len(client.requests), len(prepared)) + } + occurrence := result.Value.Occurrences[0] + if occurrence.LocationID != expectedID || occurrence.Name != "The Tavern" || len(occurrence.SourceRefs) != 1 || + occurrence.SourceRefs[0].SourceID != request.Source.ID || occurrence.SourceRefs[0].StartUnitID != 10 || occurrence.SourceRefs[0].EndUnitID != 10 { + t.Fatalf("identity/evidence=%#v", occurrence) + } + if _, err := occurrencecodec.New().Encode(result.Value); err != nil { + t.Fatalf("durable occurrence=%v", err) + } + for i, call := range client.requests { + id := PromptID + if i > 0 { + id = CorrectionPromptID + } + if prepared[i].PromptID != id || prepared[i].SelectedProfileID != request.LLMProfile || call.Target.Model != "occurrence-override" || call.Prompt.SessionID != request.SessionID { + t.Fatal("native selection lost") + } + } + initial, revised := client.requests[0].Prompt.Messages, client.requests[1].Prompt.Messages + if len(revised) != len(initial)+2 || !reflect.DeepEqual(revised[:len(initial)], initial) || revised[len(initial)].Role != "assistant" || revised[len(initial)].Content != first { + t.Fatal("fixed native context/history changed") + } + input := string(renderedInput(t, client.requests[1], LocationRegistryReferenceSlot)) + if strings.Contains(input, expectedID) || strings.Contains(input, request.Source.ID) { + t.Fatal("registry projection exposed an opaque identity") + } + if !reflect.DeepEqual(renderedInput(t, client.requests[0], LocationRegistryReferenceSlot), renderedInput(t, client.requests[1], LocationRegistryReferenceSlot)) { + t.Fatal("registry projection was regenerated") + } + } +} + +func TestNativeOccurrenceFallbackIsAnExplicitEmptyList(t *testing.T) { + failure := errors.New("provider unavailable") + for _, test := range []struct { + name string + mode contracts.ExecutionMode + response string + errors []error + wantAssessments, wantRuns int + wantError bool + }{ + {"strict unresolved", contracts.ModeStrict, unresolvedOccurrence, nil, 0, 2, true}, + {"permissive unresolved", contracts.ModePermissive, unresolvedOccurrence, nil, 0, 2, false}, + {"rejected citation", contracts.ModePermissive, invalidCitation, nil, 2, 2, false}, + {"provider unavailable", contracts.ModePermissive, "", []error{failure}, 0, 1, false}, + {"failed continuation", contracts.ModePermissive, invalidCitation, []error{nil, failure}, 1, 2, false}, + {"structural failure", contracts.ModePermissive, `{"occurrences":null}`, nil, 0, 1, false}, + } { + t.Run(test.name, func(t *testing.T) { + request := extractionRequest() + request.References, _ = operationReferences(t) + request.Operation.Controls.Mode = test.mode + request.Operation.Controls.MaxAdditionalTurns = 1 + assessments, prepared := 0, 0 + request.Operation.Execution.OnPrepared = func(context.Context, promptkit.PreparedRun) { prepared++ } + request.Operation.Assess = func(ctx context.Context, value dnd.LocationOccurrenceList) (contracts.CandidateAssessment, error) { + assessments++ + return sourceAssessment(ctx, request, value) + } + client := &fakeOccurrencesLLMClient{raw: test.response, errors: test.errors} + result, err := newExtractor(t, client, request.References).Extract(context.Background(), request) + if (err != nil) != test.wantError || assessments != test.wantAssessments || prepared != test.wantRuns { + t.Fatalf("outcome=%#v %v, assessments/runs=%d/%d", result, err, assessments, prepared) + } + if test.wantError { + if result.Outcome.Disposition != "" { + t.Fatal("strict failure published an outcome") + } + return + } + if result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || result.Value.Occurrences == nil || len(result.Value.Occurrences) != 0 { + t.Fatalf("unqualified fallback=%#v", result) + } + if _, err := occurrencecodec.New().Encode(result.Value); err != nil { + t.Fatalf("empty occurrence shape=%v", err) + } + }) + } +} + +func TestCanceledOccurrenceDeliveryCannotConstructFallback(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + request := extractionRequest() + request.References, _ = operationReferences(t) + request.Operation.Controls.Mode = contracts.ModePermissive + client := &fakeOccurrencesLLMClient{raw: knownOccurrence, onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }} + result, err := newExtractor(t, client, request.References).Extract(ctx, request) + if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" || len(client.requests) != 1 { + t.Fatalf("cancellation=%#v %v", result, err) + } +} diff --git a/internal/modules/dnd/extract/locationoccurrences/prompt_assets.go b/internal/modules/dnd/extract/locationoccurrences/prompt_assets.go deleted file mode 100644 index 3bf33b2f..00000000 --- a/internal/modules/dnd/extract/locationoccurrences/prompt_assets.go +++ /dev/null @@ -1,75 +0,0 @@ -package locationoccurrences - -import ( - "fmt" - "io/fs" - "sync" - - rootassets "gitea.maximumdirect.net/eric/notarius/assets" - "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - "gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs" - "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" -) - -const promptAssetRoot = "assets/prompts" - -var promptAssetManifest = shared.PromptAssetManifest{ - ModuleDir: PromptID, - ModuleFiles: []promptfs.ModulePromptFile{ - {Name: "prompt.yaml", Path: "prompts/prompt.yaml"}, - {Name: "location-registry.md", Path: "prompts/location-registry.md"}, - {Name: "instructions.md", Path: "prompts/instructions.md"}, - }, - SharedFiles: []string{ - "common-dnd-system.md", - "common-dnd-extraction-evidence.md", - "common-dnd-identity.md", - "common-dnd-transcript-chunk.md", - "common-dnd-references.md", - }, -} - -func moduleAssetFS() (fs.FS, error) { - assets, err := fs.Sub(rootassets.FS(), "dnd/location-occurrences") - if err != nil { - return nil, fmt.Errorf("scope location-occurrence assets: %w", err) - } - return assets, nil -} - -func RegisterPromptAssets(registry *llm.AssetRegistry) error { - assets, err := moduleAssetFS() - if err != nil { - return err - } - promptFS, err := promptAssetManifest.PromptFS(assets) - if err != nil { - return fmt.Errorf("prepare location-occurrence prompt assets: %w", err) - } - if err := registry.RegisterPromptFS(promptFS, promptAssetRoot); err != nil { - return err - } - schemas, err := fs.Sub(assets, "schemas") - if err != nil { - return fmt.Errorf("scope location-occurrence schemas: %w", err) - } - return registry.RegisterSchemaFS(schemas, ".") -} - -func promptAssetMetadata() (string, error) { - promptAssetHashOnce.Do(func() { - assets, err := moduleAssetFS() - if err != nil { - promptAssetHashErr = err - return - } - promptAssetHash, promptAssetHashErr = promptAssetManifest.Hash(assets) - }) - return promptAssetHash, promptAssetHashErr -} - -var ( - promptAssetHashOnce sync.Once - promptAssetHash string - promptAssetHashErr error -) diff --git a/internal/modules/dnd/extract/locationoccurrences/prompt_assets_test.go b/internal/modules/dnd/extract/locationoccurrences/prompt_assets_test.go deleted file mode 100644 index 4d8233ba..00000000 --- a/internal/modules/dnd/extract/locationoccurrences/prompt_assets_test.go +++ /dev/null @@ -1,53 +0,0 @@ -package locationoccurrences - -import ( - "context" - "strings" - "testing" - "time" - - "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - "gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture" - "gitea.maximumdirect.net/eric/promptkit" -) - -func TestRegisterPromptAssetsPreparesLocationOccurrencePrompt(t *testing.T) { - registry := llm.NewAssetRegistry() - if err := RegisterPromptAssets(registry); err != nil { - t.Fatal(err) - } - engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{ - "location-occurrences-test": promptkitfixture.Profile("http://127.0.0.1:1/v1", "test"), - }) - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "location-occurrences-test", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline(`{"units":[{"sentinel":"location-occurrence-transcript"}]}`), "players": promptkit.Inline("location-occurrence-player"), "party": promptkit.Inline(" "), "glossary": promptkit.Inline(" "), - "location_registry": promptkit.Inline(`{"locations":[{"name":"location-occurrence-registry","registry_refs":[{"start_unit_id":7,"end_unit_id":7}],"context":[{"unit_id":7,"text":"registry context"}]}]}`), - }, - }) - if err != nil { - t.Fatal(err) - } - if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_location_occurrences_llm.v1.json" { - t.Fatalf("prepared prompt = %#v", prepared) - } - var registryMessage string - for _, message := range prepared.Messages { - if strings.Contains(message.Content, "contextual location registry") { - registryMessage = message.Content - } - } - if !strings.Contains(registryMessage, "location-occurrence-registry") || !strings.Contains(registryMessage, `"registry_refs":[{"start_unit_id":7,"end_unit_id":7}]`) || strings.Contains(registryMessage, "source_id") || strings.Contains(registryMessage, "location:sha256") { - t.Fatalf("rendered prompt did not preserve source-free registry grounding: %s", registryMessage) - } - content := make([]string, len(prepared.Messages)) - for index, message := range prepared.Messages { - content[index] = message.Content - } - rendered := strings.Join(content, "\n") - policy := strings.ReplaceAll(rendered, "\n", " ") - if !strings.Contains(policy, "context supports that coreference") || !strings.Contains(policy, "must not create a registry location") || !strings.Contains(policy, "provenance must never replace current-chunk evidence") || !strings.Contains(policy, "complete, ordered") { - t.Fatalf("rendered prompt = %q, want contextual coreference without registry-derived evidence", rendered) - } -} diff --git a/internal/modules/dnd/extract/locationoccurrences/schema.go b/internal/modules/dnd/extract/locationoccurrences/schema.go deleted file mode 100644 index b62da94f..00000000 --- a/internal/modules/dnd/extract/locationoccurrences/schema.go +++ /dev/null @@ -1,25 +0,0 @@ -package locationoccurrences - -import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - -const ( - PromptID = "dnd.location_occurrences" - ResponseSchemaKey = llm.ResponseSchemaKey("dnd_location_occurrences_llm") - ResponseSchemaID = "notarius.dnd.location_occurrences.llm" - ResponseSchemaName = "notarius_dnd_location_occurrences_llm_v1" - SchemaVersion = "v1" -) - -func loadResponseSchema() (llm.ResponseSchema, error) { - assets, err := moduleAssetFS() - if err != nil { - return llm.ResponseSchema{}, err - } - return llm.LoadResponseSchema(assets, llm.ResponseSchemaDefinition{ - Key: ResponseSchemaKey, - ID: ResponseSchemaID, - Version: SchemaVersion, - Name: ResponseSchemaName, - AssetPath: "schemas/dnd_location_occurrences_llm.v1.json", - }) -} diff --git a/internal/modules/dnd/extract/locationoccurrences/schema_test.go b/internal/modules/dnd/extract/locationoccurrences/schema_test.go deleted file mode 100644 index 12d7c987..00000000 --- a/internal/modules/dnd/extract/locationoccurrences/schema_test.go +++ /dev/null @@ -1,97 +0,0 @@ -package locationoccurrences - -import ( - "bytes" - "encoding/json" - "strings" - "testing" - - "github.com/santhosh-tekuri/jsonschema/v6" -) - -func TestResponseSchemaRestrictsPrivateOccurrenceStructureAndKinds(t *testing.T) { - schema, err := loadResponseSchema() - if err != nil { - t.Fatal(err) - } - if schema.Key != ResponseSchemaKey || schema.ID != ResponseSchemaID || schema.Name != ResponseSchemaName || schema.Version != SchemaVersion || !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) { - t.Fatalf("schema = %#v", schema) - } - valid := map[string]any{"occurrences": []any{map[string]any{ - "name": "The Mill", "registry_refs": []any{map[string]any{"start_unit_id": 1, "end_unit_id": 2}}, "kind": "visited", - "source_refs": []any{map[string]any{"start_unit_id": 1, "end_unit_id": 2}}, - }}} - if err := validateSchema(t, valid, schema.JSONSchema); err != nil { - t.Fatalf("valid response rejected: %v", err) - } - for _, mutate := range []func(map[string]any){ - func(value map[string]any) { delete(value, "registry_refs") }, - func(value map[string]any) { value["kind"] = "other" }, - func(value map[string]any) { value["unexpected"] = true }, - func(value map[string]any) { value["registry_refs"].([]any)[0].(map[string]any)["start_unit_id"] = 0 }, - func(value map[string]any) { - value["source_refs"].([]any)[0].(map[string]any)["source_id"] = "assigned later" - }, - } { - candidate := cloneCandidate(t, valid) - mutate(candidate["occurrences"].([]any)[0].(map[string]any)) - if err := validateSchema(t, candidate, schema.JSONSchema); err == nil { - t.Fatal("schema accepted structurally invalid response") - } - } -} - -func TestResponseSchemaIsDefensiveAndContentSafe(t *testing.T) { - first, err := loadResponseSchema() - if err != nil { - t.Fatal(err) - } - first.JSONSchema[0] = '[' - second, err := loadResponseSchema() - if err != nil || !json.Valid(second.JSONSchema) || bytes.Equal(first.JSONSchema, second.JSONSchema) { - t.Fatalf("schema = %s, %v", second.JSONSchema, err) - } - if diagnostics := second.DiagnosticsMap(); diagnostics["key"] != ResponseSchemaKey || diagnostics["id"] != ResponseSchemaID { - t.Fatalf("diagnostics = %#v", diagnostics) - } else if _, ok := diagnostics["json_schema"]; ok { - t.Fatalf("diagnostics leaked schema content: %#v", diagnostics) - } -} - -func cloneCandidate(t *testing.T, value map[string]any) map[string]any { - t.Helper() - content, err := json.Marshal(value) - if err != nil { - t.Fatal(err) - } - var clone map[string]any - if err := json.Unmarshal(content, &clone); err != nil { - t.Fatal(err) - } - return clone -} - -func validateSchema(t *testing.T, value map[string]any, schemaContent []byte) error { - t.Helper() - content, err := json.Marshal(value) - if err != nil { - return err - } - instance, err := jsonschema.UnmarshalJSON(bytes.NewReader(content)) - if err != nil { - return err - } - document, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaContent)) - if err != nil { - return err - } - compiler := jsonschema.NewCompiler() - if err := compiler.AddResource("schema.json", document); err != nil { - return err - } - schema, err := compiler.Compile("schema.json") - if err != nil { - return err - } - return schema.Validate(instance) -} diff --git a/internal/modules/dnd/extract/locationoccurrences/test_helpers_test.go b/internal/modules/dnd/extract/locationoccurrences/test_helpers_test.go index 2660a5ae..36c281b6 100644 --- a/internal/modules/dnd/extract/locationoccurrences/test_helpers_test.go +++ b/internal/modules/dnd/extract/locationoccurrences/test_helpers_test.go @@ -3,14 +3,19 @@ package locationoccurrences import ( "context" "encoding/json" - "errors" + "strings" "testing" + "gitea.maximumdirect.net/eric/notarius/assets" + "go.maximumdirect.net/configkit" + "go.maximumdirect.net/promptkit" + "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" ) -func extractionRequest() contracts.TypedExtractionRequest { +func extractionRequest() contracts.TypedExtractionRequest[dnd.LocationOccurrenceList] { doc := sourceDocument() chunk := &source.Chunk{ ID: "session-occurrences:chunk:0", SourceID: doc.ID, Index: 0, @@ -18,10 +23,13 @@ func extractionRequest() contracts.TypedExtractionRequest { Content: []byte(`{"units":[10,20,30]}`), MediaType: "application/json", Units: append([]source.SourceUnit(nil), doc.Units...), } - return contracts.TypedExtractionRequest{ + return contracts.TypedExtractionRequest[dnd.LocationOccurrenceList]{ Source: doc, Chunk: chunk, SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session-occurrences.json"), SessionID: "occurrence-session", LLMProfile: "occurrence-profile", + Operation: contracts.ProducerOperation[dnd.LocationOccurrenceList]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, Cache: contracts.CacheBypass}, ReuseEligible: true, Assess: func(context.Context, dnd.LocationOccurrenceList) (contracts.CandidateAssessment, error) { + return contracts.CandidateAssessment{}, nil + }}, } } @@ -37,51 +45,95 @@ func occurrenceRefs(start, end int) []occurrenceSourceRefResponse { return []occurrenceSourceRefResponse{{StartUnitID: start, EndUnitID: end}} } -func newExtractor(t *testing.T, client contracts.StructuredLLMClient, references ...contracts.ReferenceSet) *Extractor { +func newExtractor(t *testing.T, client promptkit.LLMClient, references ...contracts.ReferenceSet) *Extractor { t.Helper() - extractor, err := New(client, Options{}, references...) + type settings struct { + PromptKit promptkit.Settings `yaml:"promptkit"` + } + definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit }) + filesystem := configkit.NewFilesystem("Notarius assets", assets.FS()) + result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("occurrence fixture", []byte(`promptkit: + profiles: + - id: occurrence-profile + endpoint: https://example.invalid/v1 + model: occurrence-test-model + api_key_required: false +`))}, configkit.LayeredOptions[settings]{Definition: definition, Underlays: []configkit.Underlay{ + promptkit.BundledOpenRouterUnderlay(), + {Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}}, + }}) if err != nil { - t.Fatalf("New() error = %v", err) + t.Fatal(err) + } + engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client)) + if err != nil { + t.Fatal(err) + } + extractor, err := New(engine, Options{}, references...) + if err != nil { + t.Fatal(err) } return extractor } -func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest) contracts.TypedExtractionRequest { +func renderedInput(t *testing.T, request promptkit.GenerateRequest, name string) []byte { + t.Helper() + key := map[string]string{"transcript": "units", "npc_registry": "npcs", "location_registry": "locations"}[name] + for _, message := range request.Prompt.Messages { + start := strings.Index(message.Content, "{") + if start < 0 { + continue + } + content := []byte(message.Content[start:]) + var fields map[string]json.RawMessage + if json.Unmarshal(content, &fields) == nil && fields[key] != nil { + return content + } + } + t.Fatalf("missing rendered %s input", name) + return nil +} + +func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest[dnd.LocationOccurrenceList]) contracts.TypedExtractionRequest[dnd.LocationOccurrenceList] { req.SourceInput = contracts.NewLLMInputMaterial("source", "application/json", []byte(`{"different":true}`), "sha256:other", "") return req } type fakeOccurrencesLLMClient struct { - response extractionResponse - err error - requests []contracts.StructuredCompletionRequest + response extractionResponse + responses []string + raw string + err error + errors []error + requests []promptkit.GenerateRequest + onGenerate func(context.Context, promptkit.GenerateRequest) } -func (client *fakeOccurrencesLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { - client.requests = append(client.requests, cloneStructuredCompletionRequest(req)) +func (client *fakeOccurrencesLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + index := len(client.requests) + client.requests = append(client.requests, request) + if client.onGenerate != nil { + client.onGenerate(ctx, request) + } + if err := ctx.Err(); err != nil { + return nil, err + } if client.err != nil { - return contracts.StructuredCompletionResponse{}, client.err + return nil, client.err } - target, ok := out.(*extractionResponse) - if !ok { - return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target") + if index < len(client.errors) && client.errors[index] != nil { + return nil, client.errors[index] } - content, err := json.Marshal(client.response) - if err != nil { - return contracts.StructuredCompletionResponse{}, err + content := client.raw + if index < len(client.responses) { + content = client.responses[index] } - if err := json.Unmarshal(content, target); err != nil { - return contracts.StructuredCompletionResponse{}, err + if content == "" { + encoded, err := json.Marshal(client.response) + if err != nil { + return nil, err + } + content = string(encoded) } - return contracts.StructuredCompletionResponse{Content: content}, nil -} - -func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest { - req.Inputs = req.Inputs.Clone() - correction, err := contracts.CloneSemanticCorrection(req.Correction) - if err != nil { - panic(err) - } - req.Correction = correction - return req + return &promptkit.GenerateResponse{Content: content}, nil } diff --git a/internal/modules/dnd/extract/npcoccurrences/extractor.go b/internal/modules/dnd/extract/npcoccurrences/extractor.go index f6854625..3feb076e 100644 --- a/internal/modules/dnd/extract/npcoccurrences/extractor.go +++ b/internal/modules/dnd/extract/npcoccurrences/extractor.go @@ -2,8 +2,12 @@ package npcoccurrences import ( "context" + "encoding/json" "fmt" "sort" + "strings" + + "go.maximumdirect.net/promptkit" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" @@ -12,10 +16,9 @@ import ( "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" ) -const ( - Key = "dnd/npc-occurrences" - mappingPolicy = "dnd.npc_occurrences.extract_mapping.v4" -) +const Key = "dnd/npc-occurrences" +const PromptID = "notarius/dnd.npc_occurrences" +const CorrectionPromptID = "notarius/dnd.npc_occurrences.repair" const ( NPCRegistryReferenceSlot = npcregistry.ReferenceSlot @@ -53,21 +56,17 @@ func referenceSlots() []contracts.ReferenceSlot { } var _ contracts.Extractor[dnd.NPCOccurrenceList] = (*Extractor)(nil) -var _ contracts.ManifestMetadataProvider = (*Extractor)(nil) -var _ pipeline.CheckpointFingerprintProvider = (*Extractor)(nil) type Options struct{} type Extractor struct { - llm contracts.StructuredLLMClient - npcResolver *npcregistry.Resolver - promptSHA string - responseSchemaSHA string + engine *promptkit.Engine + npcResolver *npcregistry.Resolver } -func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) { - if llmClient == nil { - return nil, extractorErrorf("LLM client must not be nil") +func New(engine *promptkit.Engine, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) { + if engine == nil { + return nil, extractorErrorf("native engine must not be nil") } if len(references) > 1 { return nil, extractorErrorf("at most one reference set may be supplied") @@ -80,68 +79,19 @@ func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contr if err != nil { return nil, extractorErrorf("prepare NPC registry prompt input: %w", err) } - promptSHA, err := promptAssetMetadata() - if err != nil { - return nil, extractorErrorf("load prompt metadata: %w", err) - } - responseSchema, err := loadResponseSchema() - if err != nil { - return nil, extractorErrorf("load response schema: %w", err) - } - return &Extractor{ - llm: llmClient, - npcResolver: npcResolver, - promptSHA: promptSHA, - responseSchemaSHA: responseSchema.SHA256, - }, nil + return &Extractor{engine: engine, npcResolver: npcResolver}, nil } func (e *Extractor) Key() string { return Key } func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot { return referenceSlots() } -func (e *Extractor) ManifestMetadata() map[string]any { - if e == nil { - return nil - } - metadata := map[string]any{ - "prompt_id": PromptID, - "prompt_version": SchemaVersion, - "prompt_sha256": e.promptSHA, - "mapping_policy": mappingPolicy, - "response_schema_key": string(ResponseSchemaKey), - "response_schema_id": ResponseSchemaID, - "response_schema_name": ResponseSchemaName, - "response_schema_version": SchemaVersion, - "response_schema_sha256": e.responseSchemaSHA, - } - seeded := e.npcResolver.Seeded() - if seeded.Bound() { - metadata["npc_registry_digest"] = seeded.Digest() - metadata["npc_count"] = seeded.Count() - } - return metadata -} - -func (e *Extractor) CheckpointFingerprints() []pipeline.CheckpointFingerprint { - if e == nil { - return nil - } - seeded := e.npcResolver.Seeded() - return []pipeline.CheckpointFingerprint{ - {Name: "prompt", Value: e.promptSHA}, - {Name: "response_schema", Value: e.responseSchemaSHA}, - {Name: "mapping_policy", Value: mappingPolicy}, - {Name: "npc_registry", Value: seeded.IdentityDigest()}, - } -} - -func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.NPCOccurrenceList], error) { +func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest[dnd.NPCOccurrenceList]) (contracts.TypedExtractionResult[dnd.NPCOccurrenceList], error) { if e == nil { return contracts.TypedExtractionResult[dnd.NPCOccurrenceList]{}, extractorErrorf("extractor must not be nil") } - if e.llm == nil { - return contracts.TypedExtractionResult[dnd.NPCOccurrenceList]{}, extractorErrorf("LLM client must not be nil") + if e.engine == nil { + return contracts.TypedExtractionResult[dnd.NPCOccurrenceList]{}, extractorErrorf("native engine must not be nil") } sourceInput, err := shared.PrepareChunkExtraction(ctx, req) if err != nil { @@ -156,44 +106,50 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRe return contracts.TypedExtractionResult[dnd.NPCOccurrenceList]{}, extractorErrorf("NPC registry reference is required") } - var response extractionResponse - inputs := shared.PromptInputs(sourceInput, req.References) - inputs[NPCRegistryReferenceSlot] = npcRegistry.PromptInput() - completion, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{ - StageName: Key, - PromptID: PromptID, - PromptVersion: SchemaVersion, - ProfileID: req.LLMProfile, - SessionID: req.SessionID, - StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts, - Correction: req.Correction, - Inputs: inputs, - }, &response) - if err != nil { - return contracts.TypedExtractionResult[dnd.NPCOccurrenceList]{}, extractorErrorf("complete structured output: %w", err) + sourceID := req.Source.ID + materials := shared.PromptInputs(sourceInput, req.References) + materials[NPCRegistryReferenceSlot] = npcRegistry.PromptInput() + inputs := make(map[string]promptkit.ArtifactRef, len(materials)) + for name, material := range materials { + inputs[name] = promptkit.InlineWithURI(material.OriginURI, string(material.Content)) } - candidate, err := shared.ModelCandidateFromResponse(completion) + initial := promptkit.RunRequest{PromptID: PromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID, Inputs: inputs} + value, outcome, err := pipeline.RunProducer(ctx, e.engine, req.Operation, initial, + func(_ context.Context, result *promptkit.RunResult) (*dnd.NPCOccurrenceList, []string, error) { + var response extractionResponse + if err := json.Unmarshal([]byte(result.RawOutput), &response); err != nil { + return nil, nil, extractorErrorf("decode native output: %w", err) + } + canonicalizeResponse(&response, order, sourceID) + value, err := canonicalOccurrenceList(response, sourceID, npcRegistry) + if err != nil { + return nil, []string{"Resolve every occurrence to an exact canonical name from the supplied NPC registry. Omit occurrences for identities absent from that registry; do not invent an identity. Return the complete replacement occurrence list."}, nil + } + return &value, nil, nil + }, + func(_ context.Context, guidance []string) (promptkit.RunRequest, error) { + return promptkit.RunRequest{PromptID: CorrectionPromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID, + Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline(strings.Join(guidance, "\n\n"))}}, nil + }, + func(context.Context, *dnd.NPCOccurrenceList) (dnd.NPCOccurrenceList, error) { + return dnd.NPCOccurrenceList{Occurrences: []dnd.NPCOccurrence{}}, nil + }) if err != nil { - return contracts.TypedExtractionResult[dnd.NPCOccurrenceList]{}, extractorErrorf("capture model candidate: %w", err) + return contracts.TypedExtractionResult[dnd.NPCOccurrenceList]{}, extractorErrorf("produce occurrences: %w", err) } - canonicalizeResponse(&response, order, req.Source.ID) - value, err := canonicalOccurrenceList(response, req.Source.ID, npcRegistry) - if err != nil { - return contracts.TypedExtractionResult[dnd.NPCOccurrenceList]{}, extractorErrorf("resolve NPC names against registry: %w", err) - } - return contracts.TypedExtractionResult[dnd.NPCOccurrenceList]{Value: value, ModelCandidate: candidate}, nil + return contracts.TypedExtractionResult[dnd.NPCOccurrenceList]{Value: value, Outcome: outcome}, nil } func ModuleSpec() pipeline.ModuleSpec { return pipeline.ModuleSpec{ - Key: Key, - Stage: pipeline.StageExtract, - ExecutionClass: contracts.ExecutionClassLLMBacked, - CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1, - Requires: append([]string(nil), requiredCapabilities...), - Provides: append([]string(nil), providedCapabilities...), - ArtifactKind: dnd.NPCOccurrenceListKind, - ReferenceSlots: referenceSlots(), + Key: Key, + Stage: pipeline.StageExtract, + ExecutionClass: contracts.ExecutionClassLLMBacked, + Prompts: []string{PromptID, CorrectionPromptID}, + Requires: append([]string(nil), requiredCapabilities...), + Provides: append([]string(nil), providedCapabilities...), + ArtifactKind: dnd.NPCOccurrenceListKind, + ReferenceSlots: referenceSlots(), } } @@ -203,7 +159,7 @@ func Register(registry *pipeline.ExtractorRegistry) error { if err != nil { return nil, err } - return New(request.Dependencies.LLM, options, request.References) + return New(request.Dependencies.Engine, options, request.References) }) } diff --git a/internal/modules/dnd/extract/npcoccurrences/extractor_test.go b/internal/modules/dnd/extract/npcoccurrences/extractor_test.go index 1974d426..4730f6e5 100644 --- a/internal/modules/dnd/extract/npcoccurrences/extractor_test.go +++ b/internal/modules/dnd/extract/npcoccurrences/extractor_test.go @@ -2,7 +2,6 @@ package npcoccurrences import ( "context" - "encoding/json" "errors" "reflect" "strings" @@ -105,10 +104,10 @@ func TestExtractUsesDocumentOrderForReferencesAndOccurrences(t *testing.T) { } func TestNewRequiresLLMAndRejectsAmbiguousReferenceSets(t *testing.T) { - if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "LLM client") { + if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "native engine") { t.Fatalf("New(nil) error = %v", err) } - if _, err := New(&fakeOccurrencesLLMClient{}, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one reference set") { + if _, err := New(newExtractor(t, &fakeOccurrencesLLMClient{}).engine, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one reference set") { t.Fatalf("New() error = %v", err) } } @@ -124,25 +123,19 @@ func TestExtractUsesNamesOnlyRegistryAndCurrentTranscriptEvidence(t *testing.T) t.Fatalf("Extract() error = %v", err) } request := client.requests[0] - registry := request.Inputs[NPCRegistryReferenceSlot] - if registry.Name != NPCRegistryReferenceSlot || registry.MediaType != "application/json" || string(registry.Content) != `{"npcs":[{"name":"Mira Thorn"},{"name":"Hooded Guard"}]}` { - t.Fatalf("registry prompt input = %#v, want names-only projection", registry) + registry := renderedInput(t, request, NPCRegistryReferenceSlot) + if string(registry) != `{"npcs":[{"name":"Mira Thorn"},{"name":"Hooded Guard"}]}` { + t.Fatalf("names-only registry=%s", registry) } for _, forbidden := range []string{"npc:sha256:", "other-session", "start_unit_id"} { - if strings.Contains(string(registry.Content), forbidden) { - t.Fatalf("registry prompt input leaked %q: %s", forbidden, registry.Content) + if strings.Contains(string(registry), forbidden) { + t.Fatalf("registry projection exposed %q", forbidden) } } - if strings.Contains(string(request.Inputs["transcript"].Content), "other-session") { - t.Fatal("transcript input contains registry evidence") - } - metadata, err := json.Marshal(newExtractor(t, &fakeOccurrencesLLMClient{}, references).ManifestMetadata()) - if err != nil { - t.Fatal(err) - } - if strings.Contains(string(metadata), "Mira Thorn") || strings.Contains(string(metadata), "other-session") { - t.Fatalf("manifest metadata leaked registry content: %s", metadata) + if strings.Contains(string(renderedInput(t, request, "transcript")), "other-session") { + t.Fatal("registry provenance became occurrence evidence") } + } func TestExtractRequiresBoundRegistryBeforeLLMCall(t *testing.T) { @@ -160,7 +153,7 @@ func TestExtractRequiresBoundRegistryBeforeLLMCall(t *testing.T) { Items: []contracts.ReferenceItem{{SlotName: NPCRegistryReferenceSlot, MediaType: "application/json", Content: []byte(`{"secret":"private source"}`)}}, }, }} - if _, err := New(client, Options{}, malformed); err == nil || !strings.Contains(err.Error(), "prepare NPC registry") || strings.Contains(err.Error(), "private source") { + if _, err := New(newExtractor(t, client).engine, Options{}, malformed); err == nil || !strings.Contains(err.Error(), "prepare NPC registry") || strings.Contains(err.Error(), "private source") { t.Fatalf("New() error = %v, want content-safe malformed registry error", err) } if len(client.requests) != 0 { @@ -177,7 +170,7 @@ func TestExtractRejectsUnknownNamesWithoutPartialResult(t *testing.T) { req := extractionRequest() req.References = references result, err := newExtractor(t, client, references).Extract(context.Background(), req) - if err == nil || !strings.Contains(err.Error(), "name is not in the NPC registry") { + if err == nil || result.Outcome.Disposition != "" { t.Fatalf("Extract() error = %v, want unknown name failure", err) } if len(result.Value.Occurrences) != 0 { @@ -211,13 +204,10 @@ func TestExtractResolvesGeneratedRegistryAtOperationTime(t *testing.T) { if _, err := extractor.Extract(context.Background(), req); err != nil { t.Fatalf("Extract() error = %v", err) } - if input := client.requests[0].Inputs[NPCRegistryReferenceSlot]; string(input.Content) != `{"npcs":[{"name":"Mira Thorn"}]}` || input.OriginURI != "" { - t.Fatalf("generated registry prompt input = %#v", input) - } - metadata := extractor.ManifestMetadata() - if _, ok := metadata["npc_registry_digest"]; ok { - t.Fatalf("operation-time registry leaked into static metadata: %#v", metadata) + if input := renderedInput(t, client.requests[0], NPCRegistryReferenceSlot); string(input) != `{"npcs":[{"name":"Mira Thorn"}]}` { + t.Fatalf("generated registry=%s", input) } + } func TestExtractAcceptsEmptyBoundRegistryAndEmptyResponse(t *testing.T) { @@ -253,7 +243,7 @@ func TestExtractRejectsNonemptyResponseForEmptyBoundRegistry(t *testing.T) { req := extractionRequest() req.References = references result, err := newExtractor(t, client, references).Extract(context.Background(), req) - if err == nil || !strings.Contains(err.Error(), "name is not in the NPC registry") || len(result.Value.Occurrences) != 0 { + if err == nil || result.Outcome.Disposition != "" || len(result.Value.Occurrences) != 0 { t.Fatalf("Extract() = %#v, %v; want no accepted occurrences", result, err) } } @@ -268,11 +258,11 @@ func TestExtractRejectsInvalidRequestsAndProviderFailures(t *testing.T) { name string extractor *Extractor ctx context.Context - req contracts.TypedExtractionRequest + req contracts.TypedExtractionRequest[dnd.NPCOccurrenceList] want string }{ {"nil extractor", nilExtractor, context.Background(), valid, "extractor"}, - {"nil LLM client", &Extractor{}, context.Background(), valid, "LLM client"}, + {"nil native engine", &Extractor{}, context.Background(), valid, "native engine"}, {"wrapped preflight failure", extractor, context.Background(), mismatchedSourceInputRequest(valid), "must match chunk"}, } { t.Run(test.name, func(t *testing.T) { @@ -286,9 +276,9 @@ func TestExtractRejectsInvalidRequestsAndProviderFailures(t *testing.T) { } } -func TestModuleSpecRegistrationMetadataAndFingerprints(t *testing.T) { +func TestModuleSpecRegistrationAndNativeDependencies(t *testing.T) { got := ModuleSpec() - if got.Key != Key || got.Stage != pipeline.StageExtract || got.CorrectionProtocol != contracts.CorrectionProtocolSingleResponseV1 || got.ArtifactKind != dnd.NPCOccurrenceListKind || !reflect.DeepEqual(got.Requires, []string{"chunks", "source.transcript"}) || !reflect.DeepEqual(got.Provides, []string{"dnd.npc_occurrences"}) { + if got.Key != Key || got.Stage != pipeline.StageExtract || !reflect.DeepEqual(got.Prompts, []string{PromptID, CorrectionPromptID}) || got.ArtifactKind != dnd.NPCOccurrenceListKind || !reflect.DeepEqual(got.Requires, []string{"chunks", "source.transcript"}) || !reflect.DeepEqual(got.Provides, []string{"dnd.npc_occurrences"}) { t.Fatalf("ModuleSpec() = %#v", got) } var registrySlot contracts.ReferenceSlot @@ -300,6 +290,10 @@ func TestModuleSpecRegistrationMetadataAndFingerprints(t *testing.T) { if !registrySlot.Required || !reflect.DeepEqual(registrySlot.AcceptedArtifactKinds, []contracts.ArtifactKind{dnd.NPCRegistryKind}) || registrySlot.MaxBytes != NPCRegistryMaxBytes { t.Fatalf("NPC registry slot = %#v", registrySlot) } + got.Prompts[0] = "changed" + if ModuleSpec().Prompts[0] == "changed" { + t.Fatal("prompt dependencies alias caller memory") + } got.ReferenceSlots[0].AcceptedMediaTypes[0] = "changed" if ModuleSpec().ReferenceSlots[0].AcceptedMediaTypes[0] == "changed" { t.Fatal("ModuleSpec() returned mutable reference slots") @@ -320,145 +314,4 @@ func TestModuleSpecRegistrationMetadataAndFingerprints(t *testing.T) { t.Fatal("DecodeOptions() accepted unknown option") } - references := requiredRegistryReferences(t, "Mira Thorn") - extractor := newExtractor(t, &fakeOccurrencesLLMClient{}, references) - metadata := extractor.ManifestMetadata() - for key, want := range map[string]string{ - "prompt_id": PromptID, "prompt_version": SchemaVersion, "mapping_policy": mappingPolicy, - "response_schema_key": string(ResponseSchemaKey), "response_schema_id": ResponseSchemaID, - "response_schema_name": ResponseSchemaName, "response_schema_version": SchemaVersion, - } { - if metadata[key] != want { - t.Fatalf("metadata[%q] = %#v, want %q", key, metadata[key], want) - } - } - for _, key := range []string{"prompt_sha256", "response_schema_sha256", "npc_registry_digest"} { - if value, ok := metadata[key].(string); !ok || !strings.HasPrefix(value, "sha256:") { - t.Fatalf("metadata[%q] = %#v, want digest", key, metadata[key]) - } - } - wantFingerprints := map[string]struct{}{"prompt": {}, "response_schema": {}, "mapping_policy": {}, "npc_registry": {}} - for _, fingerprint := range extractor.CheckpointFingerprints() { - delete(wantFingerprints, fingerprint.Name) - } - if len(wantFingerprints) != 0 { - t.Fatalf("missing fingerprints = %#v", wantFingerprints) - } -} - -func extractionRequest() contracts.TypedExtractionRequest { - doc := sourceDocument() - chunk := &source.Chunk{ - ID: "session-alpha:chunk:0", SourceID: doc.ID, Index: 0, - Ref: source.SourceRef{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 30}, - Content: []byte(`{"units":[10,2,7,5,20,30]}`), MediaType: "application/json", - Units: append([]source.SourceUnit(nil), doc.Units...), - } - return contracts.TypedExtractionRequest{ - Source: doc, Chunk: chunk, - SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session-alpha.json"), - SessionID: "session-123", LLMProfile: "profile-npc-occurrences", - } -} - -func sourceDocument() *source.SourceDocument { - return &source.SourceDocument{ - ID: "session-alpha", Kind: "transcript", Format: "application/vnd.seriatim.minimal+json", Digest: "sha256:source", - Units: []source.SourceUnit{ - {ID: 10, Kind: "transcript_segment", Text: "Mira Thorn is mentioned."}, - {ID: 2, Kind: "transcript_segment", Text: "Mira Thorn speaks to the party."}, - {ID: 7, Kind: "transcript_segment", Text: "Mira Thorn watches nearby."}, - {ID: 5, Kind: "transcript_segment", Text: "Mira Thorn joins the party in combat."}, - {ID: 20, Kind: "transcript_segment", Text: "Mira Thorn attacks the party."}, - {ID: 30, Kind: "transcript_segment", Text: "Mira Thorn performs a ritual."}, - }, - } -} - -func requiredRegistryReferences(t *testing.T, names ...string) contracts.ReferenceSet { - t.Helper() - npcs := make([]dnd.NPC, len(names)) - for index, name := range names { - npcs[index] = dnd.NPC{ - ID: identity.DeriveID(name), Name: name, - SourceRefs: []source.SourceRef{{SourceID: "other-session", StartUnitID: index + 1, EndUnitID: index + 1}}, - } - } - content, err := npccodec.New().Encode(dnd.NPCRegistry{NPCs: npcs}) - if err != nil { - t.Fatal(err) - } - return contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{ - NPCRegistryReferenceSlot: { - Slot: contracts.ReferenceSlot{Name: NPCRegistryReferenceSlot}, - Items: []contracts.ReferenceItem{{SlotName: NPCRegistryReferenceSlot, MediaType: npccodec.MediaType, Content: content, Origin: contracts.ReferenceOrigin{Type: "generated"}}}, - }, - }} -} - -func occurrenceRefs(start, end int) []occurrenceSourceRefResponse { - return []occurrenceSourceRefResponse{{StartUnitID: start, EndUnitID: end}} -} - -func occurrenceNames(value dnd.NPCOccurrenceList) []string { - names := make([]string, len(value.Occurrences)) - for index, occurrence := range value.Occurrences { - names[index] = occurrence.Name - } - return names -} - -func occurrenceKinds(value dnd.NPCOccurrenceList) []dnd.NPCOccurrenceKind { - kinds := make([]dnd.NPCOccurrenceKind, len(value.Occurrences)) - for index, occurrence := range value.Occurrences { - kinds[index] = occurrence.Kind - } - return kinds -} - -func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest) contracts.TypedExtractionRequest { - req.SourceInput = contracts.NewLLMInputMaterial("source", "application/json", []byte(`{"different":true}`), "sha256:other", "") - return req -} - -func newExtractor(t *testing.T, client contracts.StructuredLLMClient, references ...contracts.ReferenceSet) *Extractor { - t.Helper() - extractor, err := New(client, Options{}, references...) - if err != nil { - t.Fatalf("New() error = %v", err) - } - return extractor -} - -type fakeOccurrencesLLMClient struct { - response extractionResponse - err error - requests []contracts.StructuredCompletionRequest -} - -func (client *fakeOccurrencesLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { - client.requests = append(client.requests, cloneStructuredCompletionRequest(req)) - if client.err != nil { - return contracts.StructuredCompletionResponse{}, client.err - } - target, ok := out.(*extractionResponse) - if !ok { - return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target") - } - *target = client.response - content, err := json.Marshal(client.response) - if err != nil { - return contracts.StructuredCompletionResponse{}, err - } - return contracts.StructuredCompletionResponse{Content: content}, nil -} - -func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest { - req.Inputs = req.Inputs.Clone() - correction, err := contracts.CloneSemanticCorrection(req.Correction) - if err != nil { - panic(err) - } - req.Correction = correction - return req } diff --git a/internal/modules/dnd/extract/npcoccurrences/native_execution_test.go b/internal/modules/dnd/extract/npcoccurrences/native_execution_test.go new file mode 100644 index 00000000..50b15ef8 --- /dev/null +++ b/internal/modules/dnd/extract/npcoccurrences/native_execution_test.go @@ -0,0 +1,161 @@ +package npcoccurrences + +import ( + "context" + "errors" + "reflect" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + occurrencecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcoccurrences" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity" + refs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/npcoccurrences/source_refs" + "go.maximumdirect.net/promptkit" +) + +const unresolvedOccurrence = `{"occurrences":[{"name":"Unknown NPC","kind":"dialogue","source_refs":[{"start_unit_id":10,"end_unit_id":10}]}]}` +const knownOccurrence = `{"occurrences":[{"name":"Mira Thorn","kind":"dialogue","source_refs":[{"start_unit_id":10,"end_unit_id":10}]}]}` +const invalidCitation = `{"occurrences":[{"name":"Mira Thorn","kind":"dialogue","source_refs":[{"start_unit_id":999,"end_unit_id":999}]}]}` + +func operationReferences(t *testing.T) (contracts.ReferenceSet, string) { + return requiredRegistryReferences(t, "Mira Thorn"), identity.DeriveID("Mira Thorn") +} + +func sourceAssessment(ctx context.Context, request contracts.TypedExtractionRequest[dnd.NPCOccurrenceList], value dnd.NPCOccurrenceList) (contracts.CandidateAssessment, error) { + validator := refs.New(refs.Options{}) + result, err := validator.Validate(ctx, contracts.TypedValidationRequest[dnd.NPCOccurrenceList]{Stage: "extract", Source: request.Source, Chunk: request.Chunk, Value: value}) + if err != nil { + return contracts.CandidateAssessment{}, err + } + decision := contracts.AssessmentRejected + if result.Approved { + decision = contracts.AssessmentApproved + } + return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{ValidatorName: validator.Name(), Decision: decision, Result: result}}}, nil +} + +func TestNativeCorrectionKeepsRegistryResolutionAndInputsFixed(t *testing.T) { + for _, first := range []string{unresolvedOccurrence, invalidCitation} { + request := extractionRequest() + references, expectedID := operationReferences(t) + request.References = references + request.Operation.Controls.MaxAdditionalTurns = 1 + // Accepted work consuming an ineligible generated reference stays ineligible. + request.Operation.ReuseEligible = false + request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "occurrence-override"} + var prepared []promptkit.PreparedRun + request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = append(prepared, value) } + assessments := 0 + request.Operation.Assess = func(ctx context.Context, value dnd.NPCOccurrenceList) (contracts.CandidateAssessment, error) { + assessments++ + return sourceAssessment(ctx, request, value) + } + client := &fakeOccurrencesLLMClient{responses: []string{first, knownOccurrence}} + client.onGenerate = func(context.Context, promptkit.GenerateRequest) { + if len(client.requests) == 1 { + request.References.Slots[NPCRegistryReferenceSlot].Items[0].Content[0] = '!' + } + } + result, err := newExtractor(t, client, request.References).Extract(context.Background(), request) + expectedAssessments := 1 + if first == invalidCitation { + expectedAssessments = 2 + } + if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || result.Outcome.ReuseEligible || + len(client.requests) != 2 || len(prepared) != 2 || assessments != expectedAssessments || len(result.Value.Occurrences) != 1 { + t.Fatalf("corrected occurrences=%#v %v, assessments=%d native=%d/%d", result, err, assessments, len(client.requests), len(prepared)) + } + occurrence := result.Value.Occurrences[0] + if occurrence.NPCID != expectedID || occurrence.Name != "Mira Thorn" || len(occurrence.SourceRefs) != 1 || + occurrence.SourceRefs[0].SourceID != request.Source.ID || occurrence.SourceRefs[0].StartUnitID != 10 || occurrence.SourceRefs[0].EndUnitID != 10 { + t.Fatalf("identity/evidence=%#v", occurrence) + } + if _, err := occurrencecodec.New().Encode(result.Value); err != nil { + t.Fatalf("durable occurrence=%v", err) + } + for i, call := range client.requests { + id := PromptID + if i > 0 { + id = CorrectionPromptID + } + if prepared[i].PromptID != id || prepared[i].SelectedProfileID != request.LLMProfile || call.Target.Model != "occurrence-override" || call.Prompt.SessionID != request.SessionID { + t.Fatal("native selection lost") + } + } + initial, revised := client.requests[0].Prompt.Messages, client.requests[1].Prompt.Messages + if len(revised) != len(initial)+2 || !reflect.DeepEqual(revised[:len(initial)], initial) || revised[len(initial)].Role != "assistant" || revised[len(initial)].Content != first { + t.Fatal("fixed native context/history changed") + } + input := string(renderedInput(t, client.requests[1], NPCRegistryReferenceSlot)) + if strings.Contains(input, expectedID) || strings.Contains(input, request.Source.ID) { + t.Fatal("registry projection exposed an opaque identity") + } + if !reflect.DeepEqual(renderedInput(t, client.requests[0], NPCRegistryReferenceSlot), renderedInput(t, client.requests[1], NPCRegistryReferenceSlot)) { + t.Fatal("registry projection was regenerated") + } + } +} + +func TestNativeOccurrenceFallbackIsAnExplicitEmptyList(t *testing.T) { + failure := errors.New("provider unavailable") + for _, test := range []struct { + name string + mode contracts.ExecutionMode + response string + errors []error + wantAssessments, wantRuns int + wantError bool + }{ + {"strict unresolved", contracts.ModeStrict, unresolvedOccurrence, nil, 0, 2, true}, + {"permissive unresolved", contracts.ModePermissive, unresolvedOccurrence, nil, 0, 2, false}, + {"rejected citation", contracts.ModePermissive, invalidCitation, nil, 2, 2, false}, + {"provider unavailable", contracts.ModePermissive, "", []error{failure}, 0, 1, false}, + {"failed continuation", contracts.ModePermissive, invalidCitation, []error{nil, failure}, 1, 2, false}, + {"structural failure", contracts.ModePermissive, `{"occurrences":null}`, nil, 0, 1, false}, + } { + t.Run(test.name, func(t *testing.T) { + request := extractionRequest() + request.References, _ = operationReferences(t) + request.Operation.Controls.Mode = test.mode + request.Operation.Controls.MaxAdditionalTurns = 1 + assessments, prepared := 0, 0 + request.Operation.Execution.OnPrepared = func(context.Context, promptkit.PreparedRun) { prepared++ } + request.Operation.Assess = func(ctx context.Context, value dnd.NPCOccurrenceList) (contracts.CandidateAssessment, error) { + assessments++ + return sourceAssessment(ctx, request, value) + } + client := &fakeOccurrencesLLMClient{raw: test.response, errors: test.errors} + result, err := newExtractor(t, client, request.References).Extract(context.Background(), request) + if (err != nil) != test.wantError || assessments != test.wantAssessments || prepared != test.wantRuns { + t.Fatalf("outcome=%#v %v, assessments/runs=%d/%d", result, err, assessments, prepared) + } + if test.wantError { + if result.Outcome.Disposition != "" { + t.Fatal("strict failure published an outcome") + } + return + } + if result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || result.Value.Occurrences == nil || len(result.Value.Occurrences) != 0 { + t.Fatalf("unqualified fallback=%#v", result) + } + if _, err := occurrencecodec.New().Encode(result.Value); err != nil { + t.Fatalf("empty occurrence shape=%v", err) + } + }) + } +} + +func TestCanceledOccurrenceDeliveryCannotConstructFallback(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + request := extractionRequest() + request.References, _ = operationReferences(t) + request.Operation.Controls.Mode = contracts.ModePermissive + client := &fakeOccurrencesLLMClient{raw: knownOccurrence, onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }} + result, err := newExtractor(t, client, request.References).Extract(ctx, request) + if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" || len(client.requests) != 1 { + t.Fatalf("cancellation=%#v %v", result, err) + } +} diff --git a/internal/modules/dnd/extract/npcoccurrences/prompt_assets.go b/internal/modules/dnd/extract/npcoccurrences/prompt_assets.go deleted file mode 100644 index 791f9158..00000000 --- a/internal/modules/dnd/extract/npcoccurrences/prompt_assets.go +++ /dev/null @@ -1,75 +0,0 @@ -package npcoccurrences - -import ( - "fmt" - "io/fs" - "sync" - - rootassets "gitea.maximumdirect.net/eric/notarius/assets" - "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - "gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs" - "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" -) - -const promptAssetRoot = "assets/prompts" - -var promptAssetManifest = shared.PromptAssetManifest{ - ModuleDir: "dnd.npc_occurrences", - ModuleFiles: []promptfs.ModulePromptFile{ - {Name: "prompt.yaml", Path: "prompts/prompt.yaml"}, - {Name: "instructions.md", Path: "prompts/instructions.md"}, - }, - SharedFiles: []string{ - "common-dnd-system.md", - "common-dnd-extraction-evidence.md", - "common-dnd-identity.md", - "common-dnd-transcript-chunk.md", - "common-dnd-references.md", - "common-dnd-npc-registry.md", - }, -} - -func moduleAssetFS() (fs.FS, error) { - assets, err := fs.Sub(rootassets.FS(), "dnd/npc-occurrences") - if err != nil { - return nil, fmt.Errorf("scope NPC-occurrence assets: %w", err) - } - return assets, nil -} - -func RegisterPromptAssets(registry *llm.AssetRegistry) error { - assets, err := moduleAssetFS() - if err != nil { - return err - } - promptFS, err := promptAssetManifest.PromptFS(assets) - if err != nil { - return fmt.Errorf("prepare NPC-occurrence prompt assets: %w", err) - } - if err := registry.RegisterPromptFS(promptFS, promptAssetRoot); err != nil { - return err - } - schemas, err := fs.Sub(assets, "schemas") - if err != nil { - return fmt.Errorf("scope NPC-occurrence schemas: %w", err) - } - return registry.RegisterSchemaFS(schemas, ".") -} - -func promptAssetMetadata() (string, error) { - promptAssetHashOnce.Do(func() { - assets, err := moduleAssetFS() - if err != nil { - promptAssetHashErr = err - return - } - promptAssetHash, promptAssetHashErr = promptAssetManifest.Hash(assets) - }) - return promptAssetHash, promptAssetHashErr -} - -var ( - promptAssetHashOnce sync.Once - promptAssetHash string - promptAssetHashErr error -) diff --git a/internal/modules/dnd/extract/npcoccurrences/prompt_assets_test.go b/internal/modules/dnd/extract/npcoccurrences/prompt_assets_test.go deleted file mode 100644 index 437f63d8..00000000 --- a/internal/modules/dnd/extract/npcoccurrences/prompt_assets_test.go +++ /dev/null @@ -1,70 +0,0 @@ -package npcoccurrences - -import ( - "context" - "io/fs" - "strings" - "testing" - "time" - - "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - "gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture" - "gitea.maximumdirect.net/eric/promptkit" -) - -func TestRegisterPromptAssetsAndPrepareOccurrencePrompt(t *testing.T) { - registry := llm.NewAssetRegistry() - if err := RegisterPromptAssets(registry); err != nil { - t.Fatal(err) - } - schemaFS, err := registry.SchemaFS() - if err != nil { - t.Fatal(err) - } - if _, err := fs.ReadFile(schemaFS, "dnd_npc_occurrences_llm.v1.json"); err != nil { - t.Fatalf("response schema asset: %v", err) - } - engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{ - "npc-occurrences-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "npc-occurrences-test-model"), - }) - transcript := `{"units":[{"sentinel":"occurrence-transcript"}]}` - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "npc-occurrences-test-profile", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.InlineWithURI("file:///session.json", transcript), - "players": promptkit.Inline("occurrence-player"), - "party": promptkit.Inline("Mira: ranger"), - "glossary": promptkit.Inline("Greencloak: title"), - "npc_registry": promptkit.Inline(`{"npcs":[{"name":"occurrence-npc"}]}`), - }, - }) - if err != nil { - t.Fatal(err) - } - if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_npc_occurrences_llm.v1.json" { - t.Fatalf("prepared prompt = %#v", prepared) - } -} - -func TestPromptMetadataDoesNotExposeAssetContent(t *testing.T) { - hash, err := promptAssetMetadata() - if err != nil || !strings.HasPrefix(hash, "sha256:") { - t.Fatalf("promptAssetMetadata() = %q, %v", hash, err) - } - metadata := newExtractor(t, &fakeOccurrencesLLMClient{}).ManifestMetadata() - for _, forbidden := range []string{"common-dnd-system", "dnd_npc_occurrences_llm.v1.json"} { - if strings.Contains(strings.Join(mapValues(metadata), " "), forbidden) { - t.Fatalf("metadata leaked prompt or schema content %q: %#v", forbidden, metadata) - } - } -} - -func mapValues(values map[string]any) []string { - out := make([]string, 0, len(values)) - for _, value := range values { - if text, ok := value.(string); ok { - out = append(out, text) - } - } - return out -} diff --git a/internal/modules/dnd/extract/npcoccurrences/schema.go b/internal/modules/dnd/extract/npcoccurrences/schema.go deleted file mode 100644 index d93aec17..00000000 --- a/internal/modules/dnd/extract/npcoccurrences/schema.go +++ /dev/null @@ -1,25 +0,0 @@ -package npcoccurrences - -import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - -const ( - PromptID = "dnd.npc_occurrences" - ResponseSchemaKey = llm.ResponseSchemaKey("dnd_npc_occurrences_llm") - ResponseSchemaID = "notarius.dnd.npc_occurrences.llm" - ResponseSchemaName = "notarius_dnd_npc_occurrences_llm_v1" - SchemaVersion = "v1" -) - -func loadResponseSchema() (llm.ResponseSchema, error) { - assets, err := moduleAssetFS() - if err != nil { - return llm.ResponseSchema{}, err - } - return llm.LoadResponseSchema(assets, llm.ResponseSchemaDefinition{ - Key: ResponseSchemaKey, - ID: ResponseSchemaID, - Version: SchemaVersion, - Name: ResponseSchemaName, - AssetPath: "schemas/dnd_npc_occurrences_llm.v1.json", - }) -} diff --git a/internal/modules/dnd/extract/npcoccurrences/schema_test.go b/internal/modules/dnd/extract/npcoccurrences/schema_test.go deleted file mode 100644 index 0a3f1643..00000000 --- a/internal/modules/dnd/extract/npcoccurrences/schema_test.go +++ /dev/null @@ -1,127 +0,0 @@ -package npcoccurrences - -import ( - "bytes" - "encoding/json" - "strings" - "testing" - - "github.com/santhosh-tekuri/jsonschema/v6" - - "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" -) - -func TestResponseSchemaOwnsOnlyPrivateStructuralContract(t *testing.T) { - schema, err := loadResponseSchema() - if err != nil { - t.Fatal(err) - } - if schema.Key != ResponseSchemaKey || schema.ID != ResponseSchemaID || schema.Name != ResponseSchemaName || schema.Version != SchemaVersion || !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) { - t.Fatalf("schema = %#v", schema) - } - valid := validOccurrenceResponse() - content, err := json.Marshal(valid) - if err != nil { - t.Fatal(err) - } - if err := validateJSONSchema(content, schema.JSONSchema); err != nil { - t.Fatalf("valid response rejected: %v", err) - } - - semanticCandidate := validOccurrenceResponse() - occurrence := semanticCandidate["occurrences"].([]any)[0].(map[string]any) - occurrence["name"] = "" - ref := occurrence["source_refs"].([]any)[0].(map[string]any) - ref["start_unit_id"] = 0 - ref["end_unit_id"] = -1 - content, err = json.Marshal(semanticCandidate) - if err != nil { - t.Fatal(err) - } - if err := validateJSONSchema(content, schema.JSONSchema); err != nil { - t.Fatalf("schema rejected non-categorical validator-owned semantics: %v", err) - } - - for _, mutate := range []func(map[string]any){ - func(record map[string]any) { delete(record, "name") }, - func(record map[string]any) { record["kind"] = 1 }, - func(record map[string]any) { record["kind"] = "unsupported" }, - func(record map[string]any) { record["npc_id"] = "npc:sha256:opaque" }, - func(record map[string]any) { record["unexpected"] = true }, - func(record map[string]any) { - record["source_refs"].([]any)[0].(map[string]any)["source_id"] = "assigned later" - }, - } { - candidate := validOccurrenceResponse() - mutate(candidate["occurrences"].([]any)[0].(map[string]any)) - content, err := json.Marshal(candidate) - if err != nil { - t.Fatal(err) - } - if err := validateJSONSchema(content, schema.JSONSchema); err == nil { - t.Fatal("schema accepted structurally invalid response") - } - } -} - -func TestResponseSchemaIsDefensiveAndContentSafe(t *testing.T) { - first, err := loadResponseSchema() - if err != nil { - t.Fatal(err) - } - first.JSONSchema[0] = '[' - second, err := loadResponseSchema() - if err != nil || !json.Valid(second.JSONSchema) || bytes.Equal(first.JSONSchema, second.JSONSchema) { - t.Fatalf("second schema = %s, %v", second.JSONSchema, err) - } - if diagnostics := second.DiagnosticsMap(); diagnostics["key"] != ResponseSchemaKey || diagnostics["id"] != ResponseSchemaID { - t.Fatalf("diagnostics = %#v", diagnostics) - } else if _, ok := diagnostics["json_schema"]; ok { - t.Fatalf("diagnostics leak schema content: %#v", diagnostics) - } -} - -func TestResponseSchemaAcceptsEverySupportedKind(t *testing.T) { - schema, err := loadResponseSchema() - if err != nil { - t.Fatal(err) - } - for _, kind := range []string{string(dnd.NPCOccurrenceKindMentioned), string(dnd.NPCOccurrenceKindNoncombatPresence), string(dnd.NPCOccurrenceKindDialogue), string(dnd.NPCOccurrenceKindCombatAlly), string(dnd.NPCOccurrenceKindCombatOpponent), string(dnd.NPCOccurrenceKindOther)} { - candidate := validOccurrenceResponse() - candidate["occurrences"].([]any)[0].(map[string]any)["kind"] = kind - content, err := json.Marshal(candidate) - if err != nil { - t.Fatal(err) - } - if err := validateJSONSchema(content, schema.JSONSchema); err != nil { - t.Fatalf("supported NPC-occurrence kind %q was rejected: %v", kind, err) - } - } -} - -func validOccurrenceResponse() map[string]any { - return map[string]any{"occurrences": []any{map[string]any{ - "name": "Mira Thorn", "kind": "dialogue", - "source_refs": []any{map[string]any{"start_unit_id": 1, "end_unit_id": 2}}, - }}} -} - -func validateJSONSchema(instanceContent, schemaContent []byte) error { - instance, err := jsonschema.UnmarshalJSON(bytes.NewReader(instanceContent)) - if err != nil { - return err - } - schemaDocument, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaContent)) - if err != nil { - return err - } - compiler := jsonschema.NewCompiler() - if err := compiler.AddResource("schema.json", schemaDocument); err != nil { - return err - } - schema, err := compiler.Compile("schema.json") - if err != nil { - return err - } - return schema.Validate(instance) -} diff --git a/internal/modules/dnd/extract/npcoccurrences/test_helpers_test.go b/internal/modules/dnd/extract/npcoccurrences/test_helpers_test.go new file mode 100644 index 00000000..2cedce9e --- /dev/null +++ b/internal/modules/dnd/extract/npcoccurrences/test_helpers_test.go @@ -0,0 +1,184 @@ +package npcoccurrences + +import ( + "context" + "encoding/json" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/assets" + "go.maximumdirect.net/configkit" + "go.maximumdirect.net/promptkit" + + "gitea.maximumdirect.net/eric/notarius/internal/core/source" + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcregistry" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity" +) + +func extractionRequest() contracts.TypedExtractionRequest[dnd.NPCOccurrenceList] { + doc := sourceDocument() + chunk := &source.Chunk{ + ID: "session-alpha:chunk:0", SourceID: doc.ID, Index: 0, + Ref: source.SourceRef{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 30}, + Content: []byte(`{"units":[10,2,7,5,20,30]}`), MediaType: "application/json", + Units: append([]source.SourceUnit(nil), doc.Units...), + } + return contracts.TypedExtractionRequest[dnd.NPCOccurrenceList]{ + Source: doc, Chunk: chunk, + SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session-alpha.json"), + SessionID: "session-123", LLMProfile: "profile-npc-occurrences", + Operation: contracts.ProducerOperation[dnd.NPCOccurrenceList]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, Cache: contracts.CacheBypass}, ReuseEligible: true, Assess: func(context.Context, dnd.NPCOccurrenceList) (contracts.CandidateAssessment, error) { + return contracts.CandidateAssessment{}, nil + }}, + } +} + +func sourceDocument() *source.SourceDocument { + return &source.SourceDocument{ + ID: "session-alpha", Kind: "transcript", Format: "application/vnd.seriatim.minimal+json", Digest: "sha256:source", + Units: []source.SourceUnit{ + {ID: 10, Kind: "transcript_segment", Text: "Mira Thorn is mentioned."}, + {ID: 2, Kind: "transcript_segment", Text: "Mira Thorn speaks to the party."}, + {ID: 7, Kind: "transcript_segment", Text: "Mira Thorn watches nearby."}, + {ID: 5, Kind: "transcript_segment", Text: "Mira Thorn joins the party in combat."}, + {ID: 20, Kind: "transcript_segment", Text: "Mira Thorn attacks the party."}, + {ID: 30, Kind: "transcript_segment", Text: "Mira Thorn performs a ritual."}, + }, + } +} + +func requiredRegistryReferences(t *testing.T, names ...string) contracts.ReferenceSet { + t.Helper() + npcs := make([]dnd.NPC, len(names)) + for index, name := range names { + npcs[index] = dnd.NPC{ + ID: identity.DeriveID(name), Name: name, + SourceRefs: []source.SourceRef{{SourceID: "other-session", StartUnitID: index + 1, EndUnitID: index + 1}}, + } + } + content, err := npccodec.New().Encode(dnd.NPCRegistry{NPCs: npcs}) + if err != nil { + t.Fatal(err) + } + return contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{ + NPCRegistryReferenceSlot: { + Slot: contracts.ReferenceSlot{Name: NPCRegistryReferenceSlot}, + Items: []contracts.ReferenceItem{{SlotName: NPCRegistryReferenceSlot, MediaType: npccodec.MediaType, Content: content, Origin: contracts.ReferenceOrigin{Type: "generated"}}}, + }, + }} +} + +func occurrenceRefs(start, end int) []occurrenceSourceRefResponse { + return []occurrenceSourceRefResponse{{StartUnitID: start, EndUnitID: end}} +} + +func occurrenceNames(value dnd.NPCOccurrenceList) []string { + names := make([]string, len(value.Occurrences)) + for index, occurrence := range value.Occurrences { + names[index] = occurrence.Name + } + return names +} + +func occurrenceKinds(value dnd.NPCOccurrenceList) []dnd.NPCOccurrenceKind { + kinds := make([]dnd.NPCOccurrenceKind, len(value.Occurrences)) + for index, occurrence := range value.Occurrences { + kinds[index] = occurrence.Kind + } + return kinds +} + +func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest[dnd.NPCOccurrenceList]) contracts.TypedExtractionRequest[dnd.NPCOccurrenceList] { + req.SourceInput = contracts.NewLLMInputMaterial("source", "application/json", []byte(`{"different":true}`), "sha256:other", "") + return req +} + +func newExtractor(t *testing.T, client promptkit.LLMClient, references ...contracts.ReferenceSet) *Extractor { + t.Helper() + type settings struct { + PromptKit promptkit.Settings `yaml:"promptkit"` + } + definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit }) + filesystem := configkit.NewFilesystem("Notarius assets", assets.FS()) + result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("occurrence fixture", []byte(`promptkit: + profiles: + - id: profile-npc-occurrences + endpoint: https://example.invalid/v1 + model: occurrence-test-model + api_key_required: false +`))}, configkit.LayeredOptions[settings]{Definition: definition, Underlays: []configkit.Underlay{ + promptkit.BundledOpenRouterUnderlay(), + {Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}}, + }}) + if err != nil { + t.Fatal(err) + } + engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client)) + if err != nil { + t.Fatal(err) + } + extractor, err := New(engine, Options{}, references...) + if err != nil { + t.Fatal(err) + } + return extractor +} + +func renderedInput(t *testing.T, request promptkit.GenerateRequest, name string) []byte { + t.Helper() + key := map[string]string{"transcript": "units", "npc_registry": "npcs", "location_registry": "locations"}[name] + for _, message := range request.Prompt.Messages { + start := strings.Index(message.Content, "{") + if start < 0 { + continue + } + content := []byte(message.Content[start:]) + var fields map[string]json.RawMessage + if json.Unmarshal(content, &fields) == nil && fields[key] != nil { + return content + } + } + t.Fatalf("missing rendered %s input", name) + return nil +} + +type fakeOccurrencesLLMClient struct { + response extractionResponse + responses []string + raw string + err error + errors []error + requests []promptkit.GenerateRequest + onGenerate func(context.Context, promptkit.GenerateRequest) +} + +func (client *fakeOccurrencesLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + index := len(client.requests) + client.requests = append(client.requests, request) + if client.onGenerate != nil { + client.onGenerate(ctx, request) + } + if err := ctx.Err(); err != nil { + return nil, err + } + if client.err != nil { + return nil, client.err + } + if index < len(client.errors) && client.errors[index] != nil { + return nil, client.errors[index] + } + content := client.raw + if index < len(client.responses) { + content = client.responses[index] + } + if content == "" { + encoded, err := json.Marshal(client.response) + if err != nil { + return nil, err + } + content = string(encoded) + } + return &promptkit.GenerateResponse{Content: content}, nil +} diff --git a/internal/modules/dnd/validate/locationoccurrences/shape/validator_test.go b/internal/modules/dnd/validate/locationoccurrences/shape/validator_test.go index 328d53ea..b859ff18 100644 --- a/internal/modules/dnd/validate/locationoccurrences/shape/validator_test.go +++ b/internal/modules/dnd/validate/locationoccurrences/shape/validator_test.go @@ -46,7 +46,7 @@ func TestValidatorRegisters(t *testing.T) { if err := Register(registry); err != nil { t.Fatal(err) } - if got, ok := registry.Spec(Key); !ok || got != Spec() || got.ExecutionClass != contracts.ExecutionClassDeterministic { + if got, ok := registry.Spec(Key); !ok || !reflect.DeepEqual(got, Spec()) || got.ExecutionClass != contracts.ExecutionClassDeterministic { t.Fatalf("registered spec = %#v, ok = %t", got, ok) } if got := New(Options{}).CheckpointFingerprints(); len(got) != 1 || got[0].Value != policy {