From 92c2382806d0b576a69833e92fa7e1902527c06e Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 22 Nov 2026 00:02:13 +0000 Subject: [PATCH] Migrate spell extraction to native execution and semantic continuation --- assets/dnd/spells/prompts/correction.md | 5 + assets/dnd/spells/prompts/instructions.md | 15 +- assets/dnd/spells/prompts/principles.md | 18 ++ assets/dnd/spells/prompts/prompt.yaml | 20 +- assets/dnd/spells/prompts/spell-catalog.md | 5 +- assets/package_test.go | 7 +- docs/roadmap/implementation.md | 18 ++ .../extract/spells/catalog_prompt_input.go | 17 +- .../modules/dnd/extract/spells/extractor.go | 155 +++++-------- .../dnd/extract/spells/extractor_test.go | 192 ++++++----------- .../spells/npc_registry_wiring_test.go | 120 ++++------- .../dnd/extract/spells/pipeline_test.go | 203 ++++++++++++++++++ .../dnd/extract/spells/prompt_assets.go | 76 ------- .../dnd/extract/spells/prompt_assets_test.go | 144 ++++--------- .../dnd/extract/spells/registry_test.go | 15 +- internal/modules/dnd/extract/spells/schema.go | 24 --- .../modules/dnd/extract/spells/schema_test.go | 66 +----- .../dnd/extract/spells/test_helpers_test.go | 101 +++++---- .../modules/dnd/shared/extraction_inputs.go | 2 +- .../dnd/shared/extraction_inputs_test.go | 12 +- .../validate/spells/catalog/validator_test.go | 2 +- 21 files changed, 563 insertions(+), 654 deletions(-) create mode 100644 assets/dnd/spells/prompts/correction.md create mode 100644 assets/dnd/spells/prompts/principles.md create mode 100644 internal/modules/dnd/extract/spells/pipeline_test.go delete mode 100644 internal/modules/dnd/extract/spells/prompt_assets.go delete mode 100644 internal/modules/dnd/extract/spells/schema.go diff --git a/assets/dnd/spells/prompts/correction.md b/assets/dnd/spells/prompts/correction.md new file mode 100644 index 00000000..d524cad2 --- /dev/null +++ b/assets/dnd/spells/prompts/correction.md @@ -0,0 +1,5 @@ +Revise the previous spell-cast list using these findings: + +{{ input "guidance" }} + +Return the complete replacement spell-cast list under the same output contract. diff --git a/assets/dnd/spells/prompts/instructions.md b/assets/dnd/spells/prompts/instructions.md index eccb32ab..ff5c0230 100644 --- a/assets/dnd/spells/prompts/instructions.md +++ b/assets/dnd/spells/prompts/instructions.md @@ -1,14 +1 @@ -Extract Dungeons & Dragons spell-cast artifacts from the provided transcript. -Include an actual casting event or an unambiguous declared casting attempt. -Exclude spell mentions, hypothetical plans, rules discussion, and catalog -matches that do not establish a casting event in the transcript. - -For every extracted cast, the transcript evidence must collectively support the -in-world caster, the spell, and the fact that the cast or declared attempt -occurred. - -Attribute every cast to its in-world caster. Map first-person player speech to -the associated player character, and attribute a spell narrated by the GM to -the in-world creature that casts it. If the caster cannot be resolved, use only -the most specific in-world identity supported by the transcript; do not invent -a name. +Extract the spell casts established by the provided transcript chunk. Return the complete spell-cast list, or an empty list when no casting event is supported. diff --git a/assets/dnd/spells/prompts/principles.md b/assets/dnd/spells/prompts/principles.md new file mode 100644 index 00000000..7db04165 --- /dev/null +++ b/assets/dnd/spells/prompts/principles.md @@ -0,0 +1,18 @@ +Extract Dungeons & Dragons spell-cast artifacts from transcript evidence. +Include an actual casting event or an unambiguous declared casting attempt. +Exclude spell mentions, hypothetical plans, rules discussion, and catalog +matches that do not establish a casting event in the transcript. + +For every extracted cast, the transcript evidence must collectively support the +in-world caster, the spell, and the fact that the cast or declared attempt +occurred. + +Attribute every cast to its in-world caster. Map first-person player speech to +the associated player character, and attribute a spell narrated by the GM to +the in-world creature that casts it. If the caster cannot be resolved, use only +the most specific in-world identity supported by the transcript; do not invent +a name. + +The spell catalog lists each spell's canonical_name and recognized aliases. +If the transcript uses an alias, select that entry's canonical_name. Return the +canonical spelling exactly; never return an alias as a spell name. diff --git a/assets/dnd/spells/prompts/prompt.yaml b/assets/dnd/spells/prompts/prompt.yaml index f62a2bb9..c85588c5 100644 --- a/assets/dnd/spells/prompts/prompt.yaml +++ b/assets/dnd/spells/prompts/prompt.yaml @@ -23,8 +23,10 @@ prompts: messages: - role: system content_file: ../../shared/prompts/common-dnd-system.md - - role: user + - role: system content_file: ../../shared/prompts/common-dnd-identity.md + - role: system + content_file: ./principles.md - role: user content_file: ../../shared/prompts/common-dnd-references.md cache_control: @@ -33,7 +35,7 @@ prompts: content_file: ../../shared/prompts/common-dnd-transcript-chunk.md cache_control: type: ephemeral - - role: user + - role: system content_file: ../../shared/prompts/common-dnd-extraction-evidence.md - role: user content_file: ../../shared/prompts/common-dnd-npc-registry.md @@ -47,3 +49,17 @@ prompts: format: json validation_mode: json_schema schema_path: dnd/spells/schemas/dnd_spells_llm.v1.json + + - id: dnd.spells.repair + default_profile: notarius/dnd-extraction + inputs: + - name: guidance + required: true + content_type: text/plain + messages: + - role: user + content_file: ./correction.md + output: + format: json + validation_mode: json_schema + schema_path: dnd/spells/schemas/dnd_spells_llm.v1.json diff --git a/assets/dnd/spells/prompts/spell-catalog.md b/assets/dnd/spells/prompts/spell-catalog.md index 554b66fd..c38c09c3 100644 --- a/assets/dnd/spells/prompts/spell-catalog.md +++ b/assets/dnd/spells/prompts/spell-catalog.md @@ -1,6 +1,3 @@ -The spell catalog for this extraction is provided below as JSON. Each entry -lists a `canonical_name` and its recognized `aliases`. If the transcript uses -an alias, select that entry's `canonical_name`. Return spell names using the -canonical spelling exactly; never return an alias as a spell name. +The spell recognition catalog for this operation: {{ input "spell_catalog" }} diff --git a/assets/package_test.go b/assets/package_test.go index 14bbb684..fb5580e5 100644 --- a/assets/package_test.go +++ b/assets/package_test.go @@ -131,8 +131,11 @@ func TestEmbeddedPromptsPrepareWithNativeContracts(t *testing.T) { if err != nil { t.Fatal(err) } - if len(details.Messages) == 0 || details.Messages[0].Role != "system" || strings.TrimSpace(details.Messages[0].Content) != strings.TrimSpace(string(content)) { - t.Fatal("shared initial system context was not preserved") + if first.Role == nil || len(details.Messages) == 0 || details.Messages[0].Role != *first.Role { + t.Fatal("declared message role was not preserved") + } + if !strings.Contains(string(content), "{{") && strings.TrimSpace(details.Messages[0].Content) != strings.TrimSpace(string(content)) { + t.Fatal("source-relative message content was not preserved") } }) } diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index d1a213c3..51811bf5 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -539,6 +539,24 @@ Native production modules are migrated in Stages 12–23. ## Stage 12: Migrate Spell Extraction As The Representative Producer +**Status:** Complete. Spell extraction now prepares fixed transcript/reference, +catalog, and NPC projections once and uses the shared native producer helper. +Added the same-schema/default-profile correction prompt, enduring system +principles, explicit run/profile/session propagation, static dependencies, and +the codec-valid empty fallback. Removed spell completion/asset/schema registries +and manifest/fingerprint metadata; retained private-response interpretation and +source-order canonicalization. Migrated focused tests to a native provider fake +and added a leaf-assembled generic-chunker spell contract covering complete ordered +assessment, final correction, strict exhaustion, permissive output, and exact +continuation history without repeated references. +Native asset, catalog/shape, isolated schema/input-helper, and focused pipeline +tests pass; runnable asset/catalog/shape vet/race checks and production pipeline/ +generic/Seriatim builds pass. Full spell/shared/source-validator tests, including +the new assembled contract, are blocked at setup by shared legacy asset helpers +importing old-path ConfigKit's removed YAML `plugin/limit`. Those tests are not +claimed passing. Central registration and remaining legacy fixtures/dependencies +remain assigned to Stages 24 and 35, with final assembled verification required. + **Goal:** prove native extraction with the real typed module and validators. **Inspect/change:** `internal/modules/dnd/extract/spells`, shared extraction input diff --git a/internal/modules/dnd/extract/spells/catalog_prompt_input.go b/internal/modules/dnd/extract/spells/catalog_prompt_input.go index 023391af..b21d8a7c 100644 --- a/internal/modules/dnd/extract/spells/catalog_prompt_input.go +++ b/internal/modules/dnd/extract/spells/catalog_prompt_input.go @@ -1,29 +1,18 @@ package spells import ( - "crypto/sha256" - "encoding/hex" "encoding/json" "fmt" - "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" spellcatalog "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/spells/catalog" ) -func newCatalogPromptInput(effective spellcatalog.EffectiveCatalog) (contracts.LLMInputMaterial, error) { +func newCatalogPromptInput(effective spellcatalog.EffectiveCatalog) ([]byte, error) { content, err := json.Marshal(struct { Spells []spellcatalog.PromptSpell `json:"spells"` }{Spells: effective.PromptSpells()}) if err != nil { - return contracts.LLMInputMaterial{}, fmt.Errorf("encode spell recognition catalog: %w", err) + return nil, fmt.Errorf("encode spell recognition catalog: %w", err) } - sum := sha256.Sum256(content) - digest := "sha256:" + hex.EncodeToString(sum[:]) - return contracts.NewLLMInputMaterial( - spellcatalog.SpellCatalogReferenceSlot, - "application/json", - content, - digest, - "", - ), nil + return content, nil } diff --git a/internal/modules/dnd/extract/spells/extractor.go b/internal/modules/dnd/extract/spells/extractor.go index ff81a144..669691c6 100644 --- a/internal/modules/dnd/extract/spells/extractor.go +++ b/internal/modules/dnd/extract/spells/extractor.go @@ -2,8 +2,12 @@ package spells import ( "context" + "encoding/json" "fmt" "sort" + "strings" + + "go.maximumdirect.net/promptkit" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" @@ -14,9 +18,8 @@ import ( ) const Key = "dnd/spells" -const SchemaVersion = "v1" - -const mappingPolicy = "dnd.spells.extract_mapping.v3" +const PromptID = "notarius/dnd.spells" +const CorrectionPromptID = "notarius/dnd.spells.repair" const ( NPCRegistryReferenceSlot = npcregistry.ReferenceSlot @@ -58,22 +61,18 @@ func referenceSlots() []contracts.ReferenceSlot { } var _ contracts.Extractor[dnd.SpellList] = (*Extractor)(nil) -var _ pipeline.CheckpointFingerprintProvider = (*Extractor)(nil) type Options struct{} type Extractor struct { - llm contracts.StructuredLLMClient - effectiveCatalog spellcatalog.EffectiveCatalog - catalogPromptInput contracts.LLMInputMaterial + engine *promptkit.Engine + catalogPromptInput []byte npcResolver *npcregistry.Resolver - promptSHA string - responseSchemaSHA string } -func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) { - if llmClient == nil { - return nil, extractorErrorf("LLM client must not be nil") +func New(engine *promptkit.Engine, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) { + if engine == nil { + return nil, extractorErrorf("native engine must not be nil") } if len(references) > 1 { return nil, extractorErrorf("at most one reference set may be supplied") @@ -94,22 +93,7 @@ func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contr if err != nil { return nil, extractorErrorf("prepare NPC registry prompt input: %w", err) } - promptSHA, err := promptAssetMetadata() - if err != nil { - return nil, extractorErrorf("load prompt metadata: %w", err) - } - responseSchema, err := loadResponseSchema() - if err != nil { - return nil, extractorErrorf("load response schema: %w", err) - } - return &Extractor{ - llm: llmClient, - effectiveCatalog: effectiveCatalog, - catalogPromptInput: catalogPromptInput, - npcResolver: npcResolver, - promptSHA: promptSHA, - responseSchemaSHA: responseSchema.SHA256, - }, nil + return &Extractor{engine: engine, catalogPromptInput: catalogPromptInput, npcResolver: npcResolver}, nil } func (e *Extractor) Key() string { @@ -120,99 +104,68 @@ func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot { return referenceSlots() } -func (e *Extractor) ManifestMetadata() map[string]any { - if e == nil { - return nil - } - metadata := map[string]any{ - "prompt_id": PromptID, - "prompt_version": SchemaVersion, - "prompt_sha256": e.promptSHA, - "catalog_base_id": e.effectiveCatalog.BaseID(), - "catalog_digest": e.effectiveCatalog.Digest(), - "catalog_overlay_ids": e.effectiveCatalog.OverlayIDs(), - "mapping_policy": mappingPolicy, - "response_schema_key": string(ResponseSchemaKey), - "response_schema_id": ResponseSchemaID, - "response_schema_name": ResponseSchemaName, - "response_schema_version": SchemaVersion, - "response_schema_sha256": e.responseSchemaSHA, - } - seeded := e.npcResolver.Seeded() - if seeded.Bound() { - metadata["npc_registry_digest"] = seeded.Digest() - metadata["npc_count"] = seeded.Count() - } - return metadata -} - -func (e *Extractor) CheckpointFingerprints() []pipeline.CheckpointFingerprint { - if e == nil { - return nil - } - fingerprints := []pipeline.CheckpointFingerprint{ - {Name: "effective_catalog", Value: e.effectiveCatalog.Digest()}, - {Name: "mapping_policy", Value: mappingPolicy}, - {Name: "prompt", Value: e.promptSHA}, - {Name: "response_schema", Value: e.responseSchemaSHA}, - } - seeded := e.npcResolver.Seeded() - fingerprints = append(fingerprints, pipeline.CheckpointFingerprint{Name: "npc_registry", Value: seeded.ProjectionDigest()}) - return fingerprints -} - -func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.SpellList], error) { +func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest[dnd.SpellList]) (contracts.TypedExtractionResult[dnd.SpellList], error) { if e == nil { return contracts.TypedExtractionResult[dnd.SpellList]{}, extractorErrorf("extractor must not be nil") } - if e.llm == nil { - return contracts.TypedExtractionResult[dnd.SpellList]{}, extractorErrorf("LLM client must not be nil") + if e.engine == nil { + return contracts.TypedExtractionResult[dnd.SpellList]{}, extractorErrorf("native engine must not be nil") } sourceInput, err := shared.PrepareChunkExtraction(ctx, req) if err != nil { return contracts.TypedExtractionResult[dnd.SpellList]{}, extractorErrorf("%w", err) } + sourceID := req.Source.ID order := shared.NewSourceRefOrder(req.Source) npcRegistry, err := e.npcResolver.Resolve(req.References) if err != nil { return contracts.TypedExtractionResult[dnd.SpellList]{}, extractorErrorf("resolve NPC registry: %w", err) } - var response extractionResponse - inputs := shared.PromptInputs(sourceInput, req.References) - inputs[spellcatalog.SpellCatalogReferenceSlot] = e.catalogPromptInput.Clone() - inputs[NPCRegistryReferenceSlot] = npcRegistry.PromptInput() - completion, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{ - StageName: Key, - PromptID: PromptID, - PromptVersion: SchemaVersion, - ProfileID: req.LLMProfile, - SessionID: req.SessionID, - StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts, - Correction: req.Correction, - Inputs: inputs, - }, &response) - if err != nil { - return contracts.TypedExtractionResult[dnd.SpellList]{}, extractorErrorf("complete structured output: %w", err) + materials := shared.PromptInputs(sourceInput, req.References) + inputs := make(map[string]promptkit.ArtifactRef, len(materials)+2) + for name, material := range materials { + inputs[name] = promptkit.InlineWithURI(material.OriginURI, string(material.Content)) } - candidate, err := shared.ModelCandidateFromResponse(completion) + inputs[spellcatalog.SpellCatalogReferenceSlot] = promptkit.Inline(string(e.catalogPromptInput)) + registryInput := npcRegistry.PromptInput() + inputs[NPCRegistryReferenceSlot] = promptkit.Inline(string(registryInput.Content)) + initial := promptkit.RunRequest{PromptID: PromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID, Inputs: inputs} + value, outcome, err := pipeline.RunProducer(ctx, e.engine, req.Operation, initial, + func(_ context.Context, result *promptkit.RunResult) (*dnd.SpellList, []string, error) { + var response extractionResponse + if err := json.Unmarshal([]byte(result.RawOutput), &response); err != nil { + return nil, nil, extractorErrorf("decode native output: %w", err) + } + canonicalizeResponse(&response, order, sourceID) + value := canonicalSpellList(response, sourceID) + return &value, nil, nil + }, + func(_ context.Context, guidance []string) (promptkit.RunRequest, error) { + return promptkit.RunRequest{ + PromptID: CorrectionPromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID, + Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline(strings.Join(guidance, "\n\n"))}, + }, nil + }, + func(context.Context, *dnd.SpellList) (dnd.SpellList, error) { + return dnd.SpellList{SpellCasts: []dnd.SpellCast{}}, nil + }) if err != nil { - return contracts.TypedExtractionResult[dnd.SpellList]{}, extractorErrorf("capture model candidate: %w", err) + return contracts.TypedExtractionResult[dnd.SpellList]{}, extractorErrorf("produce spells: %w", err) } - canonicalizeResponse(&response, order, req.Source.ID) - return contracts.TypedExtractionResult[dnd.SpellList]{Value: canonicalSpellList(response, req.Source.ID), ModelCandidate: candidate}, nil + return contracts.TypedExtractionResult[dnd.SpellList]{Value: value, Outcome: outcome}, nil } func ModuleSpec() pipeline.ModuleSpec { return pipeline.ModuleSpec{ - Key: Key, - Stage: pipeline.StageExtract, - ExecutionClass: contracts.ExecutionClassLLMBacked, - CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1, - Requires: append([]string(nil), requiredCapabilities...), - Provides: append([]string(nil), providedCapabilities...), - ArtifactKind: dnd.SpellListKind, - ReferenceSlots: referenceSlots(), + Key: Key, + Stage: pipeline.StageExtract, + ExecutionClass: contracts.ExecutionClassLLMBacked, + Prompts: []string{PromptID, CorrectionPromptID}, + Requires: append([]string(nil), requiredCapabilities...), + Provides: append([]string(nil), providedCapabilities...), + ArtifactKind: dnd.SpellListKind, + ReferenceSlots: referenceSlots(), } } @@ -222,7 +175,7 @@ func Register(registry *pipeline.ExtractorRegistry) error { if err != nil { return nil, err } - return New(request.Dependencies.LLM, options, request.References) + return New(request.Dependencies.Engine, options, request.References) }) } diff --git a/internal/modules/dnd/extract/spells/extractor_test.go b/internal/modules/dnd/extract/spells/extractor_test.go index f30bdb86..a849b7bd 100644 --- a/internal/modules/dnd/extract/spells/extractor_test.go +++ b/internal/modules/dnd/extract/spells/extractor_test.go @@ -14,6 +14,7 @@ import ( "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" spellcatalog "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/spells/catalog" + "go.maximumdirect.net/promptkit" ) func TestExtractReturnsCanonicalSpellListFromPrivateResponse(t *testing.T) { @@ -25,8 +26,11 @@ func TestExtractReturnsCanonicalSpellListFromPrivateResponse(t *testing.T) { }, }}} req := extractionRequest() + var prepared promptkit.PreparedRun + req.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = value } - result, err := newExtractor(t, client).Extract(context.Background(), req) + extractor := newExtractor(t, client) + result, err := extractor.Extract(context.Background(), req) if err != nil { t.Fatalf("Extract() error = %v, want nil", err) } @@ -44,27 +48,20 @@ func TestExtractReturnsCanonicalSpellListFromPrivateResponse(t *testing.T) { t.Fatalf("LLM calls = %d, want 1", len(client.requests)) } llmReq := client.requests[0] - if llmReq.StageName != Key || llmReq.PromptID != PromptID || llmReq.PromptVersion != SchemaVersion { - t.Fatalf("LLM request identity = %#v, want spell prompt", llmReq) + if prepared.PromptID != PromptID || prepared.SelectedProfileID != "profile-spells" || prepared.SessionID != "session-123" || llmReq.Target.Model != "spell-test-model" { + t.Fatalf("native selection lost: %#v", prepared) } - if llmReq.SessionID != "session-123" || llmReq.ProfileID != "profile-spells" { - t.Fatalf("session/profile = %q/%q, want session-123/profile-spells", llmReq.SessionID, llmReq.ProfileID) + if !strings.Contains(promptText(llmReq), string(req.Chunk.Content)) { + t.Fatal("missing exact chunk bytes") } - transcript := llmReq.Inputs["transcript"] - if transcript.Name != "transcript" || transcript.MediaType != "application/json" || transcript.Digest != "sha256:chunk" || transcript.OriginURI != "file:///session-alpha.json" { - t.Fatalf("transcript metadata = %#v", transcript) - } - if got := string(transcript.Content); got != string(req.Chunk.Content) { - t.Fatalf("transcript content = %q, want chunk content %q", got, req.Chunk.Content) - } - catalogInput := llmReq.Inputs[spellcatalog.SpellCatalogReferenceSlot] - if catalogInput.Name != spellcatalog.SpellCatalogReferenceSlot || catalogInput.MediaType != "application/json" || catalogInput.OriginURI != "" || !strings.HasPrefix(catalogInput.Digest, "sha256:") { - t.Fatalf("catalog prompt input metadata = %#v", catalogInput) + catalogInput := extractor.catalogPromptInput + if !strings.Contains(promptText(llmReq), string(catalogInput)) { + t.Fatal("missing exact catalog projection in native request") } var catalogPayload struct { Spells []spellcatalog.PromptSpell `json:"spells"` } - if err := json.Unmarshal(catalogInput.Content, &catalogPayload); err != nil { + if err := json.Unmarshal(catalogInput, &catalogPayload); err != nil { t.Fatalf("decode catalog prompt input: %v", err) } base, err := spellcatalog.LoadSRD5E2014() @@ -90,15 +87,16 @@ func TestExtractReturnsCanonicalSpellListFromPrivateResponse(t *testing.T) { func TestExtractPromptProjectsCanonicalNamesAndAliasesWithoutMetadata(t *testing.T) { client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}} - if _, err := newExtractor(t, client, overlaySpellCatalogReference()).Extract(context.Background(), extractionRequest()); err != nil { + extractor := newExtractor(t, client, overlaySpellCatalogReference()) + if _, err := extractor.Extract(context.Background(), extractionRequest()); err != nil { t.Fatalf("Extract() error = %v, want nil", err) } - input := client.requests[0].Inputs[spellcatalog.SpellCatalogReferenceSlot] - content := string(input.Content) + input := extractor.catalogPromptInput + content := string(input) var payload struct { Spells []spellcatalog.PromptSpell `json:"spells"` } - if err := json.Unmarshal(input.Content, &payload); err != nil { + if err := json.Unmarshal(input, &payload); err != nil { t.Fatalf("decode catalog prompt input: %v", err) } var aegis *spellcatalog.PromptSpell @@ -116,45 +114,6 @@ func TestExtractPromptProjectsCanonicalNamesAndAliasesWithoutMetadata(t *testing t.Fatalf("catalog prompt input leaked %q: %s", forbidden, content) } } - - metadata := newExtractor(t, &fakeSpellsLLMClient{}, overlaySpellCatalogReference()).ManifestMetadata() - if metadata["mapping_policy"] != mappingPolicy { - t.Fatalf("mapping policy metadata = %#v, want %q", metadata["mapping_policy"], mappingPolicy) - } - if metadata["catalog_base_id"] != spellcatalog.SRD5E2014ID { - t.Fatalf("catalog base metadata = %#v", metadata["catalog_base_id"]) - } - if digest, ok := metadata["catalog_digest"].(string); !ok || !strings.HasPrefix(digest, "sha256:") { - t.Fatalf("catalog digest metadata = %#v", metadata["catalog_digest"]) - } - if got, ok := metadata["catalog_overlay_ids"].([]string); !ok || !reflect.DeepEqual(got, []string{"campaign.example"}) { - t.Fatalf("catalog overlay metadata = %#v", metadata["catalog_overlay_ids"]) - } - fingerprints := newExtractor(t, &fakeSpellsLLMClient{}, overlaySpellCatalogReference()).CheckpointFingerprints() - wantFingerprints := map[string]any{ - "effective_catalog": metadata["catalog_digest"], - "mapping_policy": mappingPolicy, - "prompt": metadata["prompt_sha256"], - "response_schema": metadata["response_schema_sha256"], - "npc_registry": checkpointFingerprintMap(newExtractor(t, &fakeSpellsLLMClient{}).CheckpointFingerprints())["npc_registry"], - } - if len(fingerprints) != len(wantFingerprints) { - t.Fatalf("checkpoint fingerprints = %#v, want prompt, response schema, catalog, and NPC projection identities", fingerprints) - } - for _, fingerprint := range fingerprints { - if want, ok := wantFingerprints[fingerprint.Name]; !ok || fingerprint.Value != want { - t.Fatalf("checkpoint fingerprint %q = %q, want %#v", fingerprint.Name, fingerprint.Value, want) - } - } - encoded, err := json.Marshal(metadata) - if err != nil { - t.Fatal(err) - } - for _, forbidden := range []string{"Aegis of Emberfall", "Emberfall Aegis", "Private campaign source", "file:///private-source.json"} { - if strings.Contains(string(encoded), forbidden) { - t.Fatalf("manifest metadata leaked %q: %s", forbidden, encoded) - } - } } func TestExtractUsesAliasRecognitionToRequestCanonicalSpellNames(t *testing.T) { @@ -174,13 +133,13 @@ func TestExtractUsesAliasRecognitionToRequestCanonicalSpellNames(t *testing.T) { if got := result.Value.SpellCasts; len(got) != 1 || got[0].Spell != "Aegis of Emberfall" { t.Fatalf("spell casts = %#v, want canonical spell name", got) } - input := client.requests[0].Inputs[spellcatalog.SpellCatalogReferenceSlot] - if !strings.Contains(string(input.Content), `"aliases":["Emberfall Aegis"]`) { - t.Fatalf("catalog prompt input = %s, want transcript alias recognition", input.Content) + content := promptText(client.requests[0]) + if !strings.Contains(content, `"aliases":["Emberfall Aegis"]`) { + t.Fatalf("catalog prompt input = %s, want transcript alias recognition", content) } } -func TestExtractAliasOnlyCatalogChangesPromptMaterialAndCheckpointFingerprint(t *testing.T) { +func TestExtractAliasOnlyCatalogChangesPromptMaterial(t *testing.T) { aliasReference := spellCatalogReference(`{"schema_version":"notarius.dnd.spell-catalog-overlay.v1","catalogs":[{"id":"campaign.example","ruleset":"dnd-5e-2014","source":{"title":"Private campaign source","version":"1","url":"file:///private-source.json","license":"private"},"spells":[{"name":"Cure Wounds","aliases":["Campaign Woundweave"]}]}]}`) baseClient := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}} aliasClient := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}} @@ -193,27 +152,16 @@ func TestExtractAliasOnlyCatalogChangesPromptMaterialAndCheckpointFingerprint(t t.Fatalf("alias Extract() error = %v", err) } - baseInput := baseClient.requests[0].Inputs[spellcatalog.SpellCatalogReferenceSlot] - aliasInput := aliasClient.requests[0].Inputs[spellcatalog.SpellCatalogReferenceSlot] - if baseInput.Digest == aliasInput.Digest || string(baseInput.Content) == string(aliasInput.Content) { - t.Fatalf("alias-only catalog did not change prompt material: %q / %q", baseInput.Digest, aliasInput.Digest) - } - if !strings.Contains(string(aliasInput.Content), "Campaign Woundweave") { - t.Fatalf("alias prompt input = %s, want alias recognition", aliasInput.Content) - } - baseFingerprints := checkpointFingerprintMap(baseExtractor.CheckpointFingerprints()) - aliasFingerprints := checkpointFingerprintMap(aliasExtractor.CheckpointFingerprints()) - if baseFingerprints["effective_catalog"] == aliasFingerprints["effective_catalog"] { - t.Fatalf("effective catalog fingerprint did not change: %#v", aliasFingerprints) - } - if baseExtractor.ManifestMetadata()["prompt_id"] != aliasExtractor.ManifestMetadata()["prompt_id"] || baseExtractor.ManifestMetadata()["prompt_version"] != aliasExtractor.ManifestMetadata()["prompt_version"] { - t.Fatal("catalog-only change altered prompt identity") + baseInput := promptText(baseClient.requests[0]) + aliasInput := promptText(aliasClient.requests[0]) + if baseInput == aliasInput || !strings.Contains(aliasInput, "Campaign Woundweave") { + t.Fatal("alias-only catalog change was not reflected in native messages") } } func TestNewRejectsMalformedCatalogBeforeLLMCall(t *testing.T) { client := &fakeSpellsLLMClient{} - _, err := New(client, Options{}, spellCatalogReference(`{"schema_version":"notarius.dnd.spell-catalog-overlay.v2","catalogs":[]}`)) + _, err := New(newSpellEngine(t, client), Options{}, spellCatalogReference(`{"schema_version":"notarius.dnd.spell-catalog-overlay.v2","catalogs":[]}`)) if err == nil || !strings.Contains(err.Error(), "resolve effective spell catalog") { t.Fatalf("New() error = %v, want effective catalog error", err) } @@ -222,33 +170,6 @@ func TestNewRejectsMalformedCatalogBeforeLLMCall(t *testing.T) { } } -func TestExtractorManifestMetadataIncludesLLMSchemaProvenance(t *testing.T) { - metadata := newExtractor(t, &fakeSpellsLLMClient{}).ManifestMetadata() - tests := map[string]string{ - "prompt_id": PromptID, "prompt_version": SchemaVersion, - "response_schema_key": string(ResponseSchemaKey), "response_schema_id": ResponseSchemaID, - "response_schema_name": ResponseSchemaName, "response_schema_version": SchemaVersion, - } - for key, want := range tests { - if metadata[key] != want { - t.Fatalf("metadata[%q] = %#v, want %q", key, metadata[key], want) - } - } - for _, key := range []string{"prompt_sha256", "response_schema_sha256"} { - value, ok := metadata[key].(string) - if !ok || !strings.HasPrefix(value, "sha256:") { - t.Fatalf("metadata[%q] = %#v, want sha256 value", key, metadata[key]) - } - } -} - -func TestNilExtractorManifestMetadata(t *testing.T) { - var extractor *Extractor - if metadata := extractor.ManifestMetadata(); metadata != nil { - t.Fatalf("nil extractor metadata = %#v, want nil", metadata) - } -} - func TestExtractPassesReferencesAsPromptInputs(t *testing.T) { client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}} req := extractionRequest() @@ -261,12 +182,11 @@ func TestExtractPassesReferencesAsPromptInputs(t *testing.T) { if _, err := newExtractor(t, client).Extract(context.Background(), req); err != nil { t.Fatalf("Extract() error = %v, want nil", err) } - inputs := client.requests[0].Inputs - if string(inputs["players"].Content) != "Alice: Aria Brightmantle" || string(inputs["party"].Content) != "Aria Brightmantle: party cleric" || string(inputs["glossary"].Content) != "Brightmantle: local temple name" { - t.Fatalf("reference inputs = %#v, want configured content", inputs) - } - if strings.Contains(string(inputs["transcript"].Content), "party cleric") { - t.Fatal("transcript input contains reference content") + text := promptText(client.requests[0]) + for _, content := range []string{"Alice: Aria Brightmantle", "Aria Brightmantle: party cleric", "Brightmantle: local temple name", string(req.Chunk.Content)} { + if !strings.Contains(text, content) { + t.Fatalf("missing fixed input %q", content) + } } } @@ -282,23 +202,35 @@ func TestPromptInputsMapLegacyRosterReferenceToParty(t *testing.T) { } } -func TestExtractPreservesEmptyAndMalformedValuesForTypedValidators(t *testing.T) { - tests := []struct { - name string - response extractionResponse - wantNil bool - }{ - {name: "empty", response: extractionResponse{SpellCasts: []spellCastResponse{}}}, - {name: "missing", response: extractionResponse{}, wantNil: true}, +func TestExtractPreservesValidEmptyCollection(t *testing.T) { + result, err := newExtractor(t, &fakeSpellsLLMClient{content: []byte(`{"spell_casts":[]}`)}).Extract(context.Background(), extractionRequest()) + if err != nil || result.Value.SpellCasts == nil || len(result.Value.SpellCasts) != 0 { + t.Fatalf("empty collection: %#v %v", result, err) } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - result, err := newExtractor(t, &fakeSpellsLLMClient{response: test.response}).Extract(context.Background(), extractionRequest()) - if err != nil { - t.Fatalf("Extract() error = %v, want nil", err) +} + +func TestExtractDoesNotAssessNativeStructuralFailure(t *testing.T) { + for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} { + t.Run(string(mode), func(t *testing.T) { + client := &fakeSpellsLLMClient{content: []byte(`{"spell_casts":null}`)} + request := extractionRequest() + request.Operation.Controls.Mode = mode + assessed := 0 + request.Operation.Assess = func(context.Context, dnd.SpellList) (contracts.CandidateAssessment, error) { + assessed++ + return contracts.CandidateAssessment{}, nil } - if (result.Value.SpellCasts == nil) != test.wantNil || len(result.Value.SpellCasts) != 0 { - t.Fatalf("SpellCasts = %#v, want empty with nil=%t", result.Value.SpellCasts, test.wantNil) + result, err := newExtractor(t, client).Extract(context.Background(), request) + if assessed != 0 { + t.Fatal("structurally invalid native result reached typed validation") + } + if mode == contracts.ModeStrict { + if err == nil { + t.Fatal("strict native structural failure succeeded") + } + } else if err != nil || result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || + result.Value.SpellCasts == nil || len(result.Value.SpellCasts) != 0 { + t.Fatalf("permissive native failure = %#v, %v", result, err) } }) } @@ -319,11 +251,11 @@ func TestExtractRejectsInvalidRequests(t *testing.T) { name string extractor *Extractor ctx context.Context - req contracts.TypedExtractionRequest + req contracts.TypedExtractionRequest[dnd.SpellList] want string }{ {name: "nil extractor", extractor: nilExtractor, ctx: context.Background(), req: validReq, want: "extractor"}, - {name: "nil LLM client", extractor: &Extractor{}, ctx: context.Background(), req: validReq, want: "LLM client"}, + {name: "nil native engine", extractor: &Extractor{}, ctx: context.Background(), req: validReq, want: "native engine"}, {name: "wrapped preflight failure", extractor: validExtractor, ctx: context.Background(), req: mismatchedSourceInputRequest(validReq), want: "must match chunk"}, } for _, test := range tests { @@ -340,7 +272,7 @@ func TestExtractOrdersAndDeduplicatesEvidence(t *testing.T) { client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{ {Caster: "Borin", Spell: "Fire Bolt", SourceRefs: responseSourceRefs(2, 2)}, {Caster: "Aria", Spell: "Cure Wounds", SourceRefs: []spellSourceRefResponse{{StartUnitID: 1, EndUnitID: 2}, {StartUnitID: 1, EndUnitID: 2}}}, - {Caster: "Narrator", Spell: "Unknown"}, + {Caster: "Narrator", Spell: "Unknown", SourceRefs: []spellSourceRefResponse{}}, }}} result, err := newExtractor(t, client).Extract(context.Background(), extractionRequest()) if err != nil { diff --git a/internal/modules/dnd/extract/spells/npc_registry_wiring_test.go b/internal/modules/dnd/extract/spells/npc_registry_wiring_test.go index 1cba3756..5205244a 100644 --- a/internal/modules/dnd/extract/spells/npc_registry_wiring_test.go +++ b/internal/modules/dnd/extract/spells/npc_registry_wiring_test.go @@ -1,7 +1,6 @@ package spells import ( - "bytes" "context" "encoding/json" "strings" @@ -9,101 +8,74 @@ import ( "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" - "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcregistry" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/registry" ) -func TestSpellExtractorUsesExactUnboundNPCPromptAndOmitsRegistryIdentity(t *testing.T) { - extractor := newExtractor(t, &fakeSpellsLLMClient{}) - metadata := extractor.ManifestMetadata() - if metadata["npc_registry_digest"] != nil || metadata["npc_count"] != nil { - t.Fatalf("unbound extractor metadata = %#v, want no NPC registry fields", metadata) - } - fingerprints := checkpointFingerprintMap(extractor.CheckpointFingerprints()) - if fingerprints["npc_registry"] == "" { - t.Fatalf("unbound checkpoint fingerprints = %#v, want empty-projection identity", fingerprints) - } - - client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}} - extractor = newExtractor(t, client) - if _, err := extractor.Extract(context.Background(), extractionRequest()); err != nil { - t.Fatalf("Extract() error = %v, want nil", err) - } - input := client.requests[0].Inputs[NPCRegistryReferenceSlot] - if input.Name != NPCRegistryReferenceSlot || input.MediaType != npccodec.MediaType || input.Digest != fingerprints["npc_registry"] || input.OriginURI != "" || string(input.Content) != `{"npcs":[]}` { - t.Fatalf("NPC prompt input = %#v, want exact empty registry material", input) - } -} - -func TestSpellExtractorPreservesSemanticNPCRegistryFingerprintAndPromptWiring(t *testing.T) { +func TestSpellExtractorProjectsNPCNamesWithoutIdentityOrProvenance(t *testing.T) { value := registryFixture() canonical, err := npccodec.New().Encode(value) if err != nil { - t.Fatalf("encode NPC registry: %v", err) + t.Fatal(err) } pretty, err := json.MarshalIndent(value, "", " ") if err != nil { - t.Fatalf("MarshalIndent() error = %v", err) + t.Fatal(err) } - first := newExtractor(t, &fakeSpellsLLMClient{}, spellNPCRegistryReference(canonical, "file:///one.json")) - second := newExtractor(t, &fakeSpellsLLMClient{}, spellNPCRegistryReference(pretty, "file:///two.json")) - firstFingerprints := checkpointFingerprintMap(first.CheckpointFingerprints()) - secondFingerprints := checkpointFingerprintMap(second.CheckpointFingerprints()) - if firstFingerprints["npc_registry"] == "" || firstFingerprints["npc_registry"] != secondFingerprints["npc_registry"] { - t.Fatalf("semantic NPC fingerprints = %#v and %#v, want same npc_registry value", firstFingerprints, secondFingerprints) - } - metadata := first.ManifestMetadata() - if metadata["npc_registry_digest"] == "" || metadata["npc_registry_digest"] == firstFingerprints["npc_registry"] || metadata["npc_count"] != 1 { - t.Fatalf("NPC registry metadata = %#v, want digest and count only", metadata) - } - - client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}} - extractor := newExtractor(t, client, spellNPCRegistryReference(append([]byte("\n"), canonical...), "file:///npc-session.json")) - if _, err := extractor.Extract(context.Background(), extractionRequest()); err != nil { - t.Fatalf("Extract() error = %v, want nil", err) - } - input := client.requests[0].Inputs[NPCRegistryReferenceSlot] - if input.Name != NPCRegistryReferenceSlot || input.MediaType != npccodec.MediaType || input.Digest != firstFingerprints["npc_registry"] || input.OriginURI != "" { - t.Fatalf("NPC prompt input metadata = %#v, want semantic metadata without provenance", input) - } - if !bytes.Equal(input.Content, []byte(`{"npcs":[{"name":"Mira Thorn"}]}`)) { - t.Fatalf("NPC prompt input = %s, want names-only projection", input.Content) - } - encoded, err := json.Marshal(map[string]any{"metadata": metadata, "fingerprints": firstFingerprints}) - if err != nil { - t.Fatalf("marshal metadata: %v", err) - } - for _, forbidden := range []string{"Mira Thorn", "The Greencloak", "one.json", "two.json"} { - if strings.Contains(string(encoded), forbidden) { - t.Fatalf("metadata or fingerprints leaked %q: %s", forbidden, encoded) - } + var previous string + for _, test := range []struct { + name string + references contracts.ReferenceSet + want string + }{ + {"unbound", contracts.ReferenceSet{}, `{"npcs":[]}`}, + {"canonical", spellNPCRegistryReference(canonical, "file:///one.json"), `{"npcs":[{"name":"Mira Thorn"}]}`}, + {"different formatting and origin", spellNPCRegistryReference(pretty, "file:///two.json"), `{"npcs":[{"name":"Mira Thorn"}]}`}, + } { + t.Run(test.name, func(t *testing.T) { + client := &fakeSpellsLLMClient{content: []byte(`{"spell_casts":[]}`)} + if _, err := newExtractor(t, client, test.references).Extract(context.Background(), extractionRequest()); err != nil { + t.Fatal(err) + } + text := promptText(client.requests[0]) + if !strings.Contains(text, test.want) { + t.Fatalf("missing names-only projection %q", test.want) + } + for _, private := range []string{value.NPCs[0].ID, "npc-session", "file:///one.json", "file:///two.json"} { + if strings.Contains(text, private) { + t.Fatalf("NPC prompt leaked %q", private) + } + } + if test.name == "different formatting and origin" && text != previous { + t.Fatal("semantic NPC projection changed with provenance/formatting") + } + previous = text + }) } } -func TestSpellExtractorResolvesOperationNPCOverrideWithoutSingletonMetadata(t *testing.T) { +func TestSpellExtractorResolvesOperationNPCOverride(t *testing.T) { canonical, err := npccodec.New().Encode(registryFixture()) if err != nil { - t.Fatalf("encode NPC registry: %v", err) + t.Fatal(err) } - client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}} + client := &fakeSpellsLLMClient{content: []byte(`{"spell_casts":[]}`)} extractor := newExtractor(t, client) request := extractionRequest() request.References = spellNPCRegistryReference(canonical, "file:///generated.json") if _, err := extractor.Extract(context.Background(), request); err != nil { - t.Fatalf("Extract() error = %v", err) + t.Fatal(err) } - input := client.requests[0].Inputs[NPCRegistryReferenceSlot] - if input.Digest == "" || string(input.Content) != `{"npcs":[{"name":"Mira Thorn"}]}` || input.OriginURI != "" { - t.Fatalf("operation NPC input = %#v, want generated canonical grounding without provenance", input) + if !strings.Contains(promptText(client.requests[0]), `{"npcs":[{"name":"Mira Thorn"}]}`) { + t.Fatal("missing operation NPC projection") } - if metadata := extractor.ManifestMetadata(); metadata["npc_registry_digest"] != nil || metadata["npc_count"] != nil { - t.Fatalf("singleton metadata = %#v, want no operation-varying NPC identity", metadata) + if _, err := extractor.Extract(context.Background(), extractionRequest()); err != nil { + t.Fatal(err) } - if checkpointFingerprintMap(extractor.CheckpointFingerprints())["npc_registry"] == "" { - t.Fatalf("singleton fingerprints = %#v, want empty-projection identity", extractor.CheckpointFingerprints()) + if !strings.Contains(promptText(client.requests[1]), `{"npcs":[]}`) { + t.Fatal("operation override mutated seeded resolver") } } @@ -128,11 +100,3 @@ func spellNPCRegistryReference(content []byte, origin string) contracts.Referenc }, }} } - -func checkpointFingerprintMap(values []pipeline.CheckpointFingerprint) map[string]string { - result := make(map[string]string, len(values)) - for _, value := range values { - result[value.Name] = value.Value - } - return result -} diff --git a/internal/modules/dnd/extract/spells/pipeline_test.go b/internal/modules/dnd/extract/spells/pipeline_test.go new file mode 100644 index 00000000..339f2ea6 --- /dev/null +++ b/internal/modules/dnd/extract/spells/pipeline_test.go @@ -0,0 +1,203 @@ +package spells + +import ( + "context" + "fmt" + "os" + "path/filepath" + "reflect" + "strings" + "sync" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + spellcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/spells" + spellcatalogvalidator "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/catalog" + spellshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/shape" + spellsourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/source_refs" + spellsourcerelatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/source_relatedness" + "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/chunk/units" + "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder" + "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop" + "gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript" + "go.maximumdirect.net/promptkit" +) + +// Count the real validator boundary, not a second implementation of its rules. +type observedSpellValidator struct { + contracts.TypedValidator[dnd.SpellList] + record func(string) +} + +func (v observedSpellValidator) Validate(ctx context.Context, req contracts.TypedValidationRequest[dnd.SpellList]) (contracts.ValidationResult, error) { + v.record(v.Name()) + return v.TypedValidator.Validate(ctx, req) +} + +type spellPipelineOutput struct{} + +func (spellPipelineOutput) Key() string { return "test/spell-output" } +func (spellPipelineOutput) Encode(context.Context, contracts.OutputRequest) (contracts.OutputResult, error) { + return contracts.OutputResult{}, nil +} + +func TestNativeSpellPipelineAssessesEveryCandidateOnce(t *testing.T) { + response := func(name string) string { + return fmt.Sprintf(`{"spell_casts":[{"caster":"Aria","spell":%q,"source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`, name) + } + for _, test := range []struct { + name string + mode contracts.ExecutionMode + responses []string + wantError bool + fallbacks int + }{ + {"initial acceptance", contracts.ModeStrict, []string{response("Cure Wounds")}, false, 0}, + {"final correction accepted", contracts.ModeStrict, []string{response("Unknown Spell"), response("Unknown Spell"), response("Cure Wounds")}, false, 0}, + {"strict exhaustion", contracts.ModeStrict, []string{response("Unknown Spell"), response("Unknown Spell"), response("Unknown Spell")}, true, 0}, + {"permissive empty fallback", contracts.ModePermissive, []string{response("Unknown Spell"), response("Unknown Spell"), response("Unknown Spell")}, false, 1}, + } { + t.Run(test.name, func(t *testing.T) { + client := &fakeSpellsLLMClient{outputs: append([]string(nil), test.responses...)} + engine := newSpellEngine(t, client) + registries := pipeline.Registries{ + Inputs: pipeline.NewInputAdapterRegistry(), Chunkers: pipeline.NewChunkerRegistry(), + ArtifactCodecs: pipeline.NewArtifactCodecRegistry(), Extractors: pipeline.NewExtractorRegistry(), + Mergers: pipeline.NewMergerRegistry(), Normalizers: pipeline.NewNormalizerRegistry(), + Validators: pipeline.NewValidatorRegistry(), ValidatorChains: pipeline.NewValidatorChainRegistry(), + Outputs: pipeline.NewOutputEncoderRegistry(), + } + check := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + check(transcript.Register(registries.Inputs)) + check(units.Register(registries.Chunkers)) + check(pipeline.RegisterArtifactCodec(registries.ArtifactCodecs, spellcodec.New())) + check(Register(registries.Extractors)) + check(appendorder.RegisterTyped(registries.Mergers, dnd.SpellListKind, func(values []dnd.SpellList) (dnd.SpellList, error) { + result := dnd.SpellList{SpellCasts: []dnd.SpellCast{}} + for _, value := range values { + result.SpellCasts = append(result.SpellCasts, value.SpellCasts...) + } + return result, nil + })) + check(noop.RegisterTyped[dnd.SpellList](registries.Normalizers, dnd.SpellListKind)) + check(registries.Outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "test/spell-output", Stage: pipeline.StageOutput, ExecutionClass: contracts.ExecutionClassDeterministic}, + func() (contracts.OutputEncoder, error) { return spellPipelineOutput{}, nil })) + catalogValidator, err := spellcatalogvalidator.New(spellcatalogvalidator.Options{}) + check(err) + validators := []contracts.TypedValidator[dnd.SpellList]{ + spellshape.New(spellshape.Options{}), spellsourcerefs.New(spellsourcerefs.Options{}), + catalogValidator, spellsourcerelatedness.New(spellsourcerelatedness.Options{}), + } + var mu sync.Mutex + var calls []string + record := func(name string) { mu.Lock(); defer mu.Unlock(); calls = append(calls, name) } + var bindings []pipeline.ModuleBinding + var names []string + for _, validator := range validators { + check(pipeline.RegisterTypedValidator(registries.Validators, dnd.SpellListKind, + pipeline.ValidatorSpec{Key: validator.Name(), ExecutionClass: contracts.ExecutionClassDeterministic}, + func() (contracts.TypedValidator[dnd.SpellList], error) { + return observedSpellValidator{validator, record}, nil + })) + bindings = append(bindings, pipeline.Binding(validator.Name())) + names = append(names, validator.Name()) + } + partyFile := filepath.Join(t.TempDir(), "party.txt") + check(os.WriteFile(partyFile, []byte("party-grounding-sentinel: Aria"), 0600)) + turns := 2 + profile := pipeline.PipelineProfile{ + ID: "native-spells", Input: pipeline.Binding(transcript.Key), + Chunk: pipeline.Binding(units.Key), Output: pipeline.Binding("test/spell-output"), + LLMProfile: "profile-spells", + Artifacts: map[string]pipeline.ArtifactLaneProfile{"spells": { + Extract: pipeline.ModuleBinding{Module: Key, Mode: &test.mode, MaxAdditionalTurns: &turns, + Validators: pipeline.ValidatorOverride{Set: true, Validators: bindings}, + References: map[string]pipeline.ReferenceSource{"party": pipeline.ExternalReference(partyFile)}}, + Merge: pipeline.Binding(appendorder.Key), Normalize: pipeline.Binding(noop.Key), + }}, + } + catalog := pipeline.ModuleCatalog{ + Inputs: registries.Inputs, Chunkers: registries.Chunkers, ArtifactCodecs: registries.ArtifactCodecs, + Extractors: registries.Extractors, Mergers: registries.Mergers, Normalizers: registries.Normalizers, + Validators: registries.Validators, ValidatorChains: registries.ValidatorChains, Outputs: registries.Outputs, + } + resolved, err := pipeline.ResolvePipeline(profile, pipeline.ResolveOptions{}, catalog) + check(err) + resolved, _, err = pipeline.MaterializeReferences(resolved, catalog, pipeline.ReferenceMaterializationOptions{}) + check(err) + prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{Engine: engine}) + check(err) + model := "run-wide-spell-model" + var preparations []promptkit.PreparedRun + output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{ + Prepared: prepared, SourceID: "spell-session", SessionID: "spell-session-override", ExtractWorkers: 1, + RawInput: []byte(`{"segments":[{"id":1,"speaker":"Alice","start":0,"end":1,"text":"I cast Cure Wounds."}]}`), + Execution: contracts.ExecutionOptions{Execution: &promptkit.ExecutionTargetOverride{Model: model}, + OnPrepared: func(_ context.Context, value promptkit.PreparedRun) { preparations = append(preparations, value) }}, + }) + if (err != nil) != test.wantError { + t.Fatalf("pipeline error = %v, want error %t", err, test.wantError) + } + wantCalls := len(test.responses) + if len(client.requests) != wantCalls || len(preparations) != wantCalls { + t.Fatalf("provider/preparation calls = %d/%d, want %d", len(client.requests), len(preparations), wantCalls) + } + var expected []string + for range wantCalls { + expected = append(expected, names...) + } + if !reflect.DeepEqual(calls, expected) { + t.Fatalf("ordered assessments = %v, want %v (no outer assessment)", calls, expected) + } + for i, request := range client.requests { + if request.Target.Model != model || request.Prompt.SessionID != "spell-session-override" || preparations[i].SelectedProfileID != "profile-spells" { + t.Fatal("native profile/session/execution override lost") + } + wantID := PromptID + if i > 0 { + wantID = CorrectionPromptID + } + if preparations[i].PromptID != wantID { + t.Fatalf("prompt = %q, want %q", preparations[i].PromptID, wantID) + } + if i == 0 { + continue + } + previous := client.requests[i-1].Prompt.Messages + current := request.Prompt.Messages + if len(current) != len(previous)+2 || !reflect.DeepEqual(current[:len(previous)], previous) { + t.Fatal("native continuation did not retain exact ordered history") + } + if current[len(previous)].Role != "assistant" || current[len(previous)].Content != test.responses[i-1] || + !strings.Contains(current[len(previous)+1].Content, "recognized D&D spell") { + t.Fatal("correction lost the actual candidate or catalog guidance") + } + if strings.Count(promptText(request), "party-grounding-sentinel") != 1 { + t.Fatal("reference bytes were regenerated in correction") + } + } + if test.wantError { + return + } + if output.FallbackCount != test.fallbacks || len(output.NormalizeOutputs) != 1 { + t.Fatalf("outcome = %#v", output) + } + value, err := spellcodec.New().Decode(output.NormalizeOutputs[0].Artifact.Content) + check(err) + if test.fallbacks > 0 { + if value.SpellCasts == nil || len(value.SpellCasts) != 0 { + t.Fatalf("invalid empty fallback: %#v", value) + } + } else if len(value.SpellCasts) != 1 || value.SpellCasts[0].Spell != "Cure Wounds" || value.SpellCasts[0].SourceRefs[0].SourceID != "spell-session" { + t.Fatalf("durable spell artifact = %#v", value) + } + }) + } +} diff --git a/internal/modules/dnd/extract/spells/prompt_assets.go b/internal/modules/dnd/extract/spells/prompt_assets.go deleted file mode 100644 index 92558c93..00000000 --- a/internal/modules/dnd/extract/spells/prompt_assets.go +++ /dev/null @@ -1,76 +0,0 @@ -package spells - -import ( - "fmt" - "io/fs" - "sync" - - rootassets "gitea.maximumdirect.net/eric/notarius/assets" - "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - "gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs" - "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" -) - -const promptAssetRoot = "assets/prompts" - -var promptAssetManifest = shared.PromptAssetManifest{ - ModuleDir: "dnd.spells", - ModuleFiles: []promptfs.ModulePromptFile{ - {Name: "prompt.yaml", Path: "prompts/prompt.yaml"}, - {Name: "spell-catalog.md", Path: "prompts/spell-catalog.md"}, - {Name: "instructions.md", Path: "prompts/instructions.md"}, - }, - SharedFiles: []string{ - "common-dnd-system.md", - "common-dnd-extraction-evidence.md", - "common-dnd-identity.md", - "common-dnd-transcript-chunk.md", - "common-dnd-references.md", - "common-dnd-npc-registry.md", - }, -} - -func moduleAssetFS() (fs.FS, error) { - assets, err := fs.Sub(rootassets.FS(), "dnd/spells") - if err != nil { - return nil, fmt.Errorf("scope spell assets: %w", err) - } - return assets, nil -} - -func RegisterPromptAssets(registry *llm.AssetRegistry) error { - assets, err := moduleAssetFS() - if err != nil { - return err - } - promptFS, err := promptAssetManifest.PromptFS(assets) - if err != nil { - return fmt.Errorf("prepare spell prompt assets: %w", err) - } - if err := registry.RegisterPromptFS(promptFS, promptAssetRoot); err != nil { - return err - } - schemas, err := fs.Sub(assets, "schemas") - if err != nil { - return fmt.Errorf("scope spell schemas: %w", err) - } - return registry.RegisterSchemaFS(schemas, ".") -} - -func promptAssetMetadata() (string, error) { - promptAssetHashOnce.Do(func() { - assets, err := moduleAssetFS() - if err != nil { - promptAssetHashErr = err - return - } - promptAssetHash, promptAssetHashErr = promptAssetManifest.Hash(assets) - }) - return promptAssetHash, promptAssetHashErr -} - -var ( - promptAssetHashOnce sync.Once - promptAssetHash string - promptAssetHashErr error -) diff --git a/internal/modules/dnd/extract/spells/prompt_assets_test.go b/internal/modules/dnd/extract/spells/prompt_assets_test.go index 70a95354..a1ce8108 100644 --- a/internal/modules/dnd/extract/spells/prompt_assets_test.go +++ b/internal/modules/dnd/extract/spells/prompt_assets_test.go @@ -2,111 +2,57 @@ package spells import ( "context" - "encoding/json" + "reflect" "strings" "testing" - "time" - "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - "gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture" - "gitea.maximumdirect.net/eric/promptkit" + "go.maximumdirect.net/promptkit" ) -func TestPromptAssetsPrepareSpellPrompt(t *testing.T) { - transcript := []byte(`{"id":"session-1","segments":[{"id":"u1","text":"spell-transcript"}]}`) - prepared := prepareSpellsPrompt(t, transcript, "spell-player", "Mira: wizard", "Shield: abjuration") - - if prepared.PromptID != PromptID { - t.Fatalf("prompt id = %q, want %q", prepared.PromptID, PromptID) - } - if prepared.OutputContract.SchemaPath != "dnd_spells_llm.v1.json" { - t.Fatalf("schema path = %q, want LLM-only schema", prepared.OutputContract.SchemaPath) - } -} - -func TestPromptAssetsPrepareWithMissingOptionalReferences(t *testing.T) { - transcript := []byte(`{"id":"session-1","segments":[]}`) - prepared := prepareSpellsPrompt(t, transcript, " ", " ", " ") - - for _, message := range prepared.Messages { - if strings.Contains(message.Content, "Player list reference:") && strings.Contains(message.Content, "Party roster reference:") && strings.Contains(message.Content, "Glossary reference:") { - return +func TestSpellPromptPairSharesNativeContractAndDefaultProfile(t *testing.T) { + client := &fakeSpellsLLMClient{} + engine := newSpellEngine(t, client) + var initial promptkit.PreparedRun + for _, id := range []string{PromptID, CorrectionPromptID} { + inputs := map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline("Fix the rejected cast.")} + if id == PromptID { + inputs = map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline(`{"units":[1,2]}`), + "spell_catalog": promptkit.Inline(`{"spells":[]}`), + "npc_registry": promptkit.Inline(`{"npcs":[]}`), + "players": promptkit.Inline(" "), "party": promptkit.Inline(" "), "glossary": promptkit.Inline(" "), + } + } + handle, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: id, APIKey: "test-only", Inputs: inputs}) + if err != nil { + t.Fatal(err) + } + prepared := handle.Details() + handle.Discard() + if prepared.SelectedProfileID != "notarius/dnd-extraction" { + t.Fatalf("unexpected default profile %q", prepared.SelectedProfileID) + } + if prepared.StructuredOutput == nil || prepared.OutputContract.SchemaPath != "dnd/spells/schemas/dnd_spells_llm.v1.json" { + t.Fatal("missing native spell response schema") + } + if id == PromptID { + initial = prepared + var system string + for _, message := range prepared.Messages { + if message.Role == "system" { + system += message.Content + } + } + if !strings.Contains(system, "canonical spelling exactly") || !strings.Contains(system, "in-world caster") { + t.Fatal("enduring spell constraints missing from initial system context") + } + } else { + if !reflect.DeepEqual(initial.OutputContract, prepared.OutputContract) || len(prepared.Messages) != 1 || prepared.Messages[0].Role != "user" || !strings.Contains(prepared.Messages[0].Content, "Fix the rejected cast.") { + t.Fatal("correction must supply only findings and a replacement assignment with the same contract") + } } } - t.Fatalf("reference message did not render empty optional reference placeholders") -} - -func TestPromptAssetDiagnosticsOmitRawMaterials(t *testing.T) { - transcript := []byte(`{"secret":"source text"}`) - reference := "private party note" - prepared := prepareSpellsPrompt(t, transcript, "private player note", reference, " ") - metadata := newExtractor(t, &fakeSpellsLLMClient{}, overlaySpellCatalogReference()).ManifestMetadata() - - payload, err := json.Marshal(map[string]any{ - "prepared": map[string]any{ - "prompt_id": prepared.PromptID, - "prompt_version": prepared.PromptVersion, - "prompt_hash": prepared.PromptHash, - "rendered_prompt_hash": prepared.RenderedPromptHash, - "selected_profile_id": prepared.SelectedProfileID, - "output_contract": prepared.OutputContract, - "input_hashes": prepared.InputHashes, - "effective_model_params": prepared.EffectiveModelParams, - }, - "manifest": metadata, - }) - if err != nil { - t.Fatalf("marshal diagnostics: %v", err) - } - diagnostics := string(payload) - for _, forbidden := range []string{ - "source text", - "private player note", - reference, - "Cure Wounds", - "Aegis of Emberfall", - "Emberfall Aegis", - "Private campaign source", - "file:///private-source.json", - `"properties"`, - "spell_casts", - } { - if strings.Contains(diagnostics, forbidden) { - t.Fatalf("diagnostics leaked %q: %s", forbidden, diagnostics) - } - } - if metadata["prompt_id"] != PromptID || metadata["prompt_version"] != SchemaVersion { - t.Fatalf("manifest prompt metadata = %#v", metadata) - } - if !strings.HasPrefix(metadata["prompt_sha256"].(string), "sha256:") { - t.Fatalf("manifest prompt hash = %#v, want sha256-prefixed", metadata["prompt_sha256"]) + if len(client.requests) != 0 { + t.Fatal("preparation invoked generation") } } - -func prepareSpellsPrompt(t *testing.T, transcript []byte, players string, party string, glossary string) *promptkit.PreparedRun { - t.Helper() - registry := llm.NewAssetRegistry() - if err := RegisterPromptAssets(registry); err != nil { - t.Fatalf("register spell prompt assets: %v", err) - } - engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{ - "spell-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "spell-test-model"), - }) - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: PromptID, - PromptVersion: SchemaVersion, - ProfileID: "spell-test-profile", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.InlineWithURI("file:///session.json", string(transcript)), - "spell_catalog": promptkit.Inline(`{"spells":[{"canonical_name":"spell-catalog-sentinel","aliases":["spell-alias-sentinel"]}]}`), - "npc_registry": promptkit.Inline(`{"npcs":[{"name":"spell-npc-sentinel"}]}`), - "players": promptkit.Inline(players), - "party": promptkit.Inline(party), - "glossary": promptkit.Inline(glossary), - }, - }) - if err != nil { - t.Fatalf("Prepare() error = %v, want nil", err) - } - return prepared -} diff --git a/internal/modules/dnd/extract/spells/registry_test.go b/internal/modules/dnd/extract/spells/registry_test.go index 8c45b7cc..a345284e 100644 --- a/internal/modules/dnd/extract/spells/registry_test.go +++ b/internal/modules/dnd/extract/spells/registry_test.go @@ -10,9 +10,9 @@ import ( "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" ) -func TestNewRequiresLLMClientAndReturnsExtractor(t *testing.T) { - if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "LLM client") { - t.Fatalf("New(nil) error = %v, want LLM client error", err) +func TestNewRequiresNativeEngineAndReturnsExtractor(t *testing.T) { + if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "native engine") { + t.Fatalf("New(nil) error = %v, want native engine error", err) } extractor := newExtractor(t, &fakeSpellsLLMClient{}) if extractor.Key() != Key { @@ -23,10 +23,10 @@ func TestNewRequiresLLMClientAndReturnsExtractor(t *testing.T) { func TestModuleSpec(t *testing.T) { got := ModuleSpec() want := pipeline.ModuleSpec{ - Key: Key, - Stage: pipeline.StageExtract, - ExecutionClass: contracts.ExecutionClassLLMBacked, - CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1, + Key: Key, + Stage: pipeline.StageExtract, + ExecutionClass: contracts.ExecutionClassLLMBacked, + Prompts: []string{PromptID, CorrectionPromptID}, Requires: []string{ "chunks", "source.transcript", @@ -75,6 +75,7 @@ func TestModuleSpec(t *testing.T) { t.Fatalf("ModuleSpec() = %#v, want %#v", got, want) } + got.Prompts[0] = "changed" got.Requires[0] = "changed" got.Provides[0] = "changed" got.ReferenceSlots[0].AcceptedMediaTypes[0] = "changed" diff --git a/internal/modules/dnd/extract/spells/schema.go b/internal/modules/dnd/extract/spells/schema.go deleted file mode 100644 index b073e4a9..00000000 --- a/internal/modules/dnd/extract/spells/schema.go +++ /dev/null @@ -1,24 +0,0 @@ -package spells - -import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - -const ( - PromptID = "dnd.spells" - ResponseSchemaKey = llm.ResponseSchemaKey("dnd_spells_llm") - ResponseSchemaID = "notarius.dnd.spells.llm" - ResponseSchemaName = "notarius_dnd_spells_llm_v1" -) - -func loadResponseSchema() (llm.ResponseSchema, error) { - assets, err := moduleAssetFS() - if err != nil { - return llm.ResponseSchema{}, err - } - return llm.LoadResponseSchema(assets, llm.ResponseSchemaDefinition{ - Key: ResponseSchemaKey, - ID: ResponseSchemaID, - Version: SchemaVersion, - Name: ResponseSchemaName, - AssetPath: "schemas/dnd_spells_llm.v1.json", - }) -} diff --git a/internal/modules/dnd/extract/spells/schema_test.go b/internal/modules/dnd/extract/spells/schema_test.go index 8c625284..7184e419 100644 --- a/internal/modules/dnd/extract/spells/schema_test.go +++ b/internal/modules/dnd/extract/spells/schema_test.go @@ -3,38 +3,18 @@ package spells import ( "bytes" "encoding/json" - "strings" + "io/fs" "testing" + "gitea.maximumdirect.net/eric/notarius/assets" + "github.com/santhosh-tekuri/jsonschema/v6" ) -func TestLoadResponseSchemaUsesExtractorOwnedLLMSchema(t *testing.T) { - schema, err := loadResponseSchema() +func TestSpellResponseSchemaPreservesPrivateSelectionContract(t *testing.T) { + schema, err := fs.ReadFile(assets.FS(), "dnd/spells/schemas/dnd_spells_llm.v1.json") if err != nil { - t.Fatalf("loadResponseSchema() error = %v, want nil", err) - } - if got, want := string(schema.Key), "dnd_spells_llm"; got != want { - t.Fatalf("schema key = %q, want %q", got, want) - } - if got, want := schema.ID, "notarius.dnd.spells.llm"; got != want { - t.Fatalf("schema ID = %q, want %q", got, want) - } - if got, want := schema.Version, "v1"; got != want { - t.Fatalf("schema version = %q, want %q", got, want) - } - if got, want := schema.Name, "notarius_dnd_spells_llm_v1"; got != want { - t.Fatalf("schema name = %q, want %q", got, want) - } - if !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) { - t.Fatalf("schema metadata = %#v, want valid hashed JSON", schema) - } - var schemaDocument map[string]any - if err := json.Unmarshal(schema.JSONSchema, &schemaDocument); err != nil { - t.Fatalf("Unmarshal(schema.JSONSchema) error = %v, want nil", err) - } - if schemaDocument["$id"] != "notarius.dnd.spells.llm" { - t.Fatalf("schema $id = %#v, want private LLM schema ID", schemaDocument["$id"]) + t.Fatal(err) } valid := validSpellsResponse() @@ -42,7 +22,7 @@ func TestLoadResponseSchemaUsesExtractorOwnedLLMSchema(t *testing.T) { if err != nil { t.Fatalf("Marshal(validSpellsResponse()) error = %v, want nil", err) } - if err := validateJSONSchema(validJSON, schema.JSONSchema); err != nil { + if err := validateJSONSchema(validJSON, schema); err != nil { t.Fatalf("valid private spells response rejected: %v", err) } for _, test := range []struct { @@ -96,13 +76,13 @@ func TestLoadResponseSchemaUsesExtractorOwnedLLMSchema(t *testing.T) { if err != nil { t.Fatal(err) } - err = validateJSONSchema(content, schema.JSONSchema) + err = validateJSONSchema(content, schema) if (err == nil) != test.valid { t.Fatalf("validateJSONSchema() error = %v, want valid=%t", err, test.valid) } }) } - if err := validateJSONSchema([]byte(`{"spell_casts":`), schema.JSONSchema); err == nil { + if err := validateJSONSchema([]byte(`{"spell_casts":`), schema); err == nil { t.Fatal("validateJSONSchema() error = nil, want malformed JSON rejected") } @@ -112,37 +92,11 @@ func TestLoadResponseSchemaUsesExtractorOwnedLLMSchema(t *testing.T) { if err != nil { t.Fatalf("Marshal(response with source_id) error = %v, want nil", err) } - if err := validateJSONSchema(content, schema.JSONSchema); err == nil { + if err := validateJSONSchema(content, schema); err == nil { t.Fatal("validateJSONSchema() error = nil, want canonical source_id rejected") } } -func TestResponseSchemaJSONIsMutationSafe(t *testing.T) { - first, err := loadResponseSchema() - if err != nil { - t.Fatalf("loadResponseSchema() error = %v", err) - } - first.JSONSchema[0] = '[' - second, err := loadResponseSchema() - if err != nil || !json.Valid(second.JSONSchema) || bytes.Equal(first.JSONSchema, second.JSONSchema) { - t.Fatalf("second schema = %s, %v; want defensive valid copy", second.JSONSchema, err) - } -} - -func TestResponseSchemaDiagnosticsOmitRawSchema(t *testing.T) { - schema, err := loadResponseSchema() - if err != nil { - t.Fatalf("loadResponseSchema() error = %v", err) - } - diagnostics := schema.DiagnosticsMap() - if diagnostics["key"] != ResponseSchemaKey { - t.Fatalf("diagnostics = %#v, want response key", diagnostics) - } - if _, ok := diagnostics["json_schema"]; ok { - t.Fatalf("diagnostics include raw schema: %#v", diagnostics) - } -} - func validSpellsResponse() map[string]any { return map[string]any{ "spell_casts": []any{ diff --git a/internal/modules/dnd/extract/spells/test_helpers_test.go b/internal/modules/dnd/extract/spells/test_helpers_test.go index c4c1d925..68a26c2f 100644 --- a/internal/modules/dnd/extract/spells/test_helpers_test.go +++ b/internal/modules/dnd/extract/spells/test_helpers_test.go @@ -3,15 +3,20 @@ package spells import ( "context" "encoding/json" - "errors" + "strings" "testing" + "gitea.maximumdirect.net/eric/notarius/assets" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + "go.maximumdirect.net/configkit" + "go.maximumdirect.net/promptkit" + "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" spellcatalog "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/spells/catalog" ) -func promptExtractionRequest() contracts.TypedExtractionRequest { +func promptExtractionRequest() contracts.TypedExtractionRequest[dnd.SpellList] { doc := promptSourceDocument() chunk := &source.Chunk{ ID: "session-alpha:chunk:0", @@ -27,9 +32,12 @@ func promptExtractionRequest() contracts.TypedExtractionRequest { Units: append([]source.SourceUnit(nil), doc.Units...), Metadata: map[string]any{"ignored": "chunk metadata"}, } - return contracts.TypedExtractionRequest{ + return contracts.TypedExtractionRequest[dnd.SpellList]{ Source: doc, Chunk: chunk, + Operation: contracts.ProducerOperation[dnd.SpellList]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, Cache: contracts.CacheBypass}, ReuseEligible: true, Assess: func(context.Context, dnd.SpellList) (contracts.CandidateAssessment, error) { + return contracts.CandidateAssessment{}, nil + }}, } } @@ -91,7 +99,7 @@ func spellChunkInput(chunk *source.Chunk) contracts.LLMInputMaterial { return contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session-alpha.json") } -func extractionRequest() contracts.TypedExtractionRequest { +func extractionRequest() contracts.TypedExtractionRequest[dnd.SpellList] { req := promptExtractionRequest() req.SourceInput = spellChunkInput(req.Chunk) req.SessionID = "session-123" @@ -99,14 +107,14 @@ func extractionRequest() contracts.TypedExtractionRequest { return req } -func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest) contracts.TypedExtractionRequest { +func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest[dnd.SpellList]) contracts.TypedExtractionRequest[dnd.SpellList] { req.SourceInput = spellSourceInput() return req } -func newExtractor(t *testing.T, client contracts.StructuredLLMClient, references ...contracts.ReferenceSet) *Extractor { +func newExtractor(t *testing.T, client promptkit.LLMClient, references ...contracts.ReferenceSet) *Extractor { t.Helper() - extractor, err := New(client, Options{}, references...) + extractor, err := New(newSpellEngine(t, client), Options{}, references...) if err != nil { t.Fatalf("New() error = %v, want nil", err) } @@ -133,49 +141,64 @@ type fakeSpellsLLMClient struct { response extractionResponse content []byte err error - requests []contracts.StructuredCompletionRequest + requests []promptkit.GenerateRequest + outputs []string } -func (client *fakeSpellsLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { - client.requests = append(client.requests, cloneStructuredCompletionRequest(req)) +func (client *fakeSpellsLLMClient) Generate(_ context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + client.requests = append(client.requests, req) if client.err != nil { - return contracts.StructuredCompletionResponse{}, client.err + return nil, client.err } - target, ok := out.(*extractionResponse) - if !ok { - return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target") - } - content := append([]byte(nil), client.content...) - if len(content) != 0 { - if err := json.Unmarshal(content, target); err != nil { - return contracts.StructuredCompletionResponse{}, err + if len(client.outputs) > 0 { + content := client.outputs[0] + if len(client.outputs) > 1 { + client.outputs = client.outputs[1:] } - } else { - *target = client.response + return &promptkit.GenerateResponse{Content: content}, nil + } + content := client.content + if len(content) == 0 { var err error content, err = json.Marshal(client.response) if err != nil { - return contracts.StructuredCompletionResponse{}, err + return nil, err } } - return contracts.StructuredCompletionResponse{Content: content}, nil + return &promptkit.GenerateResponse{Content: string(content)}, nil } -func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest { - req.Inputs = req.Inputs.Clone() - correction, err := contracts.CloneSemanticCorrection(req.Correction) - if err != nil { - panic(err) +func promptText(request promptkit.GenerateRequest) string { + var text []string + for _, message := range request.Prompt.Messages { + text = append(text, message.Content) } - req.Correction = correction - if len(req.Vars) == 0 { - req.Vars = nil - return req - } - vars := make(map[string]any, len(req.Vars)) - for key, value := range req.Vars { - vars[key] = value - } - req.Vars = vars - return req + return strings.Join(text, "\n") +} + +func newSpellEngine(t *testing.T, client promptkit.LLMClient) *promptkit.Engine { + t.Helper() + type settings struct { + PromptKit promptkit.Settings `yaml:"promptkit"` + } + definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit }) + filesystem := configkit.NewFilesystem("Notarius assets", assets.FS()) + config, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("spell provider fixture", []byte(`promptkit: + profiles: + - id: profile-spells + endpoint: https://example.invalid/v1 + model: spell-test-model + api_key_required: false +`))}, configkit.LayeredOptions[settings]{Definition: definition, Underlays: []configkit.Underlay{ + promptkit.BundledOpenRouterUnderlay(), + {Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}}, + }}) + if err != nil { + t.Fatal(err) + } + engine, err := promptkit.NewFromConfig(config, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client)) + if err != nil { + t.Fatal(err) + } + return engine } diff --git a/internal/modules/dnd/shared/extraction_inputs.go b/internal/modules/dnd/shared/extraction_inputs.go index 5a9a442b..adc96a72 100644 --- a/internal/modules/dnd/shared/extraction_inputs.go +++ b/internal/modules/dnd/shared/extraction_inputs.go @@ -10,7 +10,7 @@ import ( // PrepareChunkExtraction validates common D&D extraction prerequisites and // prepares owned chunk-scoped source material for prompt inputs. -func PrepareChunkExtraction(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.LLMInputMaterial, error) { +func PrepareChunkExtraction[T any](ctx context.Context, req contracts.TypedExtractionRequest[T]) (contracts.LLMInputMaterial, error) { if ctx == nil { return contracts.LLMInputMaterial{}, fmt.Errorf("context must not be nil") } diff --git a/internal/modules/dnd/shared/extraction_inputs_test.go b/internal/modules/dnd/shared/extraction_inputs_test.go index e6b8c87b..0ed7ebde 100644 --- a/internal/modules/dnd/shared/extraction_inputs_test.go +++ b/internal/modules/dnd/shared/extraction_inputs_test.go @@ -16,7 +16,7 @@ func TestPrepareChunkExtraction(t *testing.T) { tests := []struct { name string ctx context.Context - configure func(*contracts.TypedExtractionRequest) + configure func(*contracts.TypedExtractionRequest[any]) sourceInput contracts.LLMInputMaterial want contracts.LLMInputMaterial wantErr string @@ -33,21 +33,21 @@ func TestPrepareChunkExtraction(t *testing.T) { }, { name: "nil source", - configure: func(req *contracts.TypedExtractionRequest) { + configure: func(req *contracts.TypedExtractionRequest[any]) { req.Source = nil }, wantErr: "source must not be nil", }, { name: "nil chunk", - configure: func(req *contracts.TypedExtractionRequest) { + configure: func(req *contracts.TypedExtractionRequest[any]) { req.Chunk = nil }, wantErr: "chunk must not be nil", }, { name: "empty chunk units", - configure: func(req *contracts.TypedExtractionRequest) { + configure: func(req *contracts.TypedExtractionRequest[any]) { req.Chunk = &source.Chunk{ID: req.Chunk.ID} }, wantErr: "units must not be empty", @@ -139,7 +139,7 @@ func TestPrepareChunkExtraction(t *testing.T) { } } -func extractionRequest() contracts.TypedExtractionRequest { +func extractionRequest() contracts.TypedExtractionRequest[any] { doc := &source.SourceDocument{ID: "session-alpha"} chunk := &source.Chunk{ ID: "session-alpha:chunk:0", @@ -148,5 +148,5 @@ func extractionRequest() contracts.TypedExtractionRequest { MediaType: "application/json", Units: []source.SourceUnit{{ID: 1}}, } - return contracts.TypedExtractionRequest{Source: doc, Chunk: chunk} + return contracts.TypedExtractionRequest[any]{Source: doc, Chunk: chunk} } diff --git a/internal/modules/dnd/validate/spells/catalog/validator_test.go b/internal/modules/dnd/validate/spells/catalog/validator_test.go index 3ddf9b0e..0b216349 100644 --- a/internal/modules/dnd/validate/spells/catalog/validator_test.go +++ b/internal/modules/dnd/validate/spells/catalog/validator_test.go @@ -137,7 +137,7 @@ func TestValidatorSpecAndRegister(t *testing.T) { t.Fatalf("Register() error = %v, want nil", err) } registered, ok := registry.Spec(Key) - if !ok || registered != spec { + if !ok || !reflect.DeepEqual(registered, spec) { t.Fatalf("registry.Spec(%q) = %#v, present = %t; want %#v", Key, registered, ok, spec) } }