diff --git a/assets/dnd/item-registry/extract/prompts/correction.md b/assets/dnd/item-registry/extract/prompts/correction.md new file mode 100644 index 00000000..5964bb69 --- /dev/null +++ b/assets/dnd/item-registry/extract/prompts/correction.md @@ -0,0 +1,7 @@ +Revise the previous item registry using these findings: + +{{ input "guidance" }} + +Return the complete replacement registry, not only the changed records. Keep the +original transcript and reference context and return only JSON matching the +original response contract. diff --git a/assets/dnd/item-registry/extract/prompts/prompt.yaml b/assets/dnd/item-registry/extract/prompts/prompt.yaml index a8bd4d80..6c67d1eb 100644 --- a/assets/dnd/item-registry/extract/prompts/prompt.yaml +++ b/assets/dnd/item-registry/extract/prompts/prompt.yaml @@ -17,7 +17,7 @@ prompts: messages: - role: system content_file: ../../../shared/prompts/common-dnd-system.md - - role: user + - role: system content_file: ../../../shared/prompts/common-dnd-identity.md - role: user content_file: ../../../shared/prompts/common-dnd-references.md @@ -27,9 +27,9 @@ prompts: content_file: ../../../shared/prompts/common-dnd-transcript-chunk.md cache_control: type: ephemeral - - role: user + - role: system content_file: ../../../shared/prompts/common-dnd-extraction-evidence.md - - role: user + - role: system content_file: ./instructions.md cache_control: type: ephemeral @@ -37,3 +37,17 @@ prompts: format: json validation_mode: json_schema schema_path: dnd/item-registry/extract/schemas/dnd_item_registry_llm.v1.json + + - id: dnd.item_registry.repair + default_profile: notarius/dnd-extraction + inputs: + - name: guidance + required: true + content_type: text/plain + messages: + - role: user + content_file: ./correction.md + output: + format: json + validation_mode: json_schema + schema_path: dnd/item-registry/extract/schemas/dnd_item_registry_llm.v1.json diff --git a/assets/dnd/location-registry/extract/prompts/correction.md b/assets/dnd/location-registry/extract/prompts/correction.md new file mode 100644 index 00000000..1a0c19be --- /dev/null +++ b/assets/dnd/location-registry/extract/prompts/correction.md @@ -0,0 +1,7 @@ +Revise the previous location registry using these findings: + +{{ input "guidance" }} + +Return the complete replacement registry, not only the changed records. Keep the +original transcript and reference context and return only JSON matching the +original response contract. diff --git a/assets/dnd/location-registry/extract/prompts/prompt.yaml b/assets/dnd/location-registry/extract/prompts/prompt.yaml index af80929f..3e009bcc 100644 --- a/assets/dnd/location-registry/extract/prompts/prompt.yaml +++ b/assets/dnd/location-registry/extract/prompts/prompt.yaml @@ -17,7 +17,7 @@ prompts: messages: - role: system content_file: ../../../shared/prompts/common-dnd-system.md - - role: user + - role: system content_file: ../../../shared/prompts/common-dnd-identity.md - role: user content_file: ../../../shared/prompts/common-dnd-references.md @@ -27,9 +27,9 @@ prompts: content_file: ../../../shared/prompts/common-dnd-transcript-chunk.md cache_control: type: ephemeral - - role: user + - role: system content_file: ../../../shared/prompts/common-dnd-extraction-evidence.md - - role: user + - role: system content_file: ./instructions.md cache_control: type: ephemeral @@ -37,3 +37,17 @@ prompts: format: json validation_mode: json_schema schema_path: dnd/location-registry/extract/schemas/dnd_location_registry_llm.v1.json + + - id: dnd.location_registry.repair + default_profile: notarius/dnd-extraction + inputs: + - name: guidance + required: true + content_type: text/plain + messages: + - role: user + content_file: ./correction.md + output: + format: json + validation_mode: json_schema + schema_path: dnd/location-registry/extract/schemas/dnd_location_registry_llm.v1.json diff --git a/assets/dnd/npc-registry/extract/prompts/correction.md b/assets/dnd/npc-registry/extract/prompts/correction.md new file mode 100644 index 00000000..49018024 --- /dev/null +++ b/assets/dnd/npc-registry/extract/prompts/correction.md @@ -0,0 +1,7 @@ +Revise the previous NPC registry using these findings: + +{{ input "guidance" }} + +Return the complete replacement registry, not only the changed records. Keep the +original transcript and reference context and return only JSON matching the +original response contract. diff --git a/assets/dnd/npc-registry/extract/prompts/prompt.yaml b/assets/dnd/npc-registry/extract/prompts/prompt.yaml index 1b209d1c..d4fa4100 100644 --- a/assets/dnd/npc-registry/extract/prompts/prompt.yaml +++ b/assets/dnd/npc-registry/extract/prompts/prompt.yaml @@ -17,7 +17,7 @@ prompts: messages: - role: system content_file: ../../../shared/prompts/common-dnd-system.md - - role: user + - role: system content_file: ../../../shared/prompts/common-dnd-identity.md - role: user content_file: ../../../shared/prompts/common-dnd-references.md @@ -27,9 +27,9 @@ prompts: content_file: ../../../shared/prompts/common-dnd-transcript-chunk.md cache_control: type: ephemeral - - role: user + - role: system content_file: ../../../shared/prompts/common-dnd-extraction-evidence.md - - role: user + - role: system content_file: ./instructions.md cache_control: type: ephemeral @@ -37,3 +37,17 @@ prompts: format: json validation_mode: json_schema schema_path: dnd/npc-registry/extract/schemas/dnd_npc_registry_llm.v1.json + + - id: dnd.npc_registry.repair + default_profile: notarius/dnd-extraction + inputs: + - name: guidance + required: true + content_type: text/plain + messages: + - role: user + content_file: ./correction.md + output: + format: json + validation_mode: json_schema + schema_path: dnd/npc-registry/extract/schemas/dnd_npc_registry_llm.v1.json diff --git a/assets/registry_extraction_test.go b/assets/registry_extraction_test.go new file mode 100644 index 00000000..4eb10313 --- /dev/null +++ b/assets/registry_extraction_test.go @@ -0,0 +1,120 @@ +package assets_test + +import ( + "context" + "encoding/json" + "io/fs" + "reflect" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/assets" + "go.maximumdirect.net/promptkit" +) + +func TestRegistryExtractionPromptPairsRetainDomainRulesAndContracts(t *testing.T) { + _, engine := loadNativeAssets(t) + for _, test := range []struct { + family, instruction string + rules []string + }{ + {"npc", "Revise the NPC citations.", []string{"proper name or a stable, individually distinguishing", "factual third-party mention", "Exclude human players"}}, + {"item", "Revise the item citations.", []string{"named unique items", "concrete reusable item types", "currency denomination", "generic weapons", "Do not use capitalization as an eligibility test"}}, + {"location", "Revise the location citations.", []string{"stable proper name or unique in-world designation", "the room", "Do not use capitalization as an eligibility test"}}, + } { + t.Run(test.family, func(t *testing.T) { + id := "notarius/dnd." + test.family + "_registry" + initial, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{ + PromptID: id, APIKey: "test-only", + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline(`{"units":[{"sentinel":"registry-transcript"}]}`), + "players": promptkit.Inline("registry-players"), "party": promptkit.Inline("registry-party"), "glossary": promptkit.Inline("registry-glossary"), + }, + }) + if err != nil { + t.Fatal(err) + } + defer initial.Discard() + next, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{ + PromptID: id + ".repair", APIKey: "test-only", Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline(test.instruction)}, + }) + if err != nil { + t.Fatal(err) + } + defer next.Discard() + first, correction := initial.Details(), next.Details() + if first.SelectedProfileID != "notarius/dnd-extraction" || correction.SelectedProfileID != first.SelectedProfileID || + first.StructuredOutput == nil || correction.StructuredOutput == nil || !reflect.DeepEqual(first.OutputContract, correction.OutputContract) { + t.Fatal("registry prompt pair lost native schema/defaults") + } + var system, all string + for _, message := range first.Messages { + all += message.Content + if message.Role == "system" { + system += message.Content + } + if strings.Contains(message.Content, "registry-transcript") || strings.Contains(message.Content, "registry-players") { + if message.CacheControl == nil || message.CacheControl.Type != promptkit.CacheControlEphemeral { + t.Fatal("fixed input cache hint lost") + } + } + } + for _, rule := range append(test.rules, "Transcript units are the only evidence", "Use the most specific supported in-world") { + if !strings.Contains(strings.Join(strings.Fields(system), " "), rule) { + t.Fatalf("missing enduring domain rule %q", rule) + } + } + for _, input := range []string{"registry-transcript", "registry-players", "registry-party", "registry-glossary"} { + if strings.Count(all, input) != 1 { + t.Fatalf("fixed input not rendered once: %q", input) + } + } + if len(correction.Messages) != 1 || correction.Messages[0].Role != "user" || !strings.Contains(correction.Messages[0].Content, test.instruction) || + strings.Contains(correction.Messages[0].Content, "registry-transcript") { + t.Fatal("correction repeated fixed inputs instead of supplying findings/assignment") + } + }) + } +} + +func TestRegistryResponseSchemasSeparateStructureFromSemanticValidity(t *testing.T) { + for _, test := range []struct{ family, field string }{ + {"npc", "npcs"}, {"item", "items"}, {"location", "locations"}, + } { + t.Run(test.family, func(t *testing.T) { + schema, err := fs.ReadFile(assets.FS(), "dnd/"+test.family+"-registry/extract/schemas/dnd_"+test.family+"_registry_llm.v1.json") + if err != nil { + t.Fatal(err) + } + for _, value := range []struct { + name, record string + valid bool + }{ + {"grounded record", `{"name":"Canonical Name","source_refs":[{"start_unit_id":1,"end_unit_id":2}]}`, true}, + {"blank name and empty evidence", `{"name":"","source_refs":[]}`, true}, + {"nonpositive source candidate", `{"name":"Canonical Name","source_refs":[{"start_unit_id":0,"end_unit_id":-1}]}`, true}, + {"missing name", `{"source_refs":[]}`, false}, + {"opaque ID", `{"name":"Canonical Name","source_refs":[],"id":"assigned-later"}`, false}, + {"wrong name type", `{"name":7,"source_refs":[]}`, false}, + {"noninteger source identifier", `{"name":"Canonical Name","source_refs":[{"start_unit_id":1.5,"end_unit_id":2}]}`, false}, + } { + t.Run(value.name, func(t *testing.T) { + content := []byte(`{"` + test.field + `":[` + value.record + `]}`) + if err := validateJSONSchema(content, schema); (err == nil) != value.valid { + t.Fatalf("schema validity=%t, error=%v", value.valid, err) + } + }) + } + for _, content := range []string{`{"` + test.field + `":[]}`, `{"` + test.field + `":null}`, `{}`} { + var value map[string]any + if err := json.Unmarshal([]byte(content), &value); err != nil { + t.Fatal(err) + } + _, isArray := value[test.field].([]any) + if err := validateJSONSchema([]byte(content), schema); (err == nil) != isArray { + t.Fatalf("collection shape %s: %v", content, err) + } + } + }) + } +} diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index def8339b..f36aafb7 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -811,6 +811,29 @@ nested same-backend admission with a limit of one. ## Stage 18: Migrate The Three Registry Extractors +**Status:** Complete. NPC, item, and location extraction now use native initial/ +correction requests through the shared typed producer helper. Transcript and +campaign reference materials are captured once per chunk; every response uses +the existing family-specific canonicalizer and deterministic identity rules. +Added same-schema/default-profile correction pairs and static dependencies, +moving enduring identity/evidence/domain principles into initial system context. +Private completion, schema/asset registration, and manifest/fingerprint metadata +were removed. Canonical naming, currency scope, same-name location anchors, +source-order/evidence behavior, and production validator-chain declarations are +unchanged. Each family has a codec-valid nonnil empty fallback, qualified by the +resolved mode/outcome and excluded from reuse, rather than an invented entity. +Retained domain regressions with native provider fakes and added per-family +citation correction through real typed validators, fixed input/history and run +override checks, strict/permissive exhaustion, ordinary empty acceptance, failed +continuation, structural failure, and cancellation coverage. Prompt-pair and +structural-versus-semantic schema checks run independently in the asset package. +Asset/shape tests and vet/race checks, focused pipeline tests, and +`git diff --check` pass. Updated the location-shape spec comparison for the +slice-bearing native dependency declaration. All three extractor package tests +remain blocked at setup by shared legacy asset helpers importing old ConfigKit's +removed YAML `plugin/limit`; their new runtime tests are not claimed passing. +Scheduled dependency cleanup and final assembled verification must execute them. + **Goal:** canonical noun extraction follows the representative native path. **Inspect/change:** `internal/modules/dnd/extract/{npcregistry,itemregistry, diff --git a/internal/modules/dnd/extract/itemregistry/extractor.go b/internal/modules/dnd/extract/itemregistry/extractor.go index 18be5320..7d83b6cd 100644 --- a/internal/modules/dnd/extract/itemregistry/extractor.go +++ b/internal/modules/dnd/extract/itemregistry/extractor.go @@ -3,19 +3,21 @@ package itemregistry import ( "context" + "encoding/json" "fmt" + "strings" + + "go.maximumdirect.net/promptkit" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" - "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" ) -const ( - Key = "dnd/item-registry" - mappingPolicy = "dnd.item_registry.extract_mapping.v2" -) +const Key = "dnd/item-registry" +const PromptID = "notarius/dnd.item_registry" +const CorrectionPromptID = "notarius/dnd.item_registry.repair" var requiredCapabilities = []string{"chunks", "source.transcript"} var providedCapabilities = []string{"dnd.item_registry"} @@ -32,75 +34,31 @@ func referenceSlots() []contracts.ReferenceSlot { } var _ contracts.Extractor[dnd.ItemRegistry] = (*Extractor)(nil) -var _ contracts.ManifestMetadataProvider = (*Extractor)(nil) -var _ pipeline.CheckpointFingerprintProvider = (*Extractor)(nil) type Options struct{} -type Extractor struct { - llm contracts.StructuredLLMClient - promptSHA string - responseSchemaSHA string -} +type Extractor struct{ engine *promptkit.Engine } -func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) { - if llmClient == nil { - return nil, extractorErrorf("LLM client must not be nil") +func New(engine *promptkit.Engine, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) { + if engine == nil { + return nil, extractorErrorf("native engine must not be nil") } if len(references) > 1 { return nil, extractorErrorf("at most one reference set may be supplied") } - promptSHA, err := promptAssetMetadata() - if err != nil { - return nil, extractorErrorf("load prompt metadata: %w", err) - } - responseSchema, err := loadResponseSchema() - if err != nil { - return nil, extractorErrorf("load response schema: %w", err) - } - return &Extractor{llm: llmClient, promptSHA: promptSHA, responseSchemaSHA: responseSchema.SHA256}, nil + return &Extractor{engine: engine}, nil } func (e *Extractor) Key() string { return Key } func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot { return referenceSlots() } -func (e *Extractor) ManifestMetadata() map[string]any { - if e == nil { - return nil - } - return map[string]any{ - "prompt_id": PromptID, - "prompt_version": SchemaVersion, - "prompt_sha256": e.promptSHA, - "response_schema_key": string(ResponseSchemaKey), - "response_schema_id": ResponseSchemaID, - "response_schema_name": ResponseSchemaName, - "response_schema_version": SchemaVersion, - "response_schema_sha256": e.responseSchemaSHA, - "identity_policy": identity.Policy, - "mapping_policy": mappingPolicy, - } -} - -func (e *Extractor) CheckpointFingerprints() []pipeline.CheckpointFingerprint { - if e == nil { - return nil - } - return []pipeline.CheckpointFingerprint{ - {Name: "prompt", Value: e.promptSHA}, - {Name: "response_schema", Value: e.responseSchemaSHA}, - {Name: "identity_policy", Value: identity.Policy}, - {Name: "mapping_policy", Value: mappingPolicy}, - } -} - -func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.ItemRegistry], error) { +func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest[dnd.ItemRegistry]) (contracts.TypedExtractionResult[dnd.ItemRegistry], error) { if e == nil { return contracts.TypedExtractionResult[dnd.ItemRegistry]{}, extractorErrorf("extractor must not be nil") } - if e.llm == nil { - return contracts.TypedExtractionResult[dnd.ItemRegistry]{}, extractorErrorf("LLM client must not be nil") + if e.engine == nil { + return contracts.TypedExtractionResult[dnd.ItemRegistry]{}, extractorErrorf("native engine must not be nil") } sourceInput, err := shared.PrepareChunkExtraction(ctx, req) if err != nil { @@ -108,29 +66,41 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRe } order := shared.NewSourceRefOrder(req.Source) - var response extractionResponse - completion, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{ - StageName: Key, PromptID: PromptID, PromptVersion: SchemaVersion, - ProfileID: req.LLMProfile, SessionID: req.SessionID, StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts, - Correction: req.Correction, - Inputs: shared.PromptInputs(sourceInput, req.References), - }, &response) - if err != nil { - return contracts.TypedExtractionResult[dnd.ItemRegistry]{}, extractorErrorf("complete structured output: %w", err) + sourceID := req.Source.ID + materials := shared.PromptInputs(sourceInput, req.References) + inputs := make(map[string]promptkit.ArtifactRef, len(materials)) + for name, material := range materials { + inputs[name] = promptkit.InlineWithURI(material.OriginURI, string(material.Content)) } - candidate, err := shared.ModelCandidateFromResponse(completion) + initial := promptkit.RunRequest{PromptID: PromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID, Inputs: inputs} + value, outcome, err := pipeline.RunProducer(ctx, e.engine, req.Operation, initial, + func(_ context.Context, result *promptkit.RunResult) (*dnd.ItemRegistry, []string, error) { + var response extractionResponse + if err := json.Unmarshal([]byte(result.RawOutput), &response); err != nil { + return nil, nil, extractorErrorf("decode native output: %w", err) + } + canonicalizeResponse(&response, order, sourceID) + value := canonicalItemRegistry(response, sourceID) + return &value, nil, nil + }, + func(_ context.Context, guidance []string) (promptkit.RunRequest, error) { + return promptkit.RunRequest{PromptID: CorrectionPromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID, + Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline(strings.Join(guidance, "\n\n"))}}, nil + }, + func(context.Context, *dnd.ItemRegistry) (dnd.ItemRegistry, error) { + return dnd.ItemRegistry{Items: []dnd.Item{}}, nil + }) if err != nil { - return contracts.TypedExtractionResult[dnd.ItemRegistry]{}, extractorErrorf("capture model candidate: %w", err) + return contracts.TypedExtractionResult[dnd.ItemRegistry]{}, extractorErrorf("produce registry: %w", err) } - canonicalizeResponse(&response, order, req.Source.ID) - return contracts.TypedExtractionResult[dnd.ItemRegistry]{Value: canonicalItemRegistry(response, req.Source.ID), ModelCandidate: candidate}, nil + return contracts.TypedExtractionResult[dnd.ItemRegistry]{Value: value, Outcome: outcome}, nil } func ModuleSpec() pipeline.ModuleSpec { return pipeline.ModuleSpec{ Key: Key, Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, - CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1, - Requires: append([]string(nil), requiredCapabilities...), Provides: append([]string(nil), providedCapabilities...), + Prompts: []string{PromptID, CorrectionPromptID}, + Requires: append([]string(nil), requiredCapabilities...), Provides: append([]string(nil), providedCapabilities...), ArtifactKind: dnd.ItemRegistryKind, ReferenceSlots: referenceSlots(), } } @@ -141,7 +111,7 @@ func Register(registry *pipeline.ExtractorRegistry) error { if err != nil { return nil, err } - return New(request.Dependencies.LLM, options, request.References) + return New(request.Dependencies.Engine, options, request.References) }) } diff --git a/internal/modules/dnd/extract/itemregistry/extractor_test.go b/internal/modules/dnd/extract/itemregistry/extractor_test.go index d12be93e..d0cb658c 100644 --- a/internal/modules/dnd/extract/itemregistry/extractor_test.go +++ b/internal/modules/dnd/extract/itemregistry/extractor_test.go @@ -72,10 +72,11 @@ func TestExtractPassesReferencesWithoutTreatingThemAsEvidence(t *testing.T) { if _, err := newExtractor(t, client).Extract(context.Background(), req); err != nil { t.Fatalf("Extract() error = %v", err) } - inputs := client.requests[0].Inputs - if string(inputs["glossary"].Content) != "Star Compass: heirloom" || strings.Contains(string(inputs["transcript"].Content), "heirloom") { - t.Fatalf("prompt inputs = %#v, want separated reference material", inputs) + messageContaining(t, client.requests[0], "Star Compass: heirloom") + if strings.Contains(messageContaining(t, client.requests[0], string(req.Chunk.Content)).Content, "heirloom") { + t.Fatal("references became source evidence") } + } func TestExtractHandlesCancellationAndFailures(t *testing.T) { @@ -88,11 +89,11 @@ func TestExtractHandlesCancellationAndFailures(t *testing.T) { for _, test := range []struct { name string extractor *Extractor - req contracts.TypedExtractionRequest + req contracts.TypedExtractionRequest[dnd.ItemRegistry] want string }{ {name: "nil extractor", extractor: nil, req: request, want: "extractor"}, - {name: "nil client", extractor: &Extractor{}, req: request, want: "LLM client"}, + {name: "nil client", extractor: &Extractor{}, req: request, want: "native engine"}, {name: "preflight", extractor: newExtractor(t, &fakeItemsLLMClient{}), req: mismatchedSourceInputRequest(request), want: "must match chunk"}, } { t.Run(test.name, func(t *testing.T) { diff --git a/internal/modules/dnd/extract/itemregistry/native_execution_test.go b/internal/modules/dnd/extract/itemregistry/native_execution_test.go new file mode 100644 index 00000000..6181445d --- /dev/null +++ b/internal/modules/dnd/extract/itemregistry/native_execution_test.go @@ -0,0 +1,156 @@ +package itemregistry + +import ( + "context" + "errors" + "reflect" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + registrycodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/itemregistry" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity" + shape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemregistry/shape" + refs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemregistry/source_refs" + relatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemregistry/source_relatedness" + "go.maximumdirect.net/promptkit" +) + +const ungroundedRegistry = `{"items":[{"name":"Star Compass","source_refs":[{"start_unit_id":999,"end_unit_id":999}]}]}` +const groundedRegistry = `{"items":[{"name":"Star Compass","source_refs":[{"start_unit_id":2,"end_unit_id":2}]}]}` +const emptyRegistry = `{"items":[]}` + +func assessRegistry(ctx context.Context, request contracts.TypedExtractionRequest[dnd.ItemRegistry], value dnd.ItemRegistry) (contracts.CandidateAssessment, error) { + assessment := contracts.CandidateAssessment{} + for _, validator := range []contracts.TypedValidator[dnd.ItemRegistry]{ + shape.New(shape.Options{}), refs.New(refs.Options{}), relatedness.New(relatedness.Options{}), + } { + result, err := validator.Validate(ctx, contracts.TypedValidationRequest[dnd.ItemRegistry]{Stage: "extract", Source: request.Source, Chunk: request.Chunk, Value: value}) + if err != nil { + return contracts.CandidateAssessment{}, err + } + decision := contracts.AssessmentRejected + if result.Approved { + decision = contracts.AssessmentApproved + } + assessment.Validators = append(assessment.Validators, contracts.ValidatorAssessment{ValidatorName: validator.Name(), Decision: decision, Result: result}) + } + return assessment, nil +} + +func TestNativeExtractionCorrectsGroundingWithFixedInputs(t *testing.T) { + client := &fakeItemsLLMClient{responses: []string{ungroundedRegistry, groundedRegistry}} + request := extractionRequest() + request.References = contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{"glossary": {Items: []contracts.ReferenceItem{{Content: []byte("fixed-reference")}}}}} + request.Operation.Controls.MaxAdditionalTurns = 1 + request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "registry-override"} + var prepared []promptkit.PreparedRun + request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = append(prepared, value) } + assessments := 0 + request.Operation.Assess = func(ctx context.Context, value dnd.ItemRegistry) (contracts.CandidateAssessment, error) { + assessments++ + assessment, err := assessRegistry(ctx, request, value) + if assessments == 1 { + if assessment.Accepted() { + t.Fatal("unresolvable citation was accepted") + } + // Changes after capture must not cause source/reference preparation again. + request.SourceInput.Content[0] = '!' + request.References.Slots["glossary"].Items[0].Content[0] = '!' + } + return assessment, err + } + result, err := newExtractor(t, client).Extract(context.Background(), request) + if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || !result.Outcome.ReuseEligible || + assessments != 2 || len(client.requests) != 2 || len(prepared) != 2 || len(result.Value.Items) != 1 { + t.Fatalf("corrected registry=%#v %v, assessments/provider/prepared=%d/%d/%d", result, err, assessments, len(client.requests), len(prepared)) + } + record := result.Value.Items[0] + if record.Name != "Star Compass" || len(record.SourceRefs) != 1 || record.SourceRefs[0].SourceID != request.Source.ID || + record.SourceRefs[0].StartUnitID != 2 || record.SourceRefs[0].EndUnitID != 2 || record.ID != identity.DeriveID(record.Name) { + t.Fatalf("canonical record identity/evidence=%#v", record) + } + if _, err := registrycodec.New().Encode(result.Value); err != nil { + t.Fatalf("durable registry encoding: %v", err) + } + for i, call := range client.requests { + id := PromptID + if i > 0 { + id = CorrectionPromptID + } + if prepared[i].PromptID != id || prepared[i].SelectedProfileID != request.LLMProfile || call.Target.Model != "registry-override" || call.Prompt.SessionID != request.SessionID { + t.Fatal("native selection/overrides lost") + } + } + initial, revised := client.requests[0].Prompt.Messages, client.requests[1].Prompt.Messages + if len(revised) != len(initial)+2 || !reflect.DeepEqual(revised[:len(initial)], initial) || + revised[len(initial)].Role != "assistant" || revised[len(initial)].Content != ungroundedRegistry { + t.Fatal("native correction lost fixed inputs or prior response") + } + messageContaining(t, client.requests[1], "fixed-reference") + messageContaining(t, client.requests[1], string(request.Chunk.Content)) +} + +func TestNativeExtractionDistinguishesEmptyAcceptanceFromFallback(t *testing.T) { + failure := errors.New("provider unavailable") + for _, test := range []struct { + name string + mode contracts.ExecutionMode + responses []string + providerErrors []error + wantAssessments, wantRuns int + wantError, wantFallback bool + }{ + {"ordinary empty", contracts.ModeStrict, []string{emptyRegistry}, nil, 1, 1, false, false}, + {"strict rejection", contracts.ModeStrict, []string{ungroundedRegistry, ungroundedRegistry}, nil, 2, 2, true, false}, + {"permissive rejection", contracts.ModePermissive, []string{ungroundedRegistry, ungroundedRegistry}, nil, 2, 2, false, true}, + {"provider failure", contracts.ModePermissive, nil, []error{failure}, 0, 1, false, true}, + {"failed correction", contracts.ModePermissive, []string{ungroundedRegistry}, []error{nil, failure}, 1, 2, false, true}, + {"structural failure", contracts.ModePermissive, []string{`{"items":null}`}, nil, 0, 1, false, true}, + } { + t.Run(test.name, func(t *testing.T) { + client := &fakeItemsLLMClient{responses: test.responses, errors: test.providerErrors} + if test.name == "structural failure" { + client.content = []byte(test.responses[0]) + } + request := extractionRequest() + request.Operation.Controls.Mode = test.mode + request.Operation.Controls.MaxAdditionalTurns = 1 + assessments, prepared := 0, 0 + request.Operation.Execution.OnPrepared = func(context.Context, promptkit.PreparedRun) { prepared++ } + request.Operation.Assess = func(ctx context.Context, value dnd.ItemRegistry) (contracts.CandidateAssessment, error) { + assessments++ + return assessRegistry(ctx, request, value) + } + result, err := newExtractor(t, client).Extract(context.Background(), request) + if (err != nil) != test.wantError || assessments != test.wantAssessments || prepared != test.wantRuns { + t.Fatalf("registry=%#v %v, assessments=%d runs=%d", result, err, assessments, prepared) + } + if test.wantError { + if result.Outcome.Disposition != "" { + t.Fatal("strict rejection produced an outcome") + } + return + } + if result.Value.Items == nil || len(result.Value.Items) != 0 || (result.Outcome.Disposition == contracts.ProducerFallback) != test.wantFallback || + result.Outcome.ReuseEligible == test.wantFallback { + t.Fatalf("unqualified empty registry=%#v", result) + } + if _, err := registrycodec.New().Encode(result.Value); err != nil { + t.Fatalf("empty durable registry invalid: %v", err) + } + }) + } +} + +func TestCanceledNativeExtractionCannotDeliverFallback(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + client := &fakeItemsLLMClient{content: []byte(groundedRegistry), onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }} + request := extractionRequest() + request.Operation.Controls.Mode = contracts.ModePermissive + result, err := newExtractor(t, client).Extract(ctx, request) + if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" || len(client.requests) != 1 { + t.Fatalf("cancellation=%#v %v", result, err) + } +} diff --git a/internal/modules/dnd/extract/itemregistry/prompt_assets.go b/internal/modules/dnd/extract/itemregistry/prompt_assets.go deleted file mode 100644 index b5ab9d7d..00000000 --- a/internal/modules/dnd/extract/itemregistry/prompt_assets.go +++ /dev/null @@ -1,74 +0,0 @@ -package itemregistry - -import ( - "fmt" - "io/fs" - "sync" - - rootassets "gitea.maximumdirect.net/eric/notarius/assets" - "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - "gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs" - "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" -) - -const promptAssetRoot = "assets/prompts" - -var promptAssetManifest = shared.PromptAssetManifest{ - ModuleDir: PromptID, - ModuleFiles: []promptfs.ModulePromptFile{ - {Name: "prompt.yaml", Path: "prompts/prompt.yaml"}, - {Name: "instructions.md", Path: "prompts/instructions.md"}, - }, - SharedFiles: []string{ - "common-dnd-system.md", - "common-dnd-extraction-evidence.md", - "common-dnd-identity.md", - "common-dnd-transcript-chunk.md", - "common-dnd-references.md", - }, -} - -func moduleAssetFS() (fs.FS, error) { - assets, err := fs.Sub(rootassets.FS(), "dnd/item-registry/extract") - if err != nil { - return nil, fmt.Errorf("scope item extraction assets: %w", err) - } - return assets, nil -} - -func RegisterPromptAssets(registry *llm.AssetRegistry) error { - assets, err := moduleAssetFS() - if err != nil { - return err - } - promptFS, err := promptAssetManifest.PromptFS(assets) - if err != nil { - return fmt.Errorf("prepare item prompt assets: %w", err) - } - if err := registry.RegisterPromptFS(promptFS, promptAssetRoot); err != nil { - return err - } - schemas, err := fs.Sub(assets, "schemas") - if err != nil { - return fmt.Errorf("scope item extraction schemas: %w", err) - } - return registry.RegisterSchemaFS(schemas, ".") -} - -func promptAssetMetadata() (string, error) { - promptAssetHashOnce.Do(func() { - assets, err := moduleAssetFS() - if err != nil { - promptAssetHashErr = err - return - } - promptAssetHash, promptAssetHashErr = promptAssetManifest.Hash(assets) - }) - return promptAssetHash, promptAssetHashErr -} - -var ( - promptAssetHashOnce sync.Once - promptAssetHash string - promptAssetHashErr error -) diff --git a/internal/modules/dnd/extract/itemregistry/prompt_assets_test.go b/internal/modules/dnd/extract/itemregistry/prompt_assets_test.go deleted file mode 100644 index a813f196..00000000 --- a/internal/modules/dnd/extract/itemregistry/prompt_assets_test.go +++ /dev/null @@ -1,41 +0,0 @@ -package itemregistry - -import ( - "context" - "strings" - "testing" - "time" - - "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - "gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture" - "gitea.maximumdirect.net/eric/promptkit" -) - -func TestRegisterPromptAssetsPreparesItemPrompt(t *testing.T) { - registry := llm.NewAssetRegistry() - if err := RegisterPromptAssets(registry); err != nil { - t.Fatalf("RegisterPromptAssets() error = %v", err) - } - engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{ - "item-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "item-test-model"), - }) - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "item-test-profile", Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline(`{"units":[{"sentinel":"item-transcript"}]}`), "players": promptkit.Inline("item-player"), "party": promptkit.Inline(" "), "glossary": promptkit.Inline(" "), - }}) - if err != nil { - t.Fatalf("Prepare() error = %v", err) - } - if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_item_registry_llm.v1.json" { - t.Fatalf("prepared prompt = %#v, want item prompt identity and schema wiring", prepared) - } - content := make([]string, len(prepared.Messages)) - for index, message := range prepared.Messages { - content[index] = message.Content - } - rendered := strings.Join(content, "\n") - for _, policy := range []string{"named unique items", "concrete reusable item types", "currency denomination", "loot", "generic weapons", "Do not use capitalization as an eligibility test"} { - if !strings.Contains(rendered, policy) { - t.Fatalf("rendered prompt omits item eligibility policy %q", policy) - } - } -} diff --git a/internal/modules/dnd/extract/itemregistry/registry_test.go b/internal/modules/dnd/extract/itemregistry/registry_test.go index 658381c4..d8f692fe 100644 --- a/internal/modules/dnd/extract/itemregistry/registry_test.go +++ b/internal/modules/dnd/extract/itemregistry/registry_test.go @@ -1,7 +1,6 @@ package itemregistry import ( - "encoding/json" "reflect" "strings" "testing" @@ -9,17 +8,16 @@ import ( "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" - "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity" ) -func TestModuleRegistrationMetadataAndRedaction(t *testing.T) { - if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "LLM client") { +func TestModuleRegistrationAndNativeDependencies(t *testing.T) { + if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "native engine") { t.Fatalf("New(nil) error = %v, want client rejection", err) } - if _, err := New(&fakeItemsLLMClient{}, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one") { + if _, err := New(newExtractor(t, &fakeItemsLLMClient{}).engine, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one") { t.Fatalf("New() error = %v, want reference-set rejection", err) } - want := pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1, Requires: []string{"chunks", "source.transcript"}, Provides: []string{"dnd.item_registry"}, ArtifactKind: dnd.ItemRegistryKind, ReferenceSlots: referenceSlots()} + want := pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{PromptID, CorrectionPromptID}, Requires: []string{"chunks", "source.transcript"}, Provides: []string{"dnd.item_registry"}, ArtifactKind: dnd.ItemRegistryKind, ReferenceSlots: referenceSlots()} if got := ModuleSpec(); !reflect.DeepEqual(got, want) { t.Fatalf("ModuleSpec() = %#v, want %#v", got, want) } @@ -36,25 +34,5 @@ func TestModuleRegistrationMetadataAndRedaction(t *testing.T) { if _, err := DecodeOptions(map[string]any{"unknown": true}); err == nil { t.Fatal("DecodeOptions() accepted unknown option") } - references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{ - "glossary": {Items: []contracts.ReferenceItem{{Content: []byte("secret-campaign-context")}}}, - }} - metadata := newExtractor(t, &fakeItemsLLMClient{}, references).ManifestMetadata() - for key, value := range map[string]string{ - "prompt_id": PromptID, "prompt_version": SchemaVersion, - "response_schema_key": string(ResponseSchemaKey), "response_schema_id": ResponseSchemaID, - "response_schema_name": ResponseSchemaName, "response_schema_version": SchemaVersion, - "identity_policy": identity.Policy, "mapping_policy": mappingPolicy, - } { - if metadata[key] != value { - t.Fatalf("metadata[%q] = %#v, want %q", key, metadata[key], value) - } - } - encoded, err := json.Marshal(metadata) - if err != nil || strings.Contains(string(encoded), "secret-campaign-context") || strings.Contains(string(encoded), "session-items") { - t.Fatalf("manifest metadata = %s, want static redacted provenance only", encoded) - } - if got := newExtractor(t, &fakeItemsLLMClient{}).CheckpointFingerprints(); len(got) != 4 || got[0].Name != "prompt" || got[1].Name != "response_schema" || got[2].Value != identity.Policy || got[3].Value != mappingPolicy { - t.Fatalf("CheckpointFingerprints() = %#v", got) - } + } diff --git a/internal/modules/dnd/extract/itemregistry/schema.go b/internal/modules/dnd/extract/itemregistry/schema.go deleted file mode 100644 index e2addf3c..00000000 --- a/internal/modules/dnd/extract/itemregistry/schema.go +++ /dev/null @@ -1,25 +0,0 @@ -package itemregistry - -import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - -const ( - PromptID = "dnd.item_registry" - ResponseSchemaKey = llm.ResponseSchemaKey("dnd_item_registry_llm") - ResponseSchemaID = "notarius.dnd.item_registry.llm" - ResponseSchemaName = "notarius_dnd_item_registry_llm_v1" - SchemaVersion = "v1" -) - -func loadResponseSchema() (llm.ResponseSchema, error) { - assets, err := moduleAssetFS() - if err != nil { - return llm.ResponseSchema{}, err - } - return llm.LoadResponseSchema(assets, llm.ResponseSchemaDefinition{ - Key: ResponseSchemaKey, - ID: ResponseSchemaID, - Version: SchemaVersion, - Name: ResponseSchemaName, - AssetPath: "schemas/dnd_item_registry_llm.v1.json", - }) -} diff --git a/internal/modules/dnd/extract/itemregistry/schema_test.go b/internal/modules/dnd/extract/itemregistry/schema_test.go deleted file mode 100644 index 35da5faf..00000000 --- a/internal/modules/dnd/extract/itemregistry/schema_test.go +++ /dev/null @@ -1,25 +0,0 @@ -package itemregistry - -import ( - "encoding/json" - "strings" - "testing" -) - -func TestItemResponseSchemaIsPrivateAndStructural(t *testing.T) { - schema, err := loadResponseSchema() - if err != nil { - t.Fatalf("loadResponseSchema() error = %v", err) - } - if schema.Key != ResponseSchemaKey || schema.ID != ResponseSchemaID || schema.Name != ResponseSchemaName || schema.Version != SchemaVersion || !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) { - t.Fatalf("schema = %#v, want item response schema", schema) - } - var document map[string]any - if err := json.Unmarshal(schema.JSONSchema, &document); err != nil { - t.Fatal(err) - } - encoded, err := json.Marshal(document) - if err != nil || strings.Contains(string(encoded), `"id"`) || !strings.Contains(string(encoded), `"additionalProperties":false`) { - t.Fatalf("schema = %s, want strict private response without durable ID", encoded) - } -} diff --git a/internal/modules/dnd/extract/itemregistry/test_helpers_test.go b/internal/modules/dnd/extract/itemregistry/test_helpers_test.go index c4a988c6..d3a1d7eb 100644 --- a/internal/modules/dnd/extract/itemregistry/test_helpers_test.go +++ b/internal/modules/dnd/extract/itemregistry/test_helpers_test.go @@ -3,14 +3,18 @@ package itemregistry import ( "context" "encoding/json" - "errors" + "strings" "testing" + "gitea.maximumdirect.net/eric/notarius/assets" "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + "go.maximumdirect.net/configkit" + "go.maximumdirect.net/promptkit" ) -func extractionRequest() contracts.TypedExtractionRequest { +func extractionRequest() contracts.TypedExtractionRequest[dnd.ItemRegistry] { doc := sourceDocument() chunk := &source.Chunk{ ID: "session-items:chunk:0", SourceID: doc.ID, Index: 0, @@ -18,10 +22,11 @@ func extractionRequest() contracts.TypedExtractionRequest { Content: []byte(`{"units":[1,2,3]}`), MediaType: "application/json", Units: append([]source.SourceUnit(nil), doc.Units...), } - return contracts.TypedExtractionRequest{ + return contracts.TypedExtractionRequest[dnd.ItemRegistry]{ Source: doc, Chunk: chunk, SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session-items.json"), SessionID: "item-session", LLMProfile: "item-profile", + Operation: registryOperation(), } } @@ -37,61 +42,97 @@ func responseSourceRefs(startUnitID, endUnitID int) []itemSourceRefResponse { return []itemSourceRefResponse{{StartUnitID: startUnitID, EndUnitID: endUnitID}} } -func newExtractor(t *testing.T, client contracts.StructuredLLMClient, references ...contracts.ReferenceSet) *Extractor { +func registryOperation() contracts.ProducerOperation[dnd.ItemRegistry] { + return contracts.ProducerOperation[dnd.ItemRegistry]{ + Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, Cache: contracts.CacheBypass}, ReuseEligible: true, + Assess: func(context.Context, dnd.ItemRegistry) (contracts.CandidateAssessment, error) { + return contracts.CandidateAssessment{}, nil + }, + } +} + +func newExtractor(t *testing.T, client promptkit.LLMClient, references ...contracts.ReferenceSet) *Extractor { t.Helper() - extractor, err := New(client, Options{}, references...) + type settings struct { + PromptKit promptkit.Settings `yaml:"promptkit"` + } + definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit }) + filesystem := configkit.NewFilesystem("Notarius assets", assets.FS()) + result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("registry fixture", []byte(`promptkit: + profiles: + - id: item-profile + endpoint: https://example.invalid/v1 + model: registry-test-model + api_key_required: false +`))}, configkit.LayeredOptions[settings]{Definition: definition, Underlays: []configkit.Underlay{ + promptkit.BundledOpenRouterUnderlay(), + {Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}}, + }}) if err != nil { - t.Fatalf("New() error = %v, want nil", err) + t.Fatal(err) + } + engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client)) + if err != nil { + t.Fatal(err) + } + extractor, err := New(engine, Options{}, references...) + if err != nil { + t.Fatal(err) } return extractor } -func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest) contracts.TypedExtractionRequest { +func messageContaining(t *testing.T, request promptkit.GenerateRequest, text string) promptkit.RenderedMessage { + t.Helper() + for _, message := range request.Prompt.Messages { + if strings.Contains(message.Content, text) { + return message + } + } + t.Fatalf("missing rendered input %q", text) + return promptkit.RenderedMessage{} +} + +func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest[dnd.ItemRegistry]) contracts.TypedExtractionRequest[dnd.ItemRegistry] { req.SourceInput = contracts.NewLLMInputMaterial("source", "application/json", []byte(`{"different":true}`), "sha256:other", "file:///other.json") return req } type fakeItemsLLMClient struct { - response extractionResponse - content []byte - err error - requests []contracts.StructuredCompletionRequest + response extractionResponse + content []byte + responses []string + err error + errors []error + requests []promptkit.GenerateRequest + onGenerate func(context.Context, promptkit.GenerateRequest) } -func (client *fakeItemsLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { - client.requests = append(client.requests, cloneStructuredCompletionRequest(req)) +func (client *fakeItemsLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + index := len(client.requests) + client.requests = append(client.requests, request) + if client.onGenerate != nil { + client.onGenerate(ctx, request) + } if err := ctx.Err(); err != nil { - return contracts.StructuredCompletionResponse{}, err + return nil, err } if client.err != nil { - return contracts.StructuredCompletionResponse{}, client.err + return nil, client.err } - target, ok := out.(*extractionResponse) - if !ok { - return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target") + if index < len(client.errors) && client.errors[index] != nil { + return nil, client.errors[index] } content := append([]byte(nil), client.content...) - if len(content) != 0 { - if err := json.Unmarshal(content, target); err != nil { - return contracts.StructuredCompletionResponse{}, err - } - } else { - *target = client.response + if index < len(client.responses) { + content = []byte(client.responses[index]) + } + if len(content) == 0 { var err error content, err = json.Marshal(client.response) if err != nil { - return contracts.StructuredCompletionResponse{}, err + return nil, err } } - return contracts.StructuredCompletionResponse{Content: content}, nil -} - -func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest { - req.Inputs = req.Inputs.Clone() - correction, err := contracts.CloneSemanticCorrection(req.Correction) - if err != nil { - panic(err) - } - req.Correction = correction - return req + return &promptkit.GenerateResponse{Content: string(content)}, nil } diff --git a/internal/modules/dnd/extract/locationregistry/extractor.go b/internal/modules/dnd/extract/locationregistry/extractor.go index c2ea7cae..cee5b3ef 100644 --- a/internal/modules/dnd/extract/locationregistry/extractor.go +++ b/internal/modules/dnd/extract/locationregistry/extractor.go @@ -3,19 +3,21 @@ package locationregistry import ( "context" + "encoding/json" "fmt" + "strings" + + "go.maximumdirect.net/promptkit" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" - "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/locations/identity" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" ) -const ( - Key = "dnd/location-registry" - mappingPolicy = "dnd.location_registry.extract_mapping.v2" -) +const Key = "dnd/location-registry" +const PromptID = "notarius/dnd.location_registry" +const CorrectionPromptID = "notarius/dnd.location_registry.repair" var requiredCapabilities = []string{"chunks", "source.transcript"} var providedCapabilities = []string{"dnd.location_registry"} @@ -32,75 +34,31 @@ func referenceSlots() []contracts.ReferenceSlot { } var _ contracts.Extractor[dnd.LocationRegistry] = (*Extractor)(nil) -var _ contracts.ManifestMetadataProvider = (*Extractor)(nil) -var _ pipeline.CheckpointFingerprintProvider = (*Extractor)(nil) type Options struct{} -type Extractor struct { - llm contracts.StructuredLLMClient - promptSHA string - responseSchemaSHA string -} +type Extractor struct{ engine *promptkit.Engine } -func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) { - if llmClient == nil { - return nil, extractorErrorf("LLM client must not be nil") +func New(engine *promptkit.Engine, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) { + if engine == nil { + return nil, extractorErrorf("native engine must not be nil") } if len(references) > 1 { return nil, extractorErrorf("at most one reference set may be supplied") } - promptSHA, err := promptAssetMetadata() - if err != nil { - return nil, extractorErrorf("load prompt metadata: %w", err) - } - responseSchema, err := loadResponseSchema() - if err != nil { - return nil, extractorErrorf("load response schema: %w", err) - } - return &Extractor{llm: llmClient, promptSHA: promptSHA, responseSchemaSHA: responseSchema.SHA256}, nil + return &Extractor{engine: engine}, nil } func (e *Extractor) Key() string { return Key } func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot { return referenceSlots() } -func (e *Extractor) ManifestMetadata() map[string]any { - if e == nil { - return nil - } - return map[string]any{ - "prompt_id": PromptID, - "prompt_version": SchemaVersion, - "prompt_sha256": e.promptSHA, - "response_schema_key": string(ResponseSchemaKey), - "response_schema_id": ResponseSchemaID, - "response_schema_name": ResponseSchemaName, - "response_schema_version": SchemaVersion, - "response_schema_sha256": e.responseSchemaSHA, - "identity_policy": identity.Policy, - "mapping_policy": mappingPolicy, - } -} - -func (e *Extractor) CheckpointFingerprints() []pipeline.CheckpointFingerprint { - if e == nil { - return nil - } - return []pipeline.CheckpointFingerprint{ - {Name: "prompt", Value: e.promptSHA}, - {Name: "response_schema", Value: e.responseSchemaSHA}, - {Name: "identity_policy", Value: identity.Policy}, - {Name: "mapping_policy", Value: mappingPolicy}, - } -} - -func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.LocationRegistry], error) { +func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest[dnd.LocationRegistry]) (contracts.TypedExtractionResult[dnd.LocationRegistry], error) { if e == nil { return contracts.TypedExtractionResult[dnd.LocationRegistry]{}, extractorErrorf("extractor must not be nil") } - if e.llm == nil { - return contracts.TypedExtractionResult[dnd.LocationRegistry]{}, extractorErrorf("LLM client must not be nil") + if e.engine == nil { + return contracts.TypedExtractionResult[dnd.LocationRegistry]{}, extractorErrorf("native engine must not be nil") } sourceInput, err := shared.PrepareChunkExtraction(ctx, req) if err != nil { @@ -108,29 +66,41 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRe } order := shared.NewSourceRefOrder(req.Source) - var response extractionResponse - completion, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{ - StageName: Key, PromptID: PromptID, PromptVersion: SchemaVersion, - ProfileID: req.LLMProfile, SessionID: req.SessionID, StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts, - Correction: req.Correction, - Inputs: shared.PromptInputs(sourceInput, req.References), - }, &response) - if err != nil { - return contracts.TypedExtractionResult[dnd.LocationRegistry]{}, extractorErrorf("complete structured output: %w", err) + sourceID := req.Source.ID + materials := shared.PromptInputs(sourceInput, req.References) + inputs := make(map[string]promptkit.ArtifactRef, len(materials)) + for name, material := range materials { + inputs[name] = promptkit.InlineWithURI(material.OriginURI, string(material.Content)) } - candidate, err := shared.ModelCandidateFromResponse(completion) + initial := promptkit.RunRequest{PromptID: PromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID, Inputs: inputs} + value, outcome, err := pipeline.RunProducer(ctx, e.engine, req.Operation, initial, + func(_ context.Context, result *promptkit.RunResult) (*dnd.LocationRegistry, []string, error) { + var response extractionResponse + if err := json.Unmarshal([]byte(result.RawOutput), &response); err != nil { + return nil, nil, extractorErrorf("decode native output: %w", err) + } + canonicalizeResponse(&response, order, sourceID) + value := canonicalLocationRegistry(response, sourceID) + return &value, nil, nil + }, + func(_ context.Context, guidance []string) (promptkit.RunRequest, error) { + return promptkit.RunRequest{PromptID: CorrectionPromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID, + Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline(strings.Join(guidance, "\n\n"))}}, nil + }, + func(context.Context, *dnd.LocationRegistry) (dnd.LocationRegistry, error) { + return dnd.LocationRegistry{Locations: []dnd.Location{}}, nil + }) if err != nil { - return contracts.TypedExtractionResult[dnd.LocationRegistry]{}, extractorErrorf("capture model candidate: %w", err) + return contracts.TypedExtractionResult[dnd.LocationRegistry]{}, extractorErrorf("produce registry: %w", err) } - canonicalizeResponse(&response, order, req.Source.ID) - return contracts.TypedExtractionResult[dnd.LocationRegistry]{Value: canonicalLocationRegistry(response, req.Source.ID), ModelCandidate: candidate}, nil + return contracts.TypedExtractionResult[dnd.LocationRegistry]{Value: value, Outcome: outcome}, nil } func ModuleSpec() pipeline.ModuleSpec { return pipeline.ModuleSpec{ Key: Key, Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, - CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1, - Requires: append([]string(nil), requiredCapabilities...), Provides: append([]string(nil), providedCapabilities...), + Prompts: []string{PromptID, CorrectionPromptID}, + Requires: append([]string(nil), requiredCapabilities...), Provides: append([]string(nil), providedCapabilities...), ArtifactKind: dnd.LocationRegistryKind, ReferenceSlots: referenceSlots(), } } @@ -141,7 +111,7 @@ func Register(registry *pipeline.ExtractorRegistry) error { if err != nil { return nil, err } - return New(request.Dependencies.LLM, options, request.References) + return New(request.Dependencies.Engine, options, request.References) }) } diff --git a/internal/modules/dnd/extract/locationregistry/extractor_test.go b/internal/modules/dnd/extract/locationregistry/extractor_test.go index 22d958f0..131e1b5d 100644 --- a/internal/modules/dnd/extract/locationregistry/extractor_test.go +++ b/internal/modules/dnd/extract/locationregistry/extractor_test.go @@ -75,10 +75,11 @@ func TestExtractPassesReferencesWithoutTreatingThemAsEvidence(t *testing.T) { if _, err := newExtractor(t, client).Extract(context.Background(), req); err != nil { t.Fatalf("Extract() error = %v", err) } - inputs := client.requests[0].Inputs - if string(inputs["glossary"].Content) != "Old Mill: abandoned granary" || strings.Contains(string(inputs["transcript"].Content), "abandoned granary") { - t.Fatalf("prompt inputs = %#v, want separated reference material", inputs) + messageContaining(t, client.requests[0], "Old Mill: abandoned granary") + if strings.Contains(messageContaining(t, client.requests[0], string(req.Chunk.Content)).Content, "abandoned granary") { + t.Fatal("references became source evidence") } + } func TestExtractDoesNotMutateRequestMaterials(t *testing.T) { @@ -105,11 +106,11 @@ func TestExtractHandlesEmptyOutputAndLocalFailures(t *testing.T) { for _, test := range []struct { name string extractor *Extractor - req contracts.TypedExtractionRequest + req contracts.TypedExtractionRequest[dnd.LocationRegistry] want string }{ {name: "nil extractor", extractor: nilExtractor, req: request, want: "extractor"}, - {name: "nil client", extractor: &Extractor{}, req: request, want: "LLM client"}, + {name: "nil client", extractor: &Extractor{}, req: request, want: "native engine"}, {name: "preflight", extractor: newExtractor(t, &fakeLocationsLLMClient{}), req: mismatchedSourceInputRequest(request), want: "must match chunk"}, } { t.Run(test.name, func(t *testing.T) { diff --git a/internal/modules/dnd/extract/locationregistry/native_execution_test.go b/internal/modules/dnd/extract/locationregistry/native_execution_test.go new file mode 100644 index 00000000..57f144de --- /dev/null +++ b/internal/modules/dnd/extract/locationregistry/native_execution_test.go @@ -0,0 +1,156 @@ +package locationregistry + +import ( + "context" + "errors" + "reflect" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + registrycodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/locationregistry" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/locations/identity" + shape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/locationregistry/shape" + refs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/locationregistry/source_refs" + relatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/locationregistry/source_relatedness" + "go.maximumdirect.net/promptkit" +) + +const ungroundedRegistry = `{"locations":[{"name":"Old Mill","source_refs":[{"start_unit_id":999,"end_unit_id":999}]}]}` +const groundedRegistry = `{"locations":[{"name":"Old Mill","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}` +const emptyRegistry = `{"locations":[]}` + +func assessRegistry(ctx context.Context, request contracts.TypedExtractionRequest[dnd.LocationRegistry], value dnd.LocationRegistry) (contracts.CandidateAssessment, error) { + assessment := contracts.CandidateAssessment{} + for _, validator := range []contracts.TypedValidator[dnd.LocationRegistry]{ + shape.New(shape.Options{}), refs.New(refs.Options{}), relatedness.New(relatedness.Options{}), + } { + result, err := validator.Validate(ctx, contracts.TypedValidationRequest[dnd.LocationRegistry]{Stage: "extract", Source: request.Source, Chunk: request.Chunk, Value: value}) + if err != nil { + return contracts.CandidateAssessment{}, err + } + decision := contracts.AssessmentRejected + if result.Approved { + decision = contracts.AssessmentApproved + } + assessment.Validators = append(assessment.Validators, contracts.ValidatorAssessment{ValidatorName: validator.Name(), Decision: decision, Result: result}) + } + return assessment, nil +} + +func TestNativeExtractionCorrectsGroundingWithFixedInputs(t *testing.T) { + client := &fakeLocationsLLMClient{responses: []string{ungroundedRegistry, groundedRegistry}} + request := extractionRequest() + request.References = contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{"glossary": {Items: []contracts.ReferenceItem{{Content: []byte("fixed-reference")}}}}} + request.Operation.Controls.MaxAdditionalTurns = 1 + request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "registry-override"} + var prepared []promptkit.PreparedRun + request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = append(prepared, value) } + assessments := 0 + request.Operation.Assess = func(ctx context.Context, value dnd.LocationRegistry) (contracts.CandidateAssessment, error) { + assessments++ + assessment, err := assessRegistry(ctx, request, value) + if assessments == 1 { + if assessment.Accepted() { + t.Fatal("unresolvable citation was accepted") + } + // Changes after capture must not cause source/reference preparation again. + request.SourceInput.Content[0] = '!' + request.References.Slots["glossary"].Items[0].Content[0] = '!' + } + return assessment, err + } + result, err := newExtractor(t, client).Extract(context.Background(), request) + if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || !result.Outcome.ReuseEligible || + assessments != 2 || len(client.requests) != 2 || len(prepared) != 2 || len(result.Value.Locations) != 1 { + t.Fatalf("corrected registry=%#v %v, assessments/provider/prepared=%d/%d/%d", result, err, assessments, len(client.requests), len(prepared)) + } + record := result.Value.Locations[0] + if record.Name != "Old Mill" || len(record.SourceRefs) != 1 || record.SourceRefs[0].SourceID != request.Source.ID || + record.SourceRefs[0].StartUnitID != 1 || record.SourceRefs[0].EndUnitID != 1 || record.ID != identity.DeriveID(record.Name, record.SourceRefs) { + t.Fatalf("canonical record identity/evidence=%#v", record) + } + if _, err := registrycodec.New().Encode(result.Value); err != nil { + t.Fatalf("durable registry encoding: %v", err) + } + for i, call := range client.requests { + id := PromptID + if i > 0 { + id = CorrectionPromptID + } + if prepared[i].PromptID != id || prepared[i].SelectedProfileID != request.LLMProfile || call.Target.Model != "registry-override" || call.Prompt.SessionID != request.SessionID { + t.Fatal("native selection/overrides lost") + } + } + initial, revised := client.requests[0].Prompt.Messages, client.requests[1].Prompt.Messages + if len(revised) != len(initial)+2 || !reflect.DeepEqual(revised[:len(initial)], initial) || + revised[len(initial)].Role != "assistant" || revised[len(initial)].Content != ungroundedRegistry { + t.Fatal("native correction lost fixed inputs or prior response") + } + messageContaining(t, client.requests[1], "fixed-reference") + messageContaining(t, client.requests[1], string(request.Chunk.Content)) +} + +func TestNativeExtractionDistinguishesEmptyAcceptanceFromFallback(t *testing.T) { + failure := errors.New("provider unavailable") + for _, test := range []struct { + name string + mode contracts.ExecutionMode + responses []string + providerErrors []error + wantAssessments, wantRuns int + wantError, wantFallback bool + }{ + {"ordinary empty", contracts.ModeStrict, []string{emptyRegistry}, nil, 1, 1, false, false}, + {"strict rejection", contracts.ModeStrict, []string{ungroundedRegistry, ungroundedRegistry}, nil, 2, 2, true, false}, + {"permissive rejection", contracts.ModePermissive, []string{ungroundedRegistry, ungroundedRegistry}, nil, 2, 2, false, true}, + {"provider failure", contracts.ModePermissive, nil, []error{failure}, 0, 1, false, true}, + {"failed correction", contracts.ModePermissive, []string{ungroundedRegistry}, []error{nil, failure}, 1, 2, false, true}, + {"structural failure", contracts.ModePermissive, []string{`{"locations":null}`}, nil, 0, 1, false, true}, + } { + t.Run(test.name, func(t *testing.T) { + client := &fakeLocationsLLMClient{responses: test.responses, errors: test.providerErrors} + if test.name == "structural failure" { + client.content = []byte(test.responses[0]) + } + request := extractionRequest() + request.Operation.Controls.Mode = test.mode + request.Operation.Controls.MaxAdditionalTurns = 1 + assessments, prepared := 0, 0 + request.Operation.Execution.OnPrepared = func(context.Context, promptkit.PreparedRun) { prepared++ } + request.Operation.Assess = func(ctx context.Context, value dnd.LocationRegistry) (contracts.CandidateAssessment, error) { + assessments++ + return assessRegistry(ctx, request, value) + } + result, err := newExtractor(t, client).Extract(context.Background(), request) + if (err != nil) != test.wantError || assessments != test.wantAssessments || prepared != test.wantRuns { + t.Fatalf("registry=%#v %v, assessments=%d runs=%d", result, err, assessments, prepared) + } + if test.wantError { + if result.Outcome.Disposition != "" { + t.Fatal("strict rejection produced an outcome") + } + return + } + if result.Value.Locations == nil || len(result.Value.Locations) != 0 || (result.Outcome.Disposition == contracts.ProducerFallback) != test.wantFallback || + result.Outcome.ReuseEligible == test.wantFallback { + t.Fatalf("unqualified empty registry=%#v", result) + } + if _, err := registrycodec.New().Encode(result.Value); err != nil { + t.Fatalf("empty durable registry invalid: %v", err) + } + }) + } +} + +func TestCanceledNativeExtractionCannotDeliverFallback(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + client := &fakeLocationsLLMClient{content: []byte(groundedRegistry), onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }} + request := extractionRequest() + request.Operation.Controls.Mode = contracts.ModePermissive + result, err := newExtractor(t, client).Extract(ctx, request) + if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" || len(client.requests) != 1 { + t.Fatalf("cancellation=%#v %v", result, err) + } +} diff --git a/internal/modules/dnd/extract/locationregistry/prompt_assets.go b/internal/modules/dnd/extract/locationregistry/prompt_assets.go deleted file mode 100644 index 5133c1ce..00000000 --- a/internal/modules/dnd/extract/locationregistry/prompt_assets.go +++ /dev/null @@ -1,74 +0,0 @@ -package locationregistry - -import ( - "fmt" - "io/fs" - "sync" - - rootassets "gitea.maximumdirect.net/eric/notarius/assets" - "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - "gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs" - "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" -) - -const promptAssetRoot = "assets/prompts" - -var promptAssetManifest = shared.PromptAssetManifest{ - ModuleDir: PromptID, - ModuleFiles: []promptfs.ModulePromptFile{ - {Name: "prompt.yaml", Path: "prompts/prompt.yaml"}, - {Name: "instructions.md", Path: "prompts/instructions.md"}, - }, - SharedFiles: []string{ - "common-dnd-system.md", - "common-dnd-extraction-evidence.md", - "common-dnd-identity.md", - "common-dnd-transcript-chunk.md", - "common-dnd-references.md", - }, -} - -func moduleAssetFS() (fs.FS, error) { - assets, err := fs.Sub(rootassets.FS(), "dnd/location-registry/extract") - if err != nil { - return nil, fmt.Errorf("scope location extraction assets: %w", err) - } - return assets, nil -} - -func RegisterPromptAssets(registry *llm.AssetRegistry) error { - assets, err := moduleAssetFS() - if err != nil { - return err - } - promptFS, err := promptAssetManifest.PromptFS(assets) - if err != nil { - return fmt.Errorf("prepare location prompt assets: %w", err) - } - if err := registry.RegisterPromptFS(promptFS, promptAssetRoot); err != nil { - return err - } - schemas, err := fs.Sub(assets, "schemas") - if err != nil { - return fmt.Errorf("scope location extraction schemas: %w", err) - } - return registry.RegisterSchemaFS(schemas, ".") -} - -func promptAssetMetadata() (string, error) { - promptAssetHashOnce.Do(func() { - assets, err := moduleAssetFS() - if err != nil { - promptAssetHashErr = err - return - } - promptAssetHash, promptAssetHashErr = promptAssetManifest.Hash(assets) - }) - return promptAssetHash, promptAssetHashErr -} - -var ( - promptAssetHashOnce sync.Once - promptAssetHash string - promptAssetHashErr error -) diff --git a/internal/modules/dnd/extract/locationregistry/prompt_assets_test.go b/internal/modules/dnd/extract/locationregistry/prompt_assets_test.go deleted file mode 100644 index d2052ee9..00000000 --- a/internal/modules/dnd/extract/locationregistry/prompt_assets_test.go +++ /dev/null @@ -1,39 +0,0 @@ -package locationregistry - -import ( - "context" - "strings" - "testing" - "time" - - "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - "gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture" - "gitea.maximumdirect.net/eric/promptkit" -) - -func TestRegisterPromptAssetsPreparesLocationPrompt(t *testing.T) { - registry := llm.NewAssetRegistry() - if err := RegisterPromptAssets(registry); err != nil { - t.Fatalf("RegisterPromptAssets() error = %v", err) - } - engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{ - "location-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "location-test-model"), - }) - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "location-test-profile", Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline(`{"units":[{"sentinel":"location-transcript"}]}`), "players": promptkit.Inline("location-player"), "party": promptkit.Inline(" "), "glossary": promptkit.Inline(" "), - }}) - if err != nil { - t.Fatalf("Prepare() error = %v", err) - } - if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_location_registry_llm.v1.json" { - t.Fatalf("prepared prompt = %#v, want location prompt identity and schema wiring", prepared) - } - content := make([]string, len(prepared.Messages)) - for index, message := range prepared.Messages { - content[index] = message.Content - } - rendered := strings.Join(content, "\n") - if !strings.Contains(rendered, "stable proper name or unique in-world designation") || !strings.Contains(rendered, "the room") || !strings.Contains(rendered, "Do not use capitalization as an eligibility test") { - t.Fatalf("rendered prompt = %q, want named-or-unique location eligibility rules", rendered) - } -} diff --git a/internal/modules/dnd/extract/locationregistry/registry_test.go b/internal/modules/dnd/extract/locationregistry/registry_test.go index 2a04caa3..455a0bcb 100644 --- a/internal/modules/dnd/extract/locationregistry/registry_test.go +++ b/internal/modules/dnd/extract/locationregistry/registry_test.go @@ -8,17 +8,16 @@ import ( "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" - "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/locations/identity" ) -func TestModuleRegistrationAndMetadata(t *testing.T) { - if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "LLM client") { +func TestModuleRegistrationAndNativeDependencies(t *testing.T) { + if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "native engine") { t.Fatalf("New(nil) error = %v, want client rejection", err) } - if _, err := New(&fakeLocationsLLMClient{}, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one") { + if _, err := New(newExtractor(t, &fakeLocationsLLMClient{}).engine, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one") { t.Fatalf("New() error = %v, want reference-set rejection", err) } - want := pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1, Requires: []string{"chunks", "source.transcript"}, Provides: []string{"dnd.location_registry"}, ArtifactKind: dnd.LocationRegistryKind, ReferenceSlots: referenceSlots()} + want := pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{PromptID, CorrectionPromptID}, Requires: []string{"chunks", "source.transcript"}, Provides: []string{"dnd.location_registry"}, ArtifactKind: dnd.LocationRegistryKind, ReferenceSlots: referenceSlots()} if got := ModuleSpec(); !reflect.DeepEqual(got, want) { t.Fatalf("ModuleSpec() = %#v, want %#v", got, want) } @@ -35,19 +34,5 @@ func TestModuleRegistrationAndMetadata(t *testing.T) { if _, err := DecodeOptions(map[string]any{"unknown": true}); err == nil { t.Fatal("DecodeOptions() accepted unknown option") } - extractor := newExtractor(t, &fakeLocationsLLMClient{}) - metadata := extractor.ManifestMetadata() - for key, value := range map[string]string{ - "prompt_id": PromptID, "prompt_version": SchemaVersion, - "response_schema_key": string(ResponseSchemaKey), "response_schema_id": ResponseSchemaID, - "response_schema_name": ResponseSchemaName, "response_schema_version": SchemaVersion, - "identity_policy": identity.Policy, "mapping_policy": mappingPolicy, - } { - if metadata[key] != value { - t.Fatalf("metadata[%q] = %#v, want %q", key, metadata[key], value) - } - } - if got := extractor.CheckpointFingerprints(); len(got) != 4 || got[0].Name != "prompt" || got[1].Name != "response_schema" || got[2].Value != identity.Policy || got[3].Value != mappingPolicy { - t.Fatalf("CheckpointFingerprints() = %#v", got) - } + } diff --git a/internal/modules/dnd/extract/locationregistry/schema.go b/internal/modules/dnd/extract/locationregistry/schema.go deleted file mode 100644 index b6578c7d..00000000 --- a/internal/modules/dnd/extract/locationregistry/schema.go +++ /dev/null @@ -1,25 +0,0 @@ -package locationregistry - -import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - -const ( - PromptID = "dnd.location_registry" - ResponseSchemaKey = llm.ResponseSchemaKey("dnd_location_registry_llm") - ResponseSchemaID = "notarius.dnd.location_registry.llm" - ResponseSchemaName = "notarius_dnd_location_registry_llm_v1" - SchemaVersion = "v1" -) - -func loadResponseSchema() (llm.ResponseSchema, error) { - assets, err := moduleAssetFS() - if err != nil { - return llm.ResponseSchema{}, err - } - return llm.LoadResponseSchema(assets, llm.ResponseSchemaDefinition{ - Key: ResponseSchemaKey, - ID: ResponseSchemaID, - Version: SchemaVersion, - Name: ResponseSchemaName, - AssetPath: "schemas/dnd_location_registry_llm.v1.json", - }) -} diff --git a/internal/modules/dnd/extract/locationregistry/schema_test.go b/internal/modules/dnd/extract/locationregistry/schema_test.go deleted file mode 100644 index 16b690cc..00000000 --- a/internal/modules/dnd/extract/locationregistry/schema_test.go +++ /dev/null @@ -1,25 +0,0 @@ -package locationregistry - -import ( - "encoding/json" - "strings" - "testing" -) - -func TestLocationResponseSchemaIsPrivateAndStructural(t *testing.T) { - schema, err := loadResponseSchema() - if err != nil { - t.Fatalf("loadResponseSchema() error = %v", err) - } - if schema.Key != ResponseSchemaKey || schema.ID != ResponseSchemaID || schema.Name != ResponseSchemaName || schema.Version != SchemaVersion || !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) { - t.Fatalf("schema = %#v, want location response schema", schema) - } - var document map[string]any - if err := json.Unmarshal(schema.JSONSchema, &document); err != nil { - t.Fatal(err) - } - encoded, err := json.Marshal(document) - if err != nil || strings.Contains(string(encoded), `"id"`) { - t.Fatalf("schema = %s, want no durable ID field", encoded) - } -} diff --git a/internal/modules/dnd/extract/locationregistry/test_helpers_test.go b/internal/modules/dnd/extract/locationregistry/test_helpers_test.go index d7143766..5fd71f03 100644 --- a/internal/modules/dnd/extract/locationregistry/test_helpers_test.go +++ b/internal/modules/dnd/extract/locationregistry/test_helpers_test.go @@ -3,14 +3,18 @@ package locationregistry import ( "context" "encoding/json" - "errors" + "strings" "testing" + "gitea.maximumdirect.net/eric/notarius/assets" "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + "go.maximumdirect.net/configkit" + "go.maximumdirect.net/promptkit" ) -func extractionRequest() contracts.TypedExtractionRequest { +func extractionRequest() contracts.TypedExtractionRequest[dnd.LocationRegistry] { doc := sourceDocument() chunk := &source.Chunk{ ID: "session-locations:chunk:0", SourceID: doc.ID, Index: 0, @@ -18,10 +22,11 @@ func extractionRequest() contracts.TypedExtractionRequest { Content: []byte(`{"units":[1,2,3]}`), MediaType: "application/json", Units: append([]source.SourceUnit(nil), doc.Units...), } - return contracts.TypedExtractionRequest{ + return contracts.TypedExtractionRequest[dnd.LocationRegistry]{ Source: doc, Chunk: chunk, SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session-locations.json"), SessionID: "location-session", LLMProfile: "location-profile", + Operation: registryOperation(), } } @@ -37,58 +42,97 @@ func responseSourceRefs(startUnitID, endUnitID int) []locationSourceRefResponse return []locationSourceRefResponse{{StartUnitID: startUnitID, EndUnitID: endUnitID}} } -func newExtractor(t *testing.T, client contracts.StructuredLLMClient, references ...contracts.ReferenceSet) *Extractor { +func registryOperation() contracts.ProducerOperation[dnd.LocationRegistry] { + return contracts.ProducerOperation[dnd.LocationRegistry]{ + Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, Cache: contracts.CacheBypass}, ReuseEligible: true, + Assess: func(context.Context, dnd.LocationRegistry) (contracts.CandidateAssessment, error) { + return contracts.CandidateAssessment{}, nil + }, + } +} + +func newExtractor(t *testing.T, client promptkit.LLMClient, references ...contracts.ReferenceSet) *Extractor { t.Helper() - extractor, err := New(client, Options{}, references...) + type settings struct { + PromptKit promptkit.Settings `yaml:"promptkit"` + } + definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit }) + filesystem := configkit.NewFilesystem("Notarius assets", assets.FS()) + result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("registry fixture", []byte(`promptkit: + profiles: + - id: location-profile + endpoint: https://example.invalid/v1 + model: registry-test-model + api_key_required: false +`))}, configkit.LayeredOptions[settings]{Definition: definition, Underlays: []configkit.Underlay{ + promptkit.BundledOpenRouterUnderlay(), + {Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}}, + }}) if err != nil { - t.Fatalf("New() error = %v, want nil", err) + t.Fatal(err) + } + engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client)) + if err != nil { + t.Fatal(err) + } + extractor, err := New(engine, Options{}, references...) + if err != nil { + t.Fatal(err) } return extractor } -func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest) contracts.TypedExtractionRequest { +func messageContaining(t *testing.T, request promptkit.GenerateRequest, text string) promptkit.RenderedMessage { + t.Helper() + for _, message := range request.Prompt.Messages { + if strings.Contains(message.Content, text) { + return message + } + } + t.Fatalf("missing rendered input %q", text) + return promptkit.RenderedMessage{} +} + +func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest[dnd.LocationRegistry]) contracts.TypedExtractionRequest[dnd.LocationRegistry] { req.SourceInput = contracts.NewLLMInputMaterial("source", "application/json", []byte(`{"different":true}`), "sha256:other", "file:///other.json") return req } type fakeLocationsLLMClient struct { - response extractionResponse - content []byte - err error - requests []contracts.StructuredCompletionRequest + response extractionResponse + content []byte + responses []string + err error + errors []error + requests []promptkit.GenerateRequest + onGenerate func(context.Context, promptkit.GenerateRequest) } -func (client *fakeLocationsLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { - client.requests = append(client.requests, cloneStructuredCompletionRequest(req)) - if client.err != nil { - return contracts.StructuredCompletionResponse{}, client.err +func (client *fakeLocationsLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + index := len(client.requests) + client.requests = append(client.requests, request) + if client.onGenerate != nil { + client.onGenerate(ctx, request) } - target, ok := out.(*extractionResponse) - if !ok { - return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target") + if err := ctx.Err(); err != nil { + return nil, err + } + if client.err != nil { + return nil, client.err + } + if index < len(client.errors) && client.errors[index] != nil { + return nil, client.errors[index] } content := append([]byte(nil), client.content...) - if len(content) != 0 { - if err := json.Unmarshal(content, target); err != nil { - return contracts.StructuredCompletionResponse{}, err - } - } else { - *target = client.response + if index < len(client.responses) { + content = []byte(client.responses[index]) + } + if len(content) == 0 { var err error content, err = json.Marshal(client.response) if err != nil { - return contracts.StructuredCompletionResponse{}, err + return nil, err } } - return contracts.StructuredCompletionResponse{Content: content}, nil -} - -func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest { - req.Inputs = req.Inputs.Clone() - correction, err := contracts.CloneSemanticCorrection(req.Correction) - if err != nil { - panic(err) - } - req.Correction = correction - return req + return &promptkit.GenerateResponse{Content: string(content)}, nil } diff --git a/internal/modules/dnd/extract/npcregistry/extractor.go b/internal/modules/dnd/extract/npcregistry/extractor.go index a4759dd0..8376e55e 100644 --- a/internal/modules/dnd/extract/npcregistry/extractor.go +++ b/internal/modules/dnd/extract/npcregistry/extractor.go @@ -2,18 +2,22 @@ package npcregistry import ( "context" + "encoding/json" "fmt" + "strings" + + "go.maximumdirect.net/promptkit" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" - "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" ) const Key = "dnd/npc-registry" -const mappingPolicy = "dnd.npc_registry.extract_mapping.v3" +const PromptID = "notarius/dnd.npc_registry" +const CorrectionPromptID = "notarius/dnd.npc_registry.repair" var requiredCapabilities = []string{ "chunks", @@ -36,79 +40,31 @@ func referenceSlots() []contracts.ReferenceSlot { } var _ contracts.Extractor[dnd.NPCRegistry] = (*Extractor)(nil) -var _ contracts.ManifestMetadataProvider = (*Extractor)(nil) -var _ pipeline.CheckpointFingerprintProvider = (*Extractor)(nil) type Options struct{} -type Extractor struct { - llm contracts.StructuredLLMClient - promptSHA string - responseSchemaSHA string -} +type Extractor struct{ engine *promptkit.Engine } -func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) { - if llmClient == nil { - return nil, extractorErrorf("LLM client must not be nil") +func New(engine *promptkit.Engine, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) { + if engine == nil { + return nil, extractorErrorf("native engine must not be nil") } if len(references) > 1 { return nil, extractorErrorf("at most one reference set may be supplied") } - promptSHA, err := promptAssetMetadata() - if err != nil { - return nil, extractorErrorf("load prompt metadata: %w", err) - } - responseSchema, err := loadResponseSchema() - if err != nil { - return nil, extractorErrorf("load response schema: %w", err) - } - return &Extractor{ - llm: llmClient, - promptSHA: promptSHA, - responseSchemaSHA: responseSchema.SHA256, - }, nil + return &Extractor{engine: engine}, nil } func (e *Extractor) Key() string { return Key } func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot { return referenceSlots() } -func (e *Extractor) ManifestMetadata() map[string]any { - if e == nil { - return nil - } - return map[string]any{ - "prompt_id": PromptID, - "prompt_version": SchemaVersion, - "prompt_sha256": e.promptSHA, - "response_schema_key": string(ResponseSchemaKey), - "response_schema_id": ResponseSchemaID, - "response_schema_name": ResponseSchemaName, - "response_schema_version": SchemaVersion, - "response_schema_sha256": e.responseSchemaSHA, - "identity_policy": identity.Policy, - "mapping_policy": mappingPolicy, - } -} - -func (e *Extractor) CheckpointFingerprints() []pipeline.CheckpointFingerprint { - if e == nil { - return nil - } - return []pipeline.CheckpointFingerprint{ - {Name: "prompt", Value: e.promptSHA}, - {Name: "response_schema", Value: e.responseSchemaSHA}, - {Name: "identity_policy", Value: identity.Policy}, - {Name: "mapping_policy", Value: mappingPolicy}, - } -} - -func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.NPCRegistry], error) { +func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest[dnd.NPCRegistry]) (contracts.TypedExtractionResult[dnd.NPCRegistry], error) { if e == nil { return contracts.TypedExtractionResult[dnd.NPCRegistry]{}, extractorErrorf("extractor must not be nil") } - if e.llm == nil { - return contracts.TypedExtractionResult[dnd.NPCRegistry]{}, extractorErrorf("LLM client must not be nil") + if e.engine == nil { + return contracts.TypedExtractionResult[dnd.NPCRegistry]{}, extractorErrorf("native engine must not be nil") } sourceInput, err := shared.PrepareChunkExtraction(ctx, req) if err != nil { @@ -116,38 +72,46 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRe } order := shared.NewSourceRefOrder(req.Source) - var response extractionResponse - completion, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{ - StageName: Key, - PromptID: PromptID, - PromptVersion: SchemaVersion, - ProfileID: req.LLMProfile, - SessionID: req.SessionID, - StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts, - Correction: req.Correction, - Inputs: shared.PromptInputs(sourceInput, req.References), - }, &response) - if err != nil { - return contracts.TypedExtractionResult[dnd.NPCRegistry]{}, extractorErrorf("complete structured output: %w", err) + sourceID := req.Source.ID + materials := shared.PromptInputs(sourceInput, req.References) + inputs := make(map[string]promptkit.ArtifactRef, len(materials)) + for name, material := range materials { + inputs[name] = promptkit.InlineWithURI(material.OriginURI, string(material.Content)) } - candidate, err := shared.ModelCandidateFromResponse(completion) + initial := promptkit.RunRequest{PromptID: PromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID, Inputs: inputs} + value, outcome, err := pipeline.RunProducer(ctx, e.engine, req.Operation, initial, + func(_ context.Context, result *promptkit.RunResult) (*dnd.NPCRegistry, []string, error) { + var response extractionResponse + if err := json.Unmarshal([]byte(result.RawOutput), &response); err != nil { + return nil, nil, extractorErrorf("decode native output: %w", err) + } + canonicalizeResponse(&response, order, sourceID) + value := canonicalNPCRegistry(response, sourceID) + return &value, nil, nil + }, + func(_ context.Context, guidance []string) (promptkit.RunRequest, error) { + return promptkit.RunRequest{PromptID: CorrectionPromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID, + Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline(strings.Join(guidance, "\n\n"))}}, nil + }, + func(context.Context, *dnd.NPCRegistry) (dnd.NPCRegistry, error) { + return dnd.NPCRegistry{NPCs: []dnd.NPC{}}, nil + }) if err != nil { - return contracts.TypedExtractionResult[dnd.NPCRegistry]{}, extractorErrorf("capture model candidate: %w", err) + return contracts.TypedExtractionResult[dnd.NPCRegistry]{}, extractorErrorf("produce registry: %w", err) } - canonicalizeResponse(&response, order, req.Source.ID) - return contracts.TypedExtractionResult[dnd.NPCRegistry]{Value: canonicalNPCRegistry(response, req.Source.ID), ModelCandidate: candidate}, nil + return contracts.TypedExtractionResult[dnd.NPCRegistry]{Value: value, Outcome: outcome}, nil } func ModuleSpec() pipeline.ModuleSpec { return pipeline.ModuleSpec{ - Key: Key, - Stage: pipeline.StageExtract, - ExecutionClass: contracts.ExecutionClassLLMBacked, - CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1, - Requires: append([]string(nil), requiredCapabilities...), - Provides: append([]string(nil), providedCapabilities...), - ArtifactKind: dnd.NPCRegistryKind, - ReferenceSlots: referenceSlots(), + Key: Key, + Stage: pipeline.StageExtract, + ExecutionClass: contracts.ExecutionClassLLMBacked, + Prompts: []string{PromptID, CorrectionPromptID}, + Requires: append([]string(nil), requiredCapabilities...), + Provides: append([]string(nil), providedCapabilities...), + ArtifactKind: dnd.NPCRegistryKind, + ReferenceSlots: referenceSlots(), } } @@ -157,7 +121,7 @@ func Register(registry *pipeline.ExtractorRegistry) error { if err != nil { return nil, err } - return New(request.Dependencies.LLM, options, request.References) + return New(request.Dependencies.Engine, options, request.References) }) } diff --git a/internal/modules/dnd/extract/npcregistry/extractor_test.go b/internal/modules/dnd/extract/npcregistry/extractor_test.go index 89fe6143..a97be7bf 100644 --- a/internal/modules/dnd/extract/npcregistry/extractor_test.go +++ b/internal/modules/dnd/extract/npcregistry/extractor_test.go @@ -7,6 +7,8 @@ import ( "strings" "testing" + "go.maximumdirect.net/promptkit" + "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" @@ -28,7 +30,10 @@ func TestExtractReturnsCanonicalNPCRegistryFromPrivateResponse(t *testing.T) { }, }}} - result, err := newExtractor(t, client).Extract(context.Background(), extractionRequest()) + callRequest := extractionRequest() + var prepared promptkit.PreparedRun + callRequest.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = value } + result, err := newExtractor(t, client).Extract(context.Background(), callRequest) if err != nil { t.Fatalf("Extract() error = %v, want nil", err) } @@ -43,13 +48,11 @@ func TestExtractReturnsCanonicalNPCRegistryFromPrivateResponse(t *testing.T) { t.Fatalf("LLM calls = %d, want 1", len(client.requests)) } request := client.requests[0] - if request.StageName != Key || request.PromptID != PromptID || request.PromptVersion != SchemaVersion || request.ProfileID != "profile-npcs" || request.SessionID != "session-123" { - t.Fatalf("LLM request identity = %#v", request) - } - transcript := request.Inputs["transcript"] - if transcript.Name != "transcript" || transcript.MediaType != "application/json" || transcript.Digest != "sha256:chunk" || transcript.OriginURI != "file:///session-alpha.json" || string(transcript.Content) != string(extractionRequest().Chunk.Content) { - t.Fatalf("transcript input = %#v, want chunk-scoped material", transcript) + if prepared.PromptID != PromptID || prepared.SelectedProfileID != "profile-npcs" || request.Prompt.SessionID != "session-123" { + t.Fatal("native registry routing lost") } + messageContaining(t, request, string(callRequest.Chunk.Content)) + } func TestExtractOrdersNPCsBySourcePositionRatherThanUnitID(t *testing.T) { @@ -132,13 +135,15 @@ func TestExtractPassesCampaignReferencesAsPromptInputs(t *testing.T) { if _, err := newExtractor(t, client).Extract(context.Background(), req); err != nil { t.Fatalf("Extract() error = %v, want nil", err) } - inputs := client.requests[0].Inputs - if string(inputs["players"].Content) != "Dana: Mira" || string(inputs["party"].Content) != "Mira: ranger" || string(inputs["glossary"].Content) != "Greencloak: local title" { - t.Fatalf("reference inputs = %#v", inputs) + request := client.requests[0] + references := messageContaining(t, request, "Dana: Mira").Content + if !strings.Contains(references, "Mira: ranger") || !strings.Contains(references, "Greencloak: local title") { + t.Fatal("campaign references missing") } - if strings.Contains(string(inputs["transcript"].Content), "local title") { - t.Fatal("transcript input contains reference content") + if strings.Contains(messageContaining(t, request, string(req.Chunk.Content)).Content, "local title") { + t.Fatal("references became source evidence") } + } func TestExtractPreservesMalformedCandidatesForValidators(t *testing.T) { @@ -173,38 +178,6 @@ func TestExtractMapsRawSemanticCandidatesWithoutRepair(t *testing.T) { } } -func TestExtractForwardsCorrectionAndOwnsModelCandidate(t *testing.T) { - rawResponse := []byte(`{"npcs":[{"name":"Mira Thorn","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`) - client := &fakeNPCsLLMClient{content: append([]byte(nil), rawResponse...)} - correction, err := contracts.NewSemanticCorrection([]byte(`{"npcs":[]}`), "Retain the transcript-grounded NPC.") - if err != nil { - t.Fatalf("NewSemanticCorrection() error = %v", err) - } - request := extractionRequest() - request.Correction = correction - result, err := newExtractor(t, client).Extract(context.Background(), request) - if err != nil { - t.Fatalf("Extract() error = %v, want nil", err) - } - if len(client.requests) != 1 || client.requests[0].Correction == nil || - string(client.requests[0].Correction.AssistantResponse) != `{"npcs":[]}` || - client.requests[0].Correction.UserGuidance != "Retain the transcript-grounded NPC." { - t.Fatalf("structured request correction = %#v, want forwarded correction", client.requests) - } - correction.AssistantResponse[0] = '[' - if got := string(client.requests[0].Correction.AssistantResponse); got != `{"npcs":[]}` { - t.Fatalf("captured correction changed after caller mutation: %q", got) - } - if result.ModelCandidate == nil || result.ModelCandidate.Protocol != contracts.CorrectionProtocolSingleResponseV1 || - string(result.ModelCandidate.Response) != string(rawResponse) { - t.Fatalf("model candidate = %#v, want exact validated response", result.ModelCandidate) - } - client.content[0] = '[' - if got := string(result.ModelCandidate.Response); got != string(rawResponse) { - t.Fatalf("model candidate changed after provider buffer mutation: %q", got) - } -} - func TestExtractRetainsLocalErrorContextAndProviderFailures(t *testing.T) { request := extractionRequest() extractor := newExtractor(t, &fakeNPCsLLMClient{response: extractionResponse{NPCs: []npcResponse{}}}) @@ -212,11 +185,11 @@ func TestExtractRetainsLocalErrorContextAndProviderFailures(t *testing.T) { for _, test := range []struct { name string extractor *Extractor - req contracts.TypedExtractionRequest + req contracts.TypedExtractionRequest[dnd.NPCRegistry] want string }{ {name: "nil extractor", extractor: nilExtractor, req: request, want: "extractor"}, - {name: "nil LLM client", extractor: &Extractor{}, req: request, want: "LLM client"}, + {name: "nil native engine", extractor: &Extractor{}, req: request, want: "native engine"}, {name: "wrapped preflight failure", extractor: extractor, req: mismatchedSourceInputRequest(request), want: "must match chunk"}, } { t.Run(test.name, func(t *testing.T) { diff --git a/internal/modules/dnd/extract/npcregistry/native_execution_test.go b/internal/modules/dnd/extract/npcregistry/native_execution_test.go new file mode 100644 index 00000000..faaacc6a --- /dev/null +++ b/internal/modules/dnd/extract/npcregistry/native_execution_test.go @@ -0,0 +1,156 @@ +package npcregistry + +import ( + "context" + "errors" + "reflect" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + registrycodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcregistry" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity" + shape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/npcregistry/shape" + refs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/npcregistry/source_refs" + relatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/npcregistry/source_relatedness" + "go.maximumdirect.net/promptkit" +) + +const ungroundedRegistry = `{"npcs":[{"name":"Mira Thorn","source_refs":[{"start_unit_id":999,"end_unit_id":999}]}]}` +const groundedRegistry = `{"npcs":[{"name":"Mira Thorn","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}` +const emptyRegistry = `{"npcs":[]}` + +func assessRegistry(ctx context.Context, request contracts.TypedExtractionRequest[dnd.NPCRegistry], value dnd.NPCRegistry) (contracts.CandidateAssessment, error) { + assessment := contracts.CandidateAssessment{} + for _, validator := range []contracts.TypedValidator[dnd.NPCRegistry]{ + shape.New(shape.Options{}), refs.New(refs.Options{}), relatedness.New(relatedness.Options{}), + } { + result, err := validator.Validate(ctx, contracts.TypedValidationRequest[dnd.NPCRegistry]{Stage: "extract", Source: request.Source, Chunk: request.Chunk, Value: value}) + if err != nil { + return contracts.CandidateAssessment{}, err + } + decision := contracts.AssessmentRejected + if result.Approved { + decision = contracts.AssessmentApproved + } + assessment.Validators = append(assessment.Validators, contracts.ValidatorAssessment{ValidatorName: validator.Name(), Decision: decision, Result: result}) + } + return assessment, nil +} + +func TestNativeExtractionCorrectsGroundingWithFixedInputs(t *testing.T) { + client := &fakeNPCsLLMClient{responses: []string{ungroundedRegistry, groundedRegistry}} + request := extractionRequest() + request.References = contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{"glossary": {Items: []contracts.ReferenceItem{{Content: []byte("fixed-reference")}}}}} + request.Operation.Controls.MaxAdditionalTurns = 1 + request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "registry-override"} + var prepared []promptkit.PreparedRun + request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = append(prepared, value) } + assessments := 0 + request.Operation.Assess = func(ctx context.Context, value dnd.NPCRegistry) (contracts.CandidateAssessment, error) { + assessments++ + assessment, err := assessRegistry(ctx, request, value) + if assessments == 1 { + if assessment.Accepted() { + t.Fatal("unresolvable citation was accepted") + } + // Changes after capture must not cause source/reference preparation again. + request.SourceInput.Content[0] = '!' + request.References.Slots["glossary"].Items[0].Content[0] = '!' + } + return assessment, err + } + result, err := newExtractor(t, client).Extract(context.Background(), request) + if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || !result.Outcome.ReuseEligible || + assessments != 2 || len(client.requests) != 2 || len(prepared) != 2 || len(result.Value.NPCs) != 1 { + t.Fatalf("corrected registry=%#v %v, assessments/provider/prepared=%d/%d/%d", result, err, assessments, len(client.requests), len(prepared)) + } + record := result.Value.NPCs[0] + if record.Name != "Mira Thorn" || len(record.SourceRefs) != 1 || record.SourceRefs[0].SourceID != request.Source.ID || + record.SourceRefs[0].StartUnitID != 1 || record.SourceRefs[0].EndUnitID != 1 || record.ID != identity.DeriveID(record.Name) { + t.Fatalf("canonical record identity/evidence=%#v", record) + } + if _, err := registrycodec.New().Encode(result.Value); err != nil { + t.Fatalf("durable registry encoding: %v", err) + } + for i, call := range client.requests { + id := PromptID + if i > 0 { + id = CorrectionPromptID + } + if prepared[i].PromptID != id || prepared[i].SelectedProfileID != request.LLMProfile || call.Target.Model != "registry-override" || call.Prompt.SessionID != request.SessionID { + t.Fatal("native selection/overrides lost") + } + } + initial, revised := client.requests[0].Prompt.Messages, client.requests[1].Prompt.Messages + if len(revised) != len(initial)+2 || !reflect.DeepEqual(revised[:len(initial)], initial) || + revised[len(initial)].Role != "assistant" || revised[len(initial)].Content != ungroundedRegistry { + t.Fatal("native correction lost fixed inputs or prior response") + } + messageContaining(t, client.requests[1], "fixed-reference") + messageContaining(t, client.requests[1], string(request.Chunk.Content)) +} + +func TestNativeExtractionDistinguishesEmptyAcceptanceFromFallback(t *testing.T) { + failure := errors.New("provider unavailable") + for _, test := range []struct { + name string + mode contracts.ExecutionMode + responses []string + providerErrors []error + wantAssessments, wantRuns int + wantError, wantFallback bool + }{ + {"ordinary empty", contracts.ModeStrict, []string{emptyRegistry}, nil, 1, 1, false, false}, + {"strict rejection", contracts.ModeStrict, []string{ungroundedRegistry, ungroundedRegistry}, nil, 2, 2, true, false}, + {"permissive rejection", contracts.ModePermissive, []string{ungroundedRegistry, ungroundedRegistry}, nil, 2, 2, false, true}, + {"provider failure", contracts.ModePermissive, nil, []error{failure}, 0, 1, false, true}, + {"failed correction", contracts.ModePermissive, []string{ungroundedRegistry}, []error{nil, failure}, 1, 2, false, true}, + {"structural failure", contracts.ModePermissive, []string{`{"npcs":null}`}, nil, 0, 1, false, true}, + } { + t.Run(test.name, func(t *testing.T) { + client := &fakeNPCsLLMClient{responses: test.responses, errors: test.providerErrors} + if test.name == "structural failure" { + client.content = []byte(test.responses[0]) + } + request := extractionRequest() + request.Operation.Controls.Mode = test.mode + request.Operation.Controls.MaxAdditionalTurns = 1 + assessments, prepared := 0, 0 + request.Operation.Execution.OnPrepared = func(context.Context, promptkit.PreparedRun) { prepared++ } + request.Operation.Assess = func(ctx context.Context, value dnd.NPCRegistry) (contracts.CandidateAssessment, error) { + assessments++ + return assessRegistry(ctx, request, value) + } + result, err := newExtractor(t, client).Extract(context.Background(), request) + if (err != nil) != test.wantError || assessments != test.wantAssessments || prepared != test.wantRuns { + t.Fatalf("registry=%#v %v, assessments=%d runs=%d", result, err, assessments, prepared) + } + if test.wantError { + if result.Outcome.Disposition != "" { + t.Fatal("strict rejection produced an outcome") + } + return + } + if result.Value.NPCs == nil || len(result.Value.NPCs) != 0 || (result.Outcome.Disposition == contracts.ProducerFallback) != test.wantFallback || + result.Outcome.ReuseEligible == test.wantFallback { + t.Fatalf("unqualified empty registry=%#v", result) + } + if _, err := registrycodec.New().Encode(result.Value); err != nil { + t.Fatalf("empty durable registry invalid: %v", err) + } + }) + } +} + +func TestCanceledNativeExtractionCannotDeliverFallback(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + client := &fakeNPCsLLMClient{content: []byte(groundedRegistry), onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }} + request := extractionRequest() + request.Operation.Controls.Mode = contracts.ModePermissive + result, err := newExtractor(t, client).Extract(ctx, request) + if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" || len(client.requests) != 1 { + t.Fatalf("cancellation=%#v %v", result, err) + } +} diff --git a/internal/modules/dnd/extract/npcregistry/prompt_assets.go b/internal/modules/dnd/extract/npcregistry/prompt_assets.go deleted file mode 100644 index 991c9ea5..00000000 --- a/internal/modules/dnd/extract/npcregistry/prompt_assets.go +++ /dev/null @@ -1,74 +0,0 @@ -package npcregistry - -import ( - "fmt" - "io/fs" - "sync" - - rootassets "gitea.maximumdirect.net/eric/notarius/assets" - "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - "gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs" - "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" -) - -const promptAssetRoot = "assets/prompts" - -var promptAssetManifest = shared.PromptAssetManifest{ - ModuleDir: "dnd.npc_registry", - ModuleFiles: []promptfs.ModulePromptFile{ - {Name: "prompt.yaml", Path: "prompts/prompt.yaml"}, - {Name: "instructions.md", Path: "prompts/instructions.md"}, - }, - SharedFiles: []string{ - "common-dnd-system.md", - "common-dnd-extraction-evidence.md", - "common-dnd-identity.md", - "common-dnd-transcript-chunk.md", - "common-dnd-references.md", - }, -} - -func moduleAssetFS() (fs.FS, error) { - assets, err := fs.Sub(rootassets.FS(), "dnd/npc-registry/extract") - if err != nil { - return nil, fmt.Errorf("scope NPC extraction assets: %w", err) - } - return assets, nil -} - -func RegisterPromptAssets(registry *llm.AssetRegistry) error { - assets, err := moduleAssetFS() - if err != nil { - return err - } - promptFS, err := promptAssetManifest.PromptFS(assets) - if err != nil { - return fmt.Errorf("prepare NPC prompt assets: %w", err) - } - if err := registry.RegisterPromptFS(promptFS, promptAssetRoot); err != nil { - return err - } - schemas, err := fs.Sub(assets, "schemas") - if err != nil { - return fmt.Errorf("scope NPC extraction schemas: %w", err) - } - return registry.RegisterSchemaFS(schemas, ".") -} - -func promptAssetMetadata() (string, error) { - promptAssetHashOnce.Do(func() { - assets, err := moduleAssetFS() - if err != nil { - promptAssetHashErr = err - return - } - promptAssetHash, promptAssetHashErr = promptAssetManifest.Hash(assets) - }) - return promptAssetHash, promptAssetHashErr -} - -var ( - promptAssetHashOnce sync.Once - promptAssetHash string - promptAssetHashErr error -) diff --git a/internal/modules/dnd/extract/npcregistry/prompt_assets_test.go b/internal/modules/dnd/extract/npcregistry/prompt_assets_test.go deleted file mode 100644 index 12bf1469..00000000 --- a/internal/modules/dnd/extract/npcregistry/prompt_assets_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package npcregistry - -import ( - "context" - "encoding/json" - "strings" - "testing" - "time" - - "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - "gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture" - "gitea.maximumdirect.net/eric/promptkit" -) - -func TestRegisterPromptAssetsAndPrepareNPCPrompt(t *testing.T) { - registry := llm.NewAssetRegistry() - if err := RegisterPromptAssets(registry); err != nil { - t.Fatalf("RegisterPromptAssets() error = %v, want nil", err) - } - engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{ - "npc-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "npc-test-model"), - }) - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "npc-test-profile", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.InlineWithURI("file:///session.json", `{"units":[{"sentinel":"npc-transcript"}]}`), - "players": promptkit.Inline("npc-player"), - "party": promptkit.Inline(" "), - "glossary": promptkit.Inline(" "), - }, - }) - if err != nil { - t.Fatalf("Prepare() error = %v, want nil", err) - } - if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_npc_registry_llm.v1.json" { - t.Fatalf("prepared prompt = %#v, want NPC prompt identity and wiring", prepared) - } -} - -func TestPromptMetadataAndDiagnosticsDoNotContainRawAssets(t *testing.T) { - hash, err := promptAssetMetadata() - if err != nil || !strings.HasPrefix(hash, "sha256:") { - t.Fatalf("promptAssetMetadata() = %q, %v; want hash", hash, err) - } - metadata := newExtractor(t, &fakeNPCsLLMClient{}).ManifestMetadata() - payload, err := json.Marshal(metadata) - if err != nil { - t.Fatal(err) - } - for _, forbidden := range []string{"common-dnd-system", "dnd_npc_registry_llm.v1.json"} { - if strings.Contains(string(payload), forbidden) { - t.Fatalf("metadata leaked raw prompt/schema content %q: %s", forbidden, payload) - } - } -} diff --git a/internal/modules/dnd/extract/npcregistry/registry_test.go b/internal/modules/dnd/extract/npcregistry/registry_test.go index b6fe52b7..ea051d67 100644 --- a/internal/modules/dnd/extract/npcregistry/registry_test.go +++ b/internal/modules/dnd/extract/npcregistry/registry_test.go @@ -10,11 +10,11 @@ import ( "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" ) -func TestNewRequiresLLMClientAndRejectsAmbiguousReferences(t *testing.T) { - if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "LLM client") { +func TestNewRequiresNativeEngineAndRejectsAmbiguousReferences(t *testing.T) { + if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "native engine") { t.Fatalf("New(nil) error = %v, want dependency error", err) } - if _, err := New(&fakeNPCsLLMClient{}, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one reference set") { + if _, err := New(newExtractor(t, &fakeNPCsLLMClient{}).engine, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one reference set") { t.Fatalf("New() error = %v, want reference-set error", err) } if got := newExtractor(t, &fakeNPCsLLMClient{}).Key(); got != Key { @@ -25,13 +25,13 @@ func TestNewRequiresLLMClientAndRejectsAmbiguousReferences(t *testing.T) { func TestModuleSpecAndReferenceSlots(t *testing.T) { got := ModuleSpec() want := pipeline.ModuleSpec{ - Key: Key, - Stage: pipeline.StageExtract, - ExecutionClass: contracts.ExecutionClassLLMBacked, - CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1, - Requires: []string{"chunks", "source.transcript"}, - Provides: []string{"dnd.npc_registry"}, - ArtifactKind: dnd.NPCRegistryKind, + Key: Key, + Stage: pipeline.StageExtract, + ExecutionClass: contracts.ExecutionClassLLMBacked, + Prompts: []string{PromptID, CorrectionPromptID}, + Requires: []string{"chunks", "source.transcript"}, + Provides: []string{"dnd.npc_registry"}, + ArtifactKind: dnd.NPCRegistryKind, ReferenceSlots: []contracts.ReferenceSlot{ {Name: "glossary", Description: "Optional campaign glossary reference material used only for NPC disambiguation.", AcceptedMediaTypes: []string{"application/json", "application/x-yaml", "application/yaml", "text/markdown", "text/plain"}}, {Name: "party", Description: "Optional party roster reference material used only for NPC disambiguation.", AcceptedMediaTypes: []string{"application/json", "application/x-yaml", "application/yaml", "text/markdown", "text/plain"}}, @@ -42,6 +42,7 @@ func TestModuleSpecAndReferenceSlots(t *testing.T) { if !reflect.DeepEqual(got, want) { t.Fatalf("ModuleSpec() = %#v, want %#v", got, want) } + got.Prompts[0] = "changed" got.Requires[0] = "changed" got.Provides[0] = "changed" got.ReferenceSlots[0].AcceptedMediaTypes[0] = "changed" @@ -70,34 +71,3 @@ func TestRegisterStoresTypedModuleSpec(t *testing.T) { t.Fatalf("DecodeOptions() error = %v, want strict options error", err) } } - -func TestExtractorMetadataAndCheckpointIdentity(t *testing.T) { - extractor := newExtractor(t, &fakeNPCsLLMClient{}) - metadata := extractor.ManifestMetadata() - for key, want := range map[string]string{ - "prompt_id": PromptID, "prompt_version": SchemaVersion, - "response_schema_key": string(ResponseSchemaKey), "response_schema_id": ResponseSchemaID, - "response_schema_name": ResponseSchemaName, "response_schema_version": SchemaVersion, - "identity_policy": "dnd.npc_registry.identity.v1", - "mapping_policy": mappingPolicy, - } { - if metadata[key] != want { - t.Fatalf("metadata[%q] = %#v, want %q", key, metadata[key], want) - } - } - for _, key := range []string{"prompt_sha256", "response_schema_sha256"} { - if value, ok := metadata[key].(string); !ok || !strings.HasPrefix(value, "sha256:") { - t.Fatalf("metadata[%q] = %#v, want hash", key, metadata[key]) - } - } - fingerprints := extractor.CheckpointFingerprints() - want := map[string]string{"prompt": metadata["prompt_sha256"].(string), "response_schema": metadata["response_schema_sha256"].(string), "identity_policy": "dnd.npc_registry.identity.v1", "mapping_policy": mappingPolicy} - if len(fingerprints) != len(want) { - t.Fatalf("CheckpointFingerprints() = %#v, want %d entries", fingerprints, len(want)) - } - for _, fingerprint := range fingerprints { - if fingerprint.Value != want[fingerprint.Name] { - t.Fatalf("fingerprint %q = %q, want %#v", fingerprint.Name, fingerprint.Value, want) - } - } -} diff --git a/internal/modules/dnd/extract/npcregistry/schema.go b/internal/modules/dnd/extract/npcregistry/schema.go deleted file mode 100644 index 373ccbe9..00000000 --- a/internal/modules/dnd/extract/npcregistry/schema.go +++ /dev/null @@ -1,25 +0,0 @@ -package npcregistry - -import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - -const ( - PromptID = "dnd.npc_registry" - ResponseSchemaKey = llm.ResponseSchemaKey("dnd_npc_registry_llm") - ResponseSchemaID = "notarius.dnd.npc_registry.llm" - ResponseSchemaName = "notarius_dnd_npc_registry_llm_v1" - SchemaVersion = "v1" -) - -func loadResponseSchema() (llm.ResponseSchema, error) { - assets, err := moduleAssetFS() - if err != nil { - return llm.ResponseSchema{}, err - } - return llm.LoadResponseSchema(assets, llm.ResponseSchemaDefinition{ - Key: ResponseSchemaKey, - ID: ResponseSchemaID, - Version: SchemaVersion, - Name: ResponseSchemaName, - AssetPath: "schemas/dnd_npc_registry_llm.v1.json", - }) -} diff --git a/internal/modules/dnd/extract/npcregistry/schema_test.go b/internal/modules/dnd/extract/npcregistry/schema_test.go deleted file mode 100644 index 0916038c..00000000 --- a/internal/modules/dnd/extract/npcregistry/schema_test.go +++ /dev/null @@ -1,138 +0,0 @@ -package npcregistry - -import ( - "bytes" - "encoding/json" - "strings" - "testing" - - "github.com/santhosh-tekuri/jsonschema/v6" -) - -func TestLoadResponseSchemaUsesPrivateNPCSchema(t *testing.T) { - schema, err := loadResponseSchema() - if err != nil { - t.Fatalf("loadResponseSchema() error = %v, want nil", err) - } - if schema.Key != ResponseSchemaKey || schema.ID != ResponseSchemaID || schema.Version != SchemaVersion || schema.Name != ResponseSchemaName || !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) { - t.Fatalf("schema = %#v, want private NPC schema identity", schema) - } - valid := map[string]any{"npcs": []any{map[string]any{ - "name": "Mira Thorn", - "source_refs": []any{map[string]any{"start_unit_id": 1, "end_unit_id": 2}}, - }}} - validJSON, err := json.Marshal(valid) - if err != nil { - t.Fatal(err) - } - if err := validateJSONSchema(validJSON, schema.JSONSchema); err != nil { - t.Fatalf("valid private NPC response rejected: %v", err) - } - for _, test := range []struct { - name string - response map[string]any - valid bool - }{ - { - name: "semantic blanks and empty collections", - response: map[string]any{"npcs": []any{map[string]any{ - "name": "", "source_refs": []any{}, - }}}, - valid: true, - }, - { - name: "nonpositive unit candidates", - response: map[string]any{"npcs": []any{map[string]any{ - "name": "Mira Thorn", - "source_refs": []any{map[string]any{"start_unit_id": 0, "end_unit_id": -1}}, - }}}, - valid: true, - }, - { - name: "missing required field", - response: map[string]any{"npcs": []any{map[string]any{ - "source_refs": []any{}, - }}}, - }, - { - name: "unknown field", - response: map[string]any{"npcs": []any{map[string]any{ - "name": "Mira Thorn", "source_refs": []any{}, "id": "assigned later", - }}}, - }, - { - name: "wrong field type", - response: map[string]any{"npcs": []any{map[string]any{ - "name": 7, "source_refs": []any{}, - }}}, - }, - { - name: "noninteger source identifier", - response: map[string]any{"npcs": []any{map[string]any{ - "name": "Mira Thorn", - "source_refs": []any{map[string]any{"start_unit_id": 1.5, "end_unit_id": 2}}, - }}}, - }, - } { - t.Run(test.name, func(t *testing.T) { - content, err := json.Marshal(test.response) - if err != nil { - t.Fatal(err) - } - err = validateJSONSchema(content, schema.JSONSchema) - if (err == nil) != test.valid { - t.Fatalf("validateJSONSchema() error = %v, want valid=%t", err, test.valid) - } - }) - } - if err := validateJSONSchema([]byte(`{"npcs":`), schema.JSONSchema); err == nil { - t.Fatal("validateJSONSchema() error = nil, want malformed JSON rejected") - } - withID := map[string]any{"npcs": []any{map[string]any{ - "name": "Mira Thorn", "id": "assigned-later", - "source_refs": []any{map[string]any{"start_unit_id": 1, "end_unit_id": 2}}, - }}} - withIDJSON, err := json.Marshal(withID) - if err != nil { - t.Fatal(err) - } - if err := validateJSONSchema(withIDJSON, schema.JSONSchema); err == nil { - t.Fatal("private schema accepted framework-assigned id") - } -} - -func TestResponseSchemaIsMutationSafeAndDiagnosticsRedactContent(t *testing.T) { - first, err := loadResponseSchema() - if err != nil { - t.Fatal(err) - } - first.JSONSchema[0] = '[' - second, err := loadResponseSchema() - if err != nil || !json.Valid(second.JSONSchema) || bytes.Equal(first.JSONSchema, second.JSONSchema) { - t.Fatalf("second schema = %s, %v; want defensive copy", second.JSONSchema, err) - } - diagnostics := second.DiagnosticsMap() - if _, ok := diagnostics["json_schema"]; ok { - t.Fatalf("schema diagnostics included raw content: %#v", diagnostics) - } -} - -func validateJSONSchema(instanceContent, schemaContent []byte) error { - instance, err := jsonschema.UnmarshalJSON(bytes.NewReader(instanceContent)) - if err != nil { - return err - } - schemaDocument, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaContent)) - if err != nil { - return err - } - compiler := jsonschema.NewCompiler() - if err := compiler.AddResource("schema.json", schemaDocument); err != nil { - return err - } - compiled, err := compiler.Compile("schema.json") - if err != nil { - return err - } - return compiled.Validate(instance) -} diff --git a/internal/modules/dnd/extract/npcregistry/test_helpers_test.go b/internal/modules/dnd/extract/npcregistry/test_helpers_test.go index 7cb9f152..35f8b66c 100644 --- a/internal/modules/dnd/extract/npcregistry/test_helpers_test.go +++ b/internal/modules/dnd/extract/npcregistry/test_helpers_test.go @@ -3,14 +3,18 @@ package npcregistry import ( "context" "encoding/json" - "errors" + "strings" "testing" + "gitea.maximumdirect.net/eric/notarius/assets" "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + "go.maximumdirect.net/configkit" + "go.maximumdirect.net/promptkit" ) -func extractionRequest() contracts.TypedExtractionRequest { +func extractionRequest() contracts.TypedExtractionRequest[dnd.NPCRegistry] { doc := sourceDocument() chunk := &source.Chunk{ ID: "session-alpha:chunk:0", @@ -22,12 +26,13 @@ func extractionRequest() contracts.TypedExtractionRequest { Units: append([]source.SourceUnit(nil), doc.Units...), Metadata: map[string]any{"ignored": "chunk metadata"}, } - return contracts.TypedExtractionRequest{ + return contracts.TypedExtractionRequest[dnd.NPCRegistry]{ Source: doc, Chunk: chunk, SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session-alpha.json"), SessionID: "session-123", LLMProfile: "profile-npcs", + Operation: registryOperation(), } } @@ -49,67 +54,97 @@ func responseSourceRefs(startUnitID, endUnitID int) []npcSourceRefResponse { return []npcSourceRefResponse{{StartUnitID: startUnitID, EndUnitID: endUnitID}} } -func newExtractor(t *testing.T, client contracts.StructuredLLMClient, references ...contracts.ReferenceSet) *Extractor { +func registryOperation() contracts.ProducerOperation[dnd.NPCRegistry] { + return contracts.ProducerOperation[dnd.NPCRegistry]{ + Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, Cache: contracts.CacheBypass}, ReuseEligible: true, + Assess: func(context.Context, dnd.NPCRegistry) (contracts.CandidateAssessment, error) { + return contracts.CandidateAssessment{}, nil + }, + } +} + +func newExtractor(t *testing.T, client promptkit.LLMClient, references ...contracts.ReferenceSet) *Extractor { t.Helper() - extractor, err := New(client, Options{}, references...) + type settings struct { + PromptKit promptkit.Settings `yaml:"promptkit"` + } + definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit }) + filesystem := configkit.NewFilesystem("Notarius assets", assets.FS()) + result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("registry fixture", []byte(`promptkit: + profiles: + - id: profile-npcs + endpoint: https://example.invalid/v1 + model: registry-test-model + api_key_required: false +`))}, configkit.LayeredOptions[settings]{Definition: definition, Underlays: []configkit.Underlay{ + promptkit.BundledOpenRouterUnderlay(), + {Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}}, + }}) if err != nil { - t.Fatalf("New() error = %v, want nil", err) + t.Fatal(err) + } + engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client)) + if err != nil { + t.Fatal(err) + } + extractor, err := New(engine, Options{}, references...) + if err != nil { + t.Fatal(err) } return extractor } -func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest) contracts.TypedExtractionRequest { +func messageContaining(t *testing.T, request promptkit.GenerateRequest, text string) promptkit.RenderedMessage { + t.Helper() + for _, message := range request.Prompt.Messages { + if strings.Contains(message.Content, text) { + return message + } + } + t.Fatalf("missing rendered input %q", text) + return promptkit.RenderedMessage{} +} + +func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest[dnd.NPCRegistry]) contracts.TypedExtractionRequest[dnd.NPCRegistry] { req.SourceInput = contracts.NewLLMInputMaterial("source", "application/json", []byte(`{"different":true}`), "sha256:other", "file:///other.json") return req } -func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest { - req.Inputs = req.Inputs.Clone() - correction, err := contracts.CloneSemanticCorrection(req.Correction) - if err != nil { - panic(err) - } - req.Correction = correction - if len(req.Vars) == 0 { - req.Vars = nil - return req - } - vars := make(map[string]any, len(req.Vars)) - for key, value := range req.Vars { - vars[key] = value - } - req.Vars = vars - return req -} - type fakeNPCsLLMClient struct { - response extractionResponse - content []byte - err error - requests []contracts.StructuredCompletionRequest + response extractionResponse + content []byte + responses []string + err error + errors []error + requests []promptkit.GenerateRequest + onGenerate func(context.Context, promptkit.GenerateRequest) } -func (client *fakeNPCsLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { - client.requests = append(client.requests, cloneStructuredCompletionRequest(req)) - if client.err != nil { - return contracts.StructuredCompletionResponse{}, client.err +func (client *fakeNPCsLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + index := len(client.requests) + client.requests = append(client.requests, request) + if client.onGenerate != nil { + client.onGenerate(ctx, request) } - target, ok := out.(*extractionResponse) - if !ok { - return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target") + if err := ctx.Err(); err != nil { + return nil, err + } + if client.err != nil { + return nil, client.err + } + if index < len(client.errors) && client.errors[index] != nil { + return nil, client.errors[index] } content := append([]byte(nil), client.content...) - if len(content) != 0 { - if err := json.Unmarshal(content, target); err != nil { - return contracts.StructuredCompletionResponse{}, err - } - } else { - *target = client.response + if index < len(client.responses) { + content = []byte(client.responses[index]) + } + if len(content) == 0 { var err error content, err = json.Marshal(client.response) if err != nil { - return contracts.StructuredCompletionResponse{}, err + return nil, err } } - return contracts.StructuredCompletionResponse{Content: content}, nil + return &promptkit.GenerateResponse{Content: string(content)}, nil } diff --git a/internal/modules/dnd/validate/locationregistry/shape/validator_test.go b/internal/modules/dnd/validate/locationregistry/shape/validator_test.go index 0e248ad7..891f126a 100644 --- a/internal/modules/dnd/validate/locationregistry/shape/validator_test.go +++ b/internal/modules/dnd/validate/locationregistry/shape/validator_test.go @@ -41,7 +41,7 @@ func TestValidatorApprovesAndRegisters(t *testing.T) { if err := Register(registry); err != nil { t.Fatalf("Register() error = %v", err) } - if got, ok := registry.Spec(Key); !ok || got != Spec() || got.ExecutionClass != contracts.ExecutionClassDeterministic { + if got, ok := registry.Spec(Key); !ok || !reflect.DeepEqual(got, Spec()) || got.ExecutionClass != contracts.ExecutionClassDeterministic { t.Fatalf("registered spec = %#v, ok = %t", got, ok) } if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil {