Migrate registry extraction to native correction and explicit empty fallbacks
This commit is contained in:
7
assets/dnd/item-registry/extract/prompts/correction.md
Normal file
7
assets/dnd/item-registry/extract/prompts/correction.md
Normal file
@@ -0,0 +1,7 @@
|
||||
Revise the previous item registry using these findings:
|
||||
|
||||
{{ input "guidance" }}
|
||||
|
||||
Return the complete replacement registry, not only the changed records. Keep the
|
||||
original transcript and reference context and return only JSON matching the
|
||||
original response contract.
|
||||
@@ -17,7 +17,7 @@ prompts:
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ../../../shared/prompts/common-dnd-system.md
|
||||
- role: user
|
||||
- role: system
|
||||
content_file: ../../../shared/prompts/common-dnd-identity.md
|
||||
- role: user
|
||||
content_file: ../../../shared/prompts/common-dnd-references.md
|
||||
@@ -27,9 +27,9 @@ prompts:
|
||||
content_file: ../../../shared/prompts/common-dnd-transcript-chunk.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
- role: system
|
||||
content_file: ../../../shared/prompts/common-dnd-extraction-evidence.md
|
||||
- role: user
|
||||
- role: system
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
@@ -37,3 +37,17 @@ prompts:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd/item-registry/extract/schemas/dnd_item_registry_llm.v1.json
|
||||
|
||||
- id: dnd.item_registry.repair
|
||||
default_profile: notarius/dnd-extraction
|
||||
inputs:
|
||||
- name: guidance
|
||||
required: true
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: user
|
||||
content_file: ./correction.md
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd/item-registry/extract/schemas/dnd_item_registry_llm.v1.json
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
Revise the previous location registry using these findings:
|
||||
|
||||
{{ input "guidance" }}
|
||||
|
||||
Return the complete replacement registry, not only the changed records. Keep the
|
||||
original transcript and reference context and return only JSON matching the
|
||||
original response contract.
|
||||
@@ -17,7 +17,7 @@ prompts:
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ../../../shared/prompts/common-dnd-system.md
|
||||
- role: user
|
||||
- role: system
|
||||
content_file: ../../../shared/prompts/common-dnd-identity.md
|
||||
- role: user
|
||||
content_file: ../../../shared/prompts/common-dnd-references.md
|
||||
@@ -27,9 +27,9 @@ prompts:
|
||||
content_file: ../../../shared/prompts/common-dnd-transcript-chunk.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
- role: system
|
||||
content_file: ../../../shared/prompts/common-dnd-extraction-evidence.md
|
||||
- role: user
|
||||
- role: system
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
@@ -37,3 +37,17 @@ prompts:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd/location-registry/extract/schemas/dnd_location_registry_llm.v1.json
|
||||
|
||||
- id: dnd.location_registry.repair
|
||||
default_profile: notarius/dnd-extraction
|
||||
inputs:
|
||||
- name: guidance
|
||||
required: true
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: user
|
||||
content_file: ./correction.md
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd/location-registry/extract/schemas/dnd_location_registry_llm.v1.json
|
||||
|
||||
7
assets/dnd/npc-registry/extract/prompts/correction.md
Normal file
7
assets/dnd/npc-registry/extract/prompts/correction.md
Normal file
@@ -0,0 +1,7 @@
|
||||
Revise the previous NPC registry using these findings:
|
||||
|
||||
{{ input "guidance" }}
|
||||
|
||||
Return the complete replacement registry, not only the changed records. Keep the
|
||||
original transcript and reference context and return only JSON matching the
|
||||
original response contract.
|
||||
@@ -17,7 +17,7 @@ prompts:
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ../../../shared/prompts/common-dnd-system.md
|
||||
- role: user
|
||||
- role: system
|
||||
content_file: ../../../shared/prompts/common-dnd-identity.md
|
||||
- role: user
|
||||
content_file: ../../../shared/prompts/common-dnd-references.md
|
||||
@@ -27,9 +27,9 @@ prompts:
|
||||
content_file: ../../../shared/prompts/common-dnd-transcript-chunk.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
- role: system
|
||||
content_file: ../../../shared/prompts/common-dnd-extraction-evidence.md
|
||||
- role: user
|
||||
- role: system
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
@@ -37,3 +37,17 @@ prompts:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd/npc-registry/extract/schemas/dnd_npc_registry_llm.v1.json
|
||||
|
||||
- id: dnd.npc_registry.repair
|
||||
default_profile: notarius/dnd-extraction
|
||||
inputs:
|
||||
- name: guidance
|
||||
required: true
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: user
|
||||
content_file: ./correction.md
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd/npc-registry/extract/schemas/dnd_npc_registry_llm.v1.json
|
||||
|
||||
120
assets/registry_extraction_test.go
Normal file
120
assets/registry_extraction_test.go
Normal file
@@ -0,0 +1,120 @@
|
||||
package assets_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io/fs"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/assets"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestRegistryExtractionPromptPairsRetainDomainRulesAndContracts(t *testing.T) {
|
||||
_, engine := loadNativeAssets(t)
|
||||
for _, test := range []struct {
|
||||
family, instruction string
|
||||
rules []string
|
||||
}{
|
||||
{"npc", "Revise the NPC citations.", []string{"proper name or a stable, individually distinguishing", "factual third-party mention", "Exclude human players"}},
|
||||
{"item", "Revise the item citations.", []string{"named unique items", "concrete reusable item types", "currency denomination", "generic weapons", "Do not use capitalization as an eligibility test"}},
|
||||
{"location", "Revise the location citations.", []string{"stable proper name or unique in-world designation", "the room", "Do not use capitalization as an eligibility test"}},
|
||||
} {
|
||||
t.Run(test.family, func(t *testing.T) {
|
||||
id := "notarius/dnd." + test.family + "_registry"
|
||||
initial, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
|
||||
PromptID: id, APIKey: "test-only",
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.Inline(`{"units":[{"sentinel":"registry-transcript"}]}`),
|
||||
"players": promptkit.Inline("registry-players"), "party": promptkit.Inline("registry-party"), "glossary": promptkit.Inline("registry-glossary"),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer initial.Discard()
|
||||
next, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
|
||||
PromptID: id + ".repair", APIKey: "test-only", Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline(test.instruction)},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer next.Discard()
|
||||
first, correction := initial.Details(), next.Details()
|
||||
if first.SelectedProfileID != "notarius/dnd-extraction" || correction.SelectedProfileID != first.SelectedProfileID ||
|
||||
first.StructuredOutput == nil || correction.StructuredOutput == nil || !reflect.DeepEqual(first.OutputContract, correction.OutputContract) {
|
||||
t.Fatal("registry prompt pair lost native schema/defaults")
|
||||
}
|
||||
var system, all string
|
||||
for _, message := range first.Messages {
|
||||
all += message.Content
|
||||
if message.Role == "system" {
|
||||
system += message.Content
|
||||
}
|
||||
if strings.Contains(message.Content, "registry-transcript") || strings.Contains(message.Content, "registry-players") {
|
||||
if message.CacheControl == nil || message.CacheControl.Type != promptkit.CacheControlEphemeral {
|
||||
t.Fatal("fixed input cache hint lost")
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, rule := range append(test.rules, "Transcript units are the only evidence", "Use the most specific supported in-world") {
|
||||
if !strings.Contains(strings.Join(strings.Fields(system), " "), rule) {
|
||||
t.Fatalf("missing enduring domain rule %q", rule)
|
||||
}
|
||||
}
|
||||
for _, input := range []string{"registry-transcript", "registry-players", "registry-party", "registry-glossary"} {
|
||||
if strings.Count(all, input) != 1 {
|
||||
t.Fatalf("fixed input not rendered once: %q", input)
|
||||
}
|
||||
}
|
||||
if len(correction.Messages) != 1 || correction.Messages[0].Role != "user" || !strings.Contains(correction.Messages[0].Content, test.instruction) ||
|
||||
strings.Contains(correction.Messages[0].Content, "registry-transcript") {
|
||||
t.Fatal("correction repeated fixed inputs instead of supplying findings/assignment")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryResponseSchemasSeparateStructureFromSemanticValidity(t *testing.T) {
|
||||
for _, test := range []struct{ family, field string }{
|
||||
{"npc", "npcs"}, {"item", "items"}, {"location", "locations"},
|
||||
} {
|
||||
t.Run(test.family, func(t *testing.T) {
|
||||
schema, err := fs.ReadFile(assets.FS(), "dnd/"+test.family+"-registry/extract/schemas/dnd_"+test.family+"_registry_llm.v1.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, value := range []struct {
|
||||
name, record string
|
||||
valid bool
|
||||
}{
|
||||
{"grounded record", `{"name":"Canonical Name","source_refs":[{"start_unit_id":1,"end_unit_id":2}]}`, true},
|
||||
{"blank name and empty evidence", `{"name":"","source_refs":[]}`, true},
|
||||
{"nonpositive source candidate", `{"name":"Canonical Name","source_refs":[{"start_unit_id":0,"end_unit_id":-1}]}`, true},
|
||||
{"missing name", `{"source_refs":[]}`, false},
|
||||
{"opaque ID", `{"name":"Canonical Name","source_refs":[],"id":"assigned-later"}`, false},
|
||||
{"wrong name type", `{"name":7,"source_refs":[]}`, false},
|
||||
{"noninteger source identifier", `{"name":"Canonical Name","source_refs":[{"start_unit_id":1.5,"end_unit_id":2}]}`, false},
|
||||
} {
|
||||
t.Run(value.name, func(t *testing.T) {
|
||||
content := []byte(`{"` + test.field + `":[` + value.record + `]}`)
|
||||
if err := validateJSONSchema(content, schema); (err == nil) != value.valid {
|
||||
t.Fatalf("schema validity=%t, error=%v", value.valid, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
for _, content := range []string{`{"` + test.field + `":[]}`, `{"` + test.field + `":null}`, `{}`} {
|
||||
var value map[string]any
|
||||
if err := json.Unmarshal([]byte(content), &value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, isArray := value[test.field].([]any)
|
||||
if err := validateJSONSchema([]byte(content), schema); (err == nil) != isArray {
|
||||
t.Fatalf("collection shape %s: %v", content, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -811,6 +811,29 @@ nested same-backend admission with a limit of one.
|
||||
|
||||
## Stage 18: Migrate The Three Registry Extractors
|
||||
|
||||
**Status:** Complete. NPC, item, and location extraction now use native initial/
|
||||
correction requests through the shared typed producer helper. Transcript and
|
||||
campaign reference materials are captured once per chunk; every response uses
|
||||
the existing family-specific canonicalizer and deterministic identity rules.
|
||||
Added same-schema/default-profile correction pairs and static dependencies,
|
||||
moving enduring identity/evidence/domain principles into initial system context.
|
||||
Private completion, schema/asset registration, and manifest/fingerprint metadata
|
||||
were removed. Canonical naming, currency scope, same-name location anchors,
|
||||
source-order/evidence behavior, and production validator-chain declarations are
|
||||
unchanged. Each family has a codec-valid nonnil empty fallback, qualified by the
|
||||
resolved mode/outcome and excluded from reuse, rather than an invented entity.
|
||||
Retained domain regressions with native provider fakes and added per-family
|
||||
citation correction through real typed validators, fixed input/history and run
|
||||
override checks, strict/permissive exhaustion, ordinary empty acceptance, failed
|
||||
continuation, structural failure, and cancellation coverage. Prompt-pair and
|
||||
structural-versus-semantic schema checks run independently in the asset package.
|
||||
Asset/shape tests and vet/race checks, focused pipeline tests, and
|
||||
`git diff --check` pass. Updated the location-shape spec comparison for the
|
||||
slice-bearing native dependency declaration. All three extractor package tests
|
||||
remain blocked at setup by shared legacy asset helpers importing old ConfigKit's
|
||||
removed YAML `plugin/limit`; their new runtime tests are not claimed passing.
|
||||
Scheduled dependency cleanup and final assembled verification must execute them.
|
||||
|
||||
**Goal:** canonical noun extraction follows the representative native path.
|
||||
|
||||
**Inspect/change:** `internal/modules/dnd/extract/{npcregistry,itemregistry,
|
||||
|
||||
@@ -3,19 +3,21 @@ package itemregistry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"go.maximumdirect.net/promptkit"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||
)
|
||||
|
||||
const (
|
||||
Key = "dnd/item-registry"
|
||||
mappingPolicy = "dnd.item_registry.extract_mapping.v2"
|
||||
)
|
||||
const Key = "dnd/item-registry"
|
||||
const PromptID = "notarius/dnd.item_registry"
|
||||
const CorrectionPromptID = "notarius/dnd.item_registry.repair"
|
||||
|
||||
var requiredCapabilities = []string{"chunks", "source.transcript"}
|
||||
var providedCapabilities = []string{"dnd.item_registry"}
|
||||
@@ -32,75 +34,31 @@ func referenceSlots() []contracts.ReferenceSlot {
|
||||
}
|
||||
|
||||
var _ contracts.Extractor[dnd.ItemRegistry] = (*Extractor)(nil)
|
||||
var _ contracts.ManifestMetadataProvider = (*Extractor)(nil)
|
||||
var _ pipeline.CheckpointFingerprintProvider = (*Extractor)(nil)
|
||||
|
||||
type Options struct{}
|
||||
|
||||
type Extractor struct {
|
||||
llm contracts.StructuredLLMClient
|
||||
promptSHA string
|
||||
responseSchemaSHA string
|
||||
}
|
||||
type Extractor struct{ engine *promptkit.Engine }
|
||||
|
||||
func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) {
|
||||
if llmClient == nil {
|
||||
return nil, extractorErrorf("LLM client must not be nil")
|
||||
func New(engine *promptkit.Engine, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) {
|
||||
if engine == nil {
|
||||
return nil, extractorErrorf("native engine must not be nil")
|
||||
}
|
||||
if len(references) > 1 {
|
||||
return nil, extractorErrorf("at most one reference set may be supplied")
|
||||
}
|
||||
promptSHA, err := promptAssetMetadata()
|
||||
if err != nil {
|
||||
return nil, extractorErrorf("load prompt metadata: %w", err)
|
||||
}
|
||||
responseSchema, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
return nil, extractorErrorf("load response schema: %w", err)
|
||||
}
|
||||
return &Extractor{llm: llmClient, promptSHA: promptSHA, responseSchemaSHA: responseSchema.SHA256}, nil
|
||||
return &Extractor{engine: engine}, nil
|
||||
}
|
||||
|
||||
func (e *Extractor) Key() string { return Key }
|
||||
|
||||
func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot { return referenceSlots() }
|
||||
|
||||
func (e *Extractor) ManifestMetadata() map[string]any {
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
return map[string]any{
|
||||
"prompt_id": PromptID,
|
||||
"prompt_version": SchemaVersion,
|
||||
"prompt_sha256": e.promptSHA,
|
||||
"response_schema_key": string(ResponseSchemaKey),
|
||||
"response_schema_id": ResponseSchemaID,
|
||||
"response_schema_name": ResponseSchemaName,
|
||||
"response_schema_version": SchemaVersion,
|
||||
"response_schema_sha256": e.responseSchemaSHA,
|
||||
"identity_policy": identity.Policy,
|
||||
"mapping_policy": mappingPolicy,
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Extractor) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
return []pipeline.CheckpointFingerprint{
|
||||
{Name: "prompt", Value: e.promptSHA},
|
||||
{Name: "response_schema", Value: e.responseSchemaSHA},
|
||||
{Name: "identity_policy", Value: identity.Policy},
|
||||
{Name: "mapping_policy", Value: mappingPolicy},
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.ItemRegistry], error) {
|
||||
func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest[dnd.ItemRegistry]) (contracts.TypedExtractionResult[dnd.ItemRegistry], error) {
|
||||
if e == nil {
|
||||
return contracts.TypedExtractionResult[dnd.ItemRegistry]{}, extractorErrorf("extractor must not be nil")
|
||||
}
|
||||
if e.llm == nil {
|
||||
return contracts.TypedExtractionResult[dnd.ItemRegistry]{}, extractorErrorf("LLM client must not be nil")
|
||||
if e.engine == nil {
|
||||
return contracts.TypedExtractionResult[dnd.ItemRegistry]{}, extractorErrorf("native engine must not be nil")
|
||||
}
|
||||
sourceInput, err := shared.PrepareChunkExtraction(ctx, req)
|
||||
if err != nil {
|
||||
@@ -108,29 +66,41 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRe
|
||||
}
|
||||
order := shared.NewSourceRefOrder(req.Source)
|
||||
|
||||
var response extractionResponse
|
||||
completion, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||
StageName: Key, PromptID: PromptID, PromptVersion: SchemaVersion,
|
||||
ProfileID: req.LLMProfile, SessionID: req.SessionID, StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts,
|
||||
Correction: req.Correction,
|
||||
Inputs: shared.PromptInputs(sourceInput, req.References),
|
||||
}, &response)
|
||||
if err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.ItemRegistry]{}, extractorErrorf("complete structured output: %w", err)
|
||||
sourceID := req.Source.ID
|
||||
materials := shared.PromptInputs(sourceInput, req.References)
|
||||
inputs := make(map[string]promptkit.ArtifactRef, len(materials))
|
||||
for name, material := range materials {
|
||||
inputs[name] = promptkit.InlineWithURI(material.OriginURI, string(material.Content))
|
||||
}
|
||||
candidate, err := shared.ModelCandidateFromResponse(completion)
|
||||
initial := promptkit.RunRequest{PromptID: PromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID, Inputs: inputs}
|
||||
value, outcome, err := pipeline.RunProducer(ctx, e.engine, req.Operation, initial,
|
||||
func(_ context.Context, result *promptkit.RunResult) (*dnd.ItemRegistry, []string, error) {
|
||||
var response extractionResponse
|
||||
if err := json.Unmarshal([]byte(result.RawOutput), &response); err != nil {
|
||||
return nil, nil, extractorErrorf("decode native output: %w", err)
|
||||
}
|
||||
canonicalizeResponse(&response, order, sourceID)
|
||||
value := canonicalItemRegistry(response, sourceID)
|
||||
return &value, nil, nil
|
||||
},
|
||||
func(_ context.Context, guidance []string) (promptkit.RunRequest, error) {
|
||||
return promptkit.RunRequest{PromptID: CorrectionPromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID,
|
||||
Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline(strings.Join(guidance, "\n\n"))}}, nil
|
||||
},
|
||||
func(context.Context, *dnd.ItemRegistry) (dnd.ItemRegistry, error) {
|
||||
return dnd.ItemRegistry{Items: []dnd.Item{}}, nil
|
||||
})
|
||||
if err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.ItemRegistry]{}, extractorErrorf("capture model candidate: %w", err)
|
||||
return contracts.TypedExtractionResult[dnd.ItemRegistry]{}, extractorErrorf("produce registry: %w", err)
|
||||
}
|
||||
canonicalizeResponse(&response, order, req.Source.ID)
|
||||
return contracts.TypedExtractionResult[dnd.ItemRegistry]{Value: canonicalItemRegistry(response, req.Source.ID), ModelCandidate: candidate}, nil
|
||||
return contracts.TypedExtractionResult[dnd.ItemRegistry]{Value: value, Outcome: outcome}, nil
|
||||
}
|
||||
|
||||
func ModuleSpec() pipeline.ModuleSpec {
|
||||
return pipeline.ModuleSpec{
|
||||
Key: Key, Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked,
|
||||
CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1,
|
||||
Requires: append([]string(nil), requiredCapabilities...), Provides: append([]string(nil), providedCapabilities...),
|
||||
Prompts: []string{PromptID, CorrectionPromptID},
|
||||
Requires: append([]string(nil), requiredCapabilities...), Provides: append([]string(nil), providedCapabilities...),
|
||||
ArtifactKind: dnd.ItemRegistryKind, ReferenceSlots: referenceSlots(),
|
||||
}
|
||||
}
|
||||
@@ -141,7 +111,7 @@ func Register(registry *pipeline.ExtractorRegistry) error {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return New(request.Dependencies.LLM, options, request.References)
|
||||
return New(request.Dependencies.Engine, options, request.References)
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -72,10 +72,11 @@ func TestExtractPassesReferencesWithoutTreatingThemAsEvidence(t *testing.T) {
|
||||
if _, err := newExtractor(t, client).Extract(context.Background(), req); err != nil {
|
||||
t.Fatalf("Extract() error = %v", err)
|
||||
}
|
||||
inputs := client.requests[0].Inputs
|
||||
if string(inputs["glossary"].Content) != "Star Compass: heirloom" || strings.Contains(string(inputs["transcript"].Content), "heirloom") {
|
||||
t.Fatalf("prompt inputs = %#v, want separated reference material", inputs)
|
||||
messageContaining(t, client.requests[0], "Star Compass: heirloom")
|
||||
if strings.Contains(messageContaining(t, client.requests[0], string(req.Chunk.Content)).Content, "heirloom") {
|
||||
t.Fatal("references became source evidence")
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestExtractHandlesCancellationAndFailures(t *testing.T) {
|
||||
@@ -88,11 +89,11 @@ func TestExtractHandlesCancellationAndFailures(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
extractor *Extractor
|
||||
req contracts.TypedExtractionRequest
|
||||
req contracts.TypedExtractionRequest[dnd.ItemRegistry]
|
||||
want string
|
||||
}{
|
||||
{name: "nil extractor", extractor: nil, req: request, want: "extractor"},
|
||||
{name: "nil client", extractor: &Extractor{}, req: request, want: "LLM client"},
|
||||
{name: "nil client", extractor: &Extractor{}, req: request, want: "native engine"},
|
||||
{name: "preflight", extractor: newExtractor(t, &fakeItemsLLMClient{}), req: mismatchedSourceInputRequest(request), want: "must match chunk"},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
package itemregistry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
registrycodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/itemregistry"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity"
|
||||
shape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemregistry/shape"
|
||||
refs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemregistry/source_refs"
|
||||
relatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemregistry/source_relatedness"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
const ungroundedRegistry = `{"items":[{"name":"Star Compass","source_refs":[{"start_unit_id":999,"end_unit_id":999}]}]}`
|
||||
const groundedRegistry = `{"items":[{"name":"Star Compass","source_refs":[{"start_unit_id":2,"end_unit_id":2}]}]}`
|
||||
const emptyRegistry = `{"items":[]}`
|
||||
|
||||
func assessRegistry(ctx context.Context, request contracts.TypedExtractionRequest[dnd.ItemRegistry], value dnd.ItemRegistry) (contracts.CandidateAssessment, error) {
|
||||
assessment := contracts.CandidateAssessment{}
|
||||
for _, validator := range []contracts.TypedValidator[dnd.ItemRegistry]{
|
||||
shape.New(shape.Options{}), refs.New(refs.Options{}), relatedness.New(relatedness.Options{}),
|
||||
} {
|
||||
result, err := validator.Validate(ctx, contracts.TypedValidationRequest[dnd.ItemRegistry]{Stage: "extract", Source: request.Source, Chunk: request.Chunk, Value: value})
|
||||
if err != nil {
|
||||
return contracts.CandidateAssessment{}, err
|
||||
}
|
||||
decision := contracts.AssessmentRejected
|
||||
if result.Approved {
|
||||
decision = contracts.AssessmentApproved
|
||||
}
|
||||
assessment.Validators = append(assessment.Validators, contracts.ValidatorAssessment{ValidatorName: validator.Name(), Decision: decision, Result: result})
|
||||
}
|
||||
return assessment, nil
|
||||
}
|
||||
|
||||
func TestNativeExtractionCorrectsGroundingWithFixedInputs(t *testing.T) {
|
||||
client := &fakeItemsLLMClient{responses: []string{ungroundedRegistry, groundedRegistry}}
|
||||
request := extractionRequest()
|
||||
request.References = contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{"glossary": {Items: []contracts.ReferenceItem{{Content: []byte("fixed-reference")}}}}}
|
||||
request.Operation.Controls.MaxAdditionalTurns = 1
|
||||
request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "registry-override"}
|
||||
var prepared []promptkit.PreparedRun
|
||||
request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = append(prepared, value) }
|
||||
assessments := 0
|
||||
request.Operation.Assess = func(ctx context.Context, value dnd.ItemRegistry) (contracts.CandidateAssessment, error) {
|
||||
assessments++
|
||||
assessment, err := assessRegistry(ctx, request, value)
|
||||
if assessments == 1 {
|
||||
if assessment.Accepted() {
|
||||
t.Fatal("unresolvable citation was accepted")
|
||||
}
|
||||
// Changes after capture must not cause source/reference preparation again.
|
||||
request.SourceInput.Content[0] = '!'
|
||||
request.References.Slots["glossary"].Items[0].Content[0] = '!'
|
||||
}
|
||||
return assessment, err
|
||||
}
|
||||
result, err := newExtractor(t, client).Extract(context.Background(), request)
|
||||
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || !result.Outcome.ReuseEligible ||
|
||||
assessments != 2 || len(client.requests) != 2 || len(prepared) != 2 || len(result.Value.Items) != 1 {
|
||||
t.Fatalf("corrected registry=%#v %v, assessments/provider/prepared=%d/%d/%d", result, err, assessments, len(client.requests), len(prepared))
|
||||
}
|
||||
record := result.Value.Items[0]
|
||||
if record.Name != "Star Compass" || len(record.SourceRefs) != 1 || record.SourceRefs[0].SourceID != request.Source.ID ||
|
||||
record.SourceRefs[0].StartUnitID != 2 || record.SourceRefs[0].EndUnitID != 2 || record.ID != identity.DeriveID(record.Name) {
|
||||
t.Fatalf("canonical record identity/evidence=%#v", record)
|
||||
}
|
||||
if _, err := registrycodec.New().Encode(result.Value); err != nil {
|
||||
t.Fatalf("durable registry encoding: %v", err)
|
||||
}
|
||||
for i, call := range client.requests {
|
||||
id := PromptID
|
||||
if i > 0 {
|
||||
id = CorrectionPromptID
|
||||
}
|
||||
if prepared[i].PromptID != id || prepared[i].SelectedProfileID != request.LLMProfile || call.Target.Model != "registry-override" || call.Prompt.SessionID != request.SessionID {
|
||||
t.Fatal("native selection/overrides lost")
|
||||
}
|
||||
}
|
||||
initial, revised := client.requests[0].Prompt.Messages, client.requests[1].Prompt.Messages
|
||||
if len(revised) != len(initial)+2 || !reflect.DeepEqual(revised[:len(initial)], initial) ||
|
||||
revised[len(initial)].Role != "assistant" || revised[len(initial)].Content != ungroundedRegistry {
|
||||
t.Fatal("native correction lost fixed inputs or prior response")
|
||||
}
|
||||
messageContaining(t, client.requests[1], "fixed-reference")
|
||||
messageContaining(t, client.requests[1], string(request.Chunk.Content))
|
||||
}
|
||||
|
||||
func TestNativeExtractionDistinguishesEmptyAcceptanceFromFallback(t *testing.T) {
|
||||
failure := errors.New("provider unavailable")
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
mode contracts.ExecutionMode
|
||||
responses []string
|
||||
providerErrors []error
|
||||
wantAssessments, wantRuns int
|
||||
wantError, wantFallback bool
|
||||
}{
|
||||
{"ordinary empty", contracts.ModeStrict, []string{emptyRegistry}, nil, 1, 1, false, false},
|
||||
{"strict rejection", contracts.ModeStrict, []string{ungroundedRegistry, ungroundedRegistry}, nil, 2, 2, true, false},
|
||||
{"permissive rejection", contracts.ModePermissive, []string{ungroundedRegistry, ungroundedRegistry}, nil, 2, 2, false, true},
|
||||
{"provider failure", contracts.ModePermissive, nil, []error{failure}, 0, 1, false, true},
|
||||
{"failed correction", contracts.ModePermissive, []string{ungroundedRegistry}, []error{nil, failure}, 1, 2, false, true},
|
||||
{"structural failure", contracts.ModePermissive, []string{`{"items":null}`}, nil, 0, 1, false, true},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
client := &fakeItemsLLMClient{responses: test.responses, errors: test.providerErrors}
|
||||
if test.name == "structural failure" {
|
||||
client.content = []byte(test.responses[0])
|
||||
}
|
||||
request := extractionRequest()
|
||||
request.Operation.Controls.Mode = test.mode
|
||||
request.Operation.Controls.MaxAdditionalTurns = 1
|
||||
assessments, prepared := 0, 0
|
||||
request.Operation.Execution.OnPrepared = func(context.Context, promptkit.PreparedRun) { prepared++ }
|
||||
request.Operation.Assess = func(ctx context.Context, value dnd.ItemRegistry) (contracts.CandidateAssessment, error) {
|
||||
assessments++
|
||||
return assessRegistry(ctx, request, value)
|
||||
}
|
||||
result, err := newExtractor(t, client).Extract(context.Background(), request)
|
||||
if (err != nil) != test.wantError || assessments != test.wantAssessments || prepared != test.wantRuns {
|
||||
t.Fatalf("registry=%#v %v, assessments=%d runs=%d", result, err, assessments, prepared)
|
||||
}
|
||||
if test.wantError {
|
||||
if result.Outcome.Disposition != "" {
|
||||
t.Fatal("strict rejection produced an outcome")
|
||||
}
|
||||
return
|
||||
}
|
||||
if result.Value.Items == nil || len(result.Value.Items) != 0 || (result.Outcome.Disposition == contracts.ProducerFallback) != test.wantFallback ||
|
||||
result.Outcome.ReuseEligible == test.wantFallback {
|
||||
t.Fatalf("unqualified empty registry=%#v", result)
|
||||
}
|
||||
if _, err := registrycodec.New().Encode(result.Value); err != nil {
|
||||
t.Fatalf("empty durable registry invalid: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCanceledNativeExtractionCannotDeliverFallback(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
client := &fakeItemsLLMClient{content: []byte(groundedRegistry), onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }}
|
||||
request := extractionRequest()
|
||||
request.Operation.Controls.Mode = contracts.ModePermissive
|
||||
result, err := newExtractor(t, client).Extract(ctx, request)
|
||||
if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" || len(client.requests) != 1 {
|
||||
t.Fatalf("cancellation=%#v %v", result, err)
|
||||
}
|
||||
}
|
||||
@@ -1,74 +0,0 @@
|
||||
package itemregistry
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"sync"
|
||||
|
||||
rootassets "gitea.maximumdirect.net/eric/notarius/assets"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||
)
|
||||
|
||||
const promptAssetRoot = "assets/prompts"
|
||||
|
||||
var promptAssetManifest = shared.PromptAssetManifest{
|
||||
ModuleDir: PromptID,
|
||||
ModuleFiles: []promptfs.ModulePromptFile{
|
||||
{Name: "prompt.yaml", Path: "prompts/prompt.yaml"},
|
||||
{Name: "instructions.md", Path: "prompts/instructions.md"},
|
||||
},
|
||||
SharedFiles: []string{
|
||||
"common-dnd-system.md",
|
||||
"common-dnd-extraction-evidence.md",
|
||||
"common-dnd-identity.md",
|
||||
"common-dnd-transcript-chunk.md",
|
||||
"common-dnd-references.md",
|
||||
},
|
||||
}
|
||||
|
||||
func moduleAssetFS() (fs.FS, error) {
|
||||
assets, err := fs.Sub(rootassets.FS(), "dnd/item-registry/extract")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scope item extraction assets: %w", err)
|
||||
}
|
||||
return assets, nil
|
||||
}
|
||||
|
||||
func RegisterPromptAssets(registry *llm.AssetRegistry) error {
|
||||
assets, err := moduleAssetFS()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
promptFS, err := promptAssetManifest.PromptFS(assets)
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare item prompt assets: %w", err)
|
||||
}
|
||||
if err := registry.RegisterPromptFS(promptFS, promptAssetRoot); err != nil {
|
||||
return err
|
||||
}
|
||||
schemas, err := fs.Sub(assets, "schemas")
|
||||
if err != nil {
|
||||
return fmt.Errorf("scope item extraction schemas: %w", err)
|
||||
}
|
||||
return registry.RegisterSchemaFS(schemas, ".")
|
||||
}
|
||||
|
||||
func promptAssetMetadata() (string, error) {
|
||||
promptAssetHashOnce.Do(func() {
|
||||
assets, err := moduleAssetFS()
|
||||
if err != nil {
|
||||
promptAssetHashErr = err
|
||||
return
|
||||
}
|
||||
promptAssetHash, promptAssetHashErr = promptAssetManifest.Hash(assets)
|
||||
})
|
||||
return promptAssetHash, promptAssetHashErr
|
||||
}
|
||||
|
||||
var (
|
||||
promptAssetHashOnce sync.Once
|
||||
promptAssetHash string
|
||||
promptAssetHashErr error
|
||||
)
|
||||
@@ -1,41 +0,0 @@
|
||||
package itemregistry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestRegisterPromptAssetsPreparesItemPrompt(t *testing.T) {
|
||||
registry := llm.NewAssetRegistry()
|
||||
if err := RegisterPromptAssets(registry); err != nil {
|
||||
t.Fatalf("RegisterPromptAssets() error = %v", err)
|
||||
}
|
||||
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
|
||||
"item-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "item-test-model"),
|
||||
})
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "item-test-profile", Inputs: map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.Inline(`{"units":[{"sentinel":"item-transcript"}]}`), "players": promptkit.Inline("item-player"), "party": promptkit.Inline(" "), "glossary": promptkit.Inline(" "),
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("Prepare() error = %v", err)
|
||||
}
|
||||
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_item_registry_llm.v1.json" {
|
||||
t.Fatalf("prepared prompt = %#v, want item prompt identity and schema wiring", prepared)
|
||||
}
|
||||
content := make([]string, len(prepared.Messages))
|
||||
for index, message := range prepared.Messages {
|
||||
content[index] = message.Content
|
||||
}
|
||||
rendered := strings.Join(content, "\n")
|
||||
for _, policy := range []string{"named unique items", "concrete reusable item types", "currency denomination", "loot", "generic weapons", "Do not use capitalization as an eligibility test"} {
|
||||
if !strings.Contains(rendered, policy) {
|
||||
t.Fatalf("rendered prompt omits item eligibility policy %q", policy)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,6 @@
|
||||
package itemregistry
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -9,17 +8,16 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity"
|
||||
)
|
||||
|
||||
func TestModuleRegistrationMetadataAndRedaction(t *testing.T) {
|
||||
if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "LLM client") {
|
||||
func TestModuleRegistrationAndNativeDependencies(t *testing.T) {
|
||||
if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "native engine") {
|
||||
t.Fatalf("New(nil) error = %v, want client rejection", err)
|
||||
}
|
||||
if _, err := New(&fakeItemsLLMClient{}, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one") {
|
||||
if _, err := New(newExtractor(t, &fakeItemsLLMClient{}).engine, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one") {
|
||||
t.Fatalf("New() error = %v, want reference-set rejection", err)
|
||||
}
|
||||
want := pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1, Requires: []string{"chunks", "source.transcript"}, Provides: []string{"dnd.item_registry"}, ArtifactKind: dnd.ItemRegistryKind, ReferenceSlots: referenceSlots()}
|
||||
want := pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{PromptID, CorrectionPromptID}, Requires: []string{"chunks", "source.transcript"}, Provides: []string{"dnd.item_registry"}, ArtifactKind: dnd.ItemRegistryKind, ReferenceSlots: referenceSlots()}
|
||||
if got := ModuleSpec(); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
|
||||
}
|
||||
@@ -36,25 +34,5 @@ func TestModuleRegistrationMetadataAndRedaction(t *testing.T) {
|
||||
if _, err := DecodeOptions(map[string]any{"unknown": true}); err == nil {
|
||||
t.Fatal("DecodeOptions() accepted unknown option")
|
||||
}
|
||||
references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
|
||||
"glossary": {Items: []contracts.ReferenceItem{{Content: []byte("secret-campaign-context")}}},
|
||||
}}
|
||||
metadata := newExtractor(t, &fakeItemsLLMClient{}, references).ManifestMetadata()
|
||||
for key, value := range map[string]string{
|
||||
"prompt_id": PromptID, "prompt_version": SchemaVersion,
|
||||
"response_schema_key": string(ResponseSchemaKey), "response_schema_id": ResponseSchemaID,
|
||||
"response_schema_name": ResponseSchemaName, "response_schema_version": SchemaVersion,
|
||||
"identity_policy": identity.Policy, "mapping_policy": mappingPolicy,
|
||||
} {
|
||||
if metadata[key] != value {
|
||||
t.Fatalf("metadata[%q] = %#v, want %q", key, metadata[key], value)
|
||||
}
|
||||
}
|
||||
encoded, err := json.Marshal(metadata)
|
||||
if err != nil || strings.Contains(string(encoded), "secret-campaign-context") || strings.Contains(string(encoded), "session-items") {
|
||||
t.Fatalf("manifest metadata = %s, want static redacted provenance only", encoded)
|
||||
}
|
||||
if got := newExtractor(t, &fakeItemsLLMClient{}).CheckpointFingerprints(); len(got) != 4 || got[0].Name != "prompt" || got[1].Name != "response_schema" || got[2].Value != identity.Policy || got[3].Value != mappingPolicy {
|
||||
t.Fatalf("CheckpointFingerprints() = %#v", got)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
package itemregistry
|
||||
|
||||
import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
|
||||
const (
|
||||
PromptID = "dnd.item_registry"
|
||||
ResponseSchemaKey = llm.ResponseSchemaKey("dnd_item_registry_llm")
|
||||
ResponseSchemaID = "notarius.dnd.item_registry.llm"
|
||||
ResponseSchemaName = "notarius_dnd_item_registry_llm_v1"
|
||||
SchemaVersion = "v1"
|
||||
)
|
||||
|
||||
func loadResponseSchema() (llm.ResponseSchema, error) {
|
||||
assets, err := moduleAssetFS()
|
||||
if err != nil {
|
||||
return llm.ResponseSchema{}, err
|
||||
}
|
||||
return llm.LoadResponseSchema(assets, llm.ResponseSchemaDefinition{
|
||||
Key: ResponseSchemaKey,
|
||||
ID: ResponseSchemaID,
|
||||
Version: SchemaVersion,
|
||||
Name: ResponseSchemaName,
|
||||
AssetPath: "schemas/dnd_item_registry_llm.v1.json",
|
||||
})
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
package itemregistry
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestItemResponseSchemaIsPrivateAndStructural(t *testing.T) {
|
||||
schema, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
t.Fatalf("loadResponseSchema() error = %v", err)
|
||||
}
|
||||
if schema.Key != ResponseSchemaKey || schema.ID != ResponseSchemaID || schema.Name != ResponseSchemaName || schema.Version != SchemaVersion || !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) {
|
||||
t.Fatalf("schema = %#v, want item response schema", schema)
|
||||
}
|
||||
var document map[string]any
|
||||
if err := json.Unmarshal(schema.JSONSchema, &document); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
encoded, err := json.Marshal(document)
|
||||
if err != nil || strings.Contains(string(encoded), `"id"`) || !strings.Contains(string(encoded), `"additionalProperties":false`) {
|
||||
t.Fatalf("schema = %s, want strict private response without durable ID", encoded)
|
||||
}
|
||||
}
|
||||
@@ -3,14 +3,18 @@ package itemregistry
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/assets"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func extractionRequest() contracts.TypedExtractionRequest {
|
||||
func extractionRequest() contracts.TypedExtractionRequest[dnd.ItemRegistry] {
|
||||
doc := sourceDocument()
|
||||
chunk := &source.Chunk{
|
||||
ID: "session-items:chunk:0", SourceID: doc.ID, Index: 0,
|
||||
@@ -18,10 +22,11 @@ func extractionRequest() contracts.TypedExtractionRequest {
|
||||
Content: []byte(`{"units":[1,2,3]}`), MediaType: "application/json",
|
||||
Units: append([]source.SourceUnit(nil), doc.Units...),
|
||||
}
|
||||
return contracts.TypedExtractionRequest{
|
||||
return contracts.TypedExtractionRequest[dnd.ItemRegistry]{
|
||||
Source: doc, Chunk: chunk,
|
||||
SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session-items.json"),
|
||||
SessionID: "item-session", LLMProfile: "item-profile",
|
||||
Operation: registryOperation(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -37,61 +42,97 @@ func responseSourceRefs(startUnitID, endUnitID int) []itemSourceRefResponse {
|
||||
return []itemSourceRefResponse{{StartUnitID: startUnitID, EndUnitID: endUnitID}}
|
||||
}
|
||||
|
||||
func newExtractor(t *testing.T, client contracts.StructuredLLMClient, references ...contracts.ReferenceSet) *Extractor {
|
||||
func registryOperation() contracts.ProducerOperation[dnd.ItemRegistry] {
|
||||
return contracts.ProducerOperation[dnd.ItemRegistry]{
|
||||
Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, Cache: contracts.CacheBypass}, ReuseEligible: true,
|
||||
Assess: func(context.Context, dnd.ItemRegistry) (contracts.CandidateAssessment, error) {
|
||||
return contracts.CandidateAssessment{}, nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func newExtractor(t *testing.T, client promptkit.LLMClient, references ...contracts.ReferenceSet) *Extractor {
|
||||
t.Helper()
|
||||
extractor, err := New(client, Options{}, references...)
|
||||
type settings struct {
|
||||
PromptKit promptkit.Settings `yaml:"promptkit"`
|
||||
}
|
||||
definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit })
|
||||
filesystem := configkit.NewFilesystem("Notarius assets", assets.FS())
|
||||
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("registry fixture", []byte(`promptkit:
|
||||
profiles:
|
||||
- id: item-profile
|
||||
endpoint: https://example.invalid/v1
|
||||
model: registry-test-model
|
||||
api_key_required: false
|
||||
`))}, configkit.LayeredOptions[settings]{Definition: definition, Underlays: []configkit.Underlay{
|
||||
promptkit.BundledOpenRouterUnderlay(),
|
||||
{Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v, want nil", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
extractor, err := New(engine, Options{}, references...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return extractor
|
||||
}
|
||||
|
||||
func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest) contracts.TypedExtractionRequest {
|
||||
func messageContaining(t *testing.T, request promptkit.GenerateRequest, text string) promptkit.RenderedMessage {
|
||||
t.Helper()
|
||||
for _, message := range request.Prompt.Messages {
|
||||
if strings.Contains(message.Content, text) {
|
||||
return message
|
||||
}
|
||||
}
|
||||
t.Fatalf("missing rendered input %q", text)
|
||||
return promptkit.RenderedMessage{}
|
||||
}
|
||||
|
||||
func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest[dnd.ItemRegistry]) contracts.TypedExtractionRequest[dnd.ItemRegistry] {
|
||||
req.SourceInput = contracts.NewLLMInputMaterial("source", "application/json", []byte(`{"different":true}`), "sha256:other", "file:///other.json")
|
||||
return req
|
||||
}
|
||||
|
||||
type fakeItemsLLMClient struct {
|
||||
response extractionResponse
|
||||
content []byte
|
||||
err error
|
||||
requests []contracts.StructuredCompletionRequest
|
||||
response extractionResponse
|
||||
content []byte
|
||||
responses []string
|
||||
err error
|
||||
errors []error
|
||||
requests []promptkit.GenerateRequest
|
||||
onGenerate func(context.Context, promptkit.GenerateRequest)
|
||||
}
|
||||
|
||||
func (client *fakeItemsLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
client.requests = append(client.requests, cloneStructuredCompletionRequest(req))
|
||||
func (client *fakeItemsLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
index := len(client.requests)
|
||||
client.requests = append(client.requests, request)
|
||||
if client.onGenerate != nil {
|
||||
client.onGenerate(ctx, request)
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
return nil, err
|
||||
}
|
||||
if client.err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, client.err
|
||||
return nil, client.err
|
||||
}
|
||||
target, ok := out.(*extractionResponse)
|
||||
if !ok {
|
||||
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target")
|
||||
if index < len(client.errors) && client.errors[index] != nil {
|
||||
return nil, client.errors[index]
|
||||
}
|
||||
content := append([]byte(nil), client.content...)
|
||||
if len(content) != 0 {
|
||||
if err := json.Unmarshal(content, target); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
}
|
||||
} else {
|
||||
*target = client.response
|
||||
if index < len(client.responses) {
|
||||
content = []byte(client.responses[index])
|
||||
}
|
||||
if len(content) == 0 {
|
||||
var err error
|
||||
content, err = json.Marshal(client.response)
|
||||
if err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{Content: content}, nil
|
||||
}
|
||||
|
||||
func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest {
|
||||
req.Inputs = req.Inputs.Clone()
|
||||
correction, err := contracts.CloneSemanticCorrection(req.Correction)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
req.Correction = correction
|
||||
return req
|
||||
return &promptkit.GenerateResponse{Content: string(content)}, nil
|
||||
}
|
||||
|
||||
@@ -3,19 +3,21 @@ package locationregistry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"go.maximumdirect.net/promptkit"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/locations/identity"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||
)
|
||||
|
||||
const (
|
||||
Key = "dnd/location-registry"
|
||||
mappingPolicy = "dnd.location_registry.extract_mapping.v2"
|
||||
)
|
||||
const Key = "dnd/location-registry"
|
||||
const PromptID = "notarius/dnd.location_registry"
|
||||
const CorrectionPromptID = "notarius/dnd.location_registry.repair"
|
||||
|
||||
var requiredCapabilities = []string{"chunks", "source.transcript"}
|
||||
var providedCapabilities = []string{"dnd.location_registry"}
|
||||
@@ -32,75 +34,31 @@ func referenceSlots() []contracts.ReferenceSlot {
|
||||
}
|
||||
|
||||
var _ contracts.Extractor[dnd.LocationRegistry] = (*Extractor)(nil)
|
||||
var _ contracts.ManifestMetadataProvider = (*Extractor)(nil)
|
||||
var _ pipeline.CheckpointFingerprintProvider = (*Extractor)(nil)
|
||||
|
||||
type Options struct{}
|
||||
|
||||
type Extractor struct {
|
||||
llm contracts.StructuredLLMClient
|
||||
promptSHA string
|
||||
responseSchemaSHA string
|
||||
}
|
||||
type Extractor struct{ engine *promptkit.Engine }
|
||||
|
||||
func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) {
|
||||
if llmClient == nil {
|
||||
return nil, extractorErrorf("LLM client must not be nil")
|
||||
func New(engine *promptkit.Engine, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) {
|
||||
if engine == nil {
|
||||
return nil, extractorErrorf("native engine must not be nil")
|
||||
}
|
||||
if len(references) > 1 {
|
||||
return nil, extractorErrorf("at most one reference set may be supplied")
|
||||
}
|
||||
promptSHA, err := promptAssetMetadata()
|
||||
if err != nil {
|
||||
return nil, extractorErrorf("load prompt metadata: %w", err)
|
||||
}
|
||||
responseSchema, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
return nil, extractorErrorf("load response schema: %w", err)
|
||||
}
|
||||
return &Extractor{llm: llmClient, promptSHA: promptSHA, responseSchemaSHA: responseSchema.SHA256}, nil
|
||||
return &Extractor{engine: engine}, nil
|
||||
}
|
||||
|
||||
func (e *Extractor) Key() string { return Key }
|
||||
|
||||
func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot { return referenceSlots() }
|
||||
|
||||
func (e *Extractor) ManifestMetadata() map[string]any {
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
return map[string]any{
|
||||
"prompt_id": PromptID,
|
||||
"prompt_version": SchemaVersion,
|
||||
"prompt_sha256": e.promptSHA,
|
||||
"response_schema_key": string(ResponseSchemaKey),
|
||||
"response_schema_id": ResponseSchemaID,
|
||||
"response_schema_name": ResponseSchemaName,
|
||||
"response_schema_version": SchemaVersion,
|
||||
"response_schema_sha256": e.responseSchemaSHA,
|
||||
"identity_policy": identity.Policy,
|
||||
"mapping_policy": mappingPolicy,
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Extractor) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
return []pipeline.CheckpointFingerprint{
|
||||
{Name: "prompt", Value: e.promptSHA},
|
||||
{Name: "response_schema", Value: e.responseSchemaSHA},
|
||||
{Name: "identity_policy", Value: identity.Policy},
|
||||
{Name: "mapping_policy", Value: mappingPolicy},
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.LocationRegistry], error) {
|
||||
func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest[dnd.LocationRegistry]) (contracts.TypedExtractionResult[dnd.LocationRegistry], error) {
|
||||
if e == nil {
|
||||
return contracts.TypedExtractionResult[dnd.LocationRegistry]{}, extractorErrorf("extractor must not be nil")
|
||||
}
|
||||
if e.llm == nil {
|
||||
return contracts.TypedExtractionResult[dnd.LocationRegistry]{}, extractorErrorf("LLM client must not be nil")
|
||||
if e.engine == nil {
|
||||
return contracts.TypedExtractionResult[dnd.LocationRegistry]{}, extractorErrorf("native engine must not be nil")
|
||||
}
|
||||
sourceInput, err := shared.PrepareChunkExtraction(ctx, req)
|
||||
if err != nil {
|
||||
@@ -108,29 +66,41 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRe
|
||||
}
|
||||
order := shared.NewSourceRefOrder(req.Source)
|
||||
|
||||
var response extractionResponse
|
||||
completion, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||
StageName: Key, PromptID: PromptID, PromptVersion: SchemaVersion,
|
||||
ProfileID: req.LLMProfile, SessionID: req.SessionID, StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts,
|
||||
Correction: req.Correction,
|
||||
Inputs: shared.PromptInputs(sourceInput, req.References),
|
||||
}, &response)
|
||||
if err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.LocationRegistry]{}, extractorErrorf("complete structured output: %w", err)
|
||||
sourceID := req.Source.ID
|
||||
materials := shared.PromptInputs(sourceInput, req.References)
|
||||
inputs := make(map[string]promptkit.ArtifactRef, len(materials))
|
||||
for name, material := range materials {
|
||||
inputs[name] = promptkit.InlineWithURI(material.OriginURI, string(material.Content))
|
||||
}
|
||||
candidate, err := shared.ModelCandidateFromResponse(completion)
|
||||
initial := promptkit.RunRequest{PromptID: PromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID, Inputs: inputs}
|
||||
value, outcome, err := pipeline.RunProducer(ctx, e.engine, req.Operation, initial,
|
||||
func(_ context.Context, result *promptkit.RunResult) (*dnd.LocationRegistry, []string, error) {
|
||||
var response extractionResponse
|
||||
if err := json.Unmarshal([]byte(result.RawOutput), &response); err != nil {
|
||||
return nil, nil, extractorErrorf("decode native output: %w", err)
|
||||
}
|
||||
canonicalizeResponse(&response, order, sourceID)
|
||||
value := canonicalLocationRegistry(response, sourceID)
|
||||
return &value, nil, nil
|
||||
},
|
||||
func(_ context.Context, guidance []string) (promptkit.RunRequest, error) {
|
||||
return promptkit.RunRequest{PromptID: CorrectionPromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID,
|
||||
Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline(strings.Join(guidance, "\n\n"))}}, nil
|
||||
},
|
||||
func(context.Context, *dnd.LocationRegistry) (dnd.LocationRegistry, error) {
|
||||
return dnd.LocationRegistry{Locations: []dnd.Location{}}, nil
|
||||
})
|
||||
if err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.LocationRegistry]{}, extractorErrorf("capture model candidate: %w", err)
|
||||
return contracts.TypedExtractionResult[dnd.LocationRegistry]{}, extractorErrorf("produce registry: %w", err)
|
||||
}
|
||||
canonicalizeResponse(&response, order, req.Source.ID)
|
||||
return contracts.TypedExtractionResult[dnd.LocationRegistry]{Value: canonicalLocationRegistry(response, req.Source.ID), ModelCandidate: candidate}, nil
|
||||
return contracts.TypedExtractionResult[dnd.LocationRegistry]{Value: value, Outcome: outcome}, nil
|
||||
}
|
||||
|
||||
func ModuleSpec() pipeline.ModuleSpec {
|
||||
return pipeline.ModuleSpec{
|
||||
Key: Key, Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked,
|
||||
CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1,
|
||||
Requires: append([]string(nil), requiredCapabilities...), Provides: append([]string(nil), providedCapabilities...),
|
||||
Prompts: []string{PromptID, CorrectionPromptID},
|
||||
Requires: append([]string(nil), requiredCapabilities...), Provides: append([]string(nil), providedCapabilities...),
|
||||
ArtifactKind: dnd.LocationRegistryKind, ReferenceSlots: referenceSlots(),
|
||||
}
|
||||
}
|
||||
@@ -141,7 +111,7 @@ func Register(registry *pipeline.ExtractorRegistry) error {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return New(request.Dependencies.LLM, options, request.References)
|
||||
return New(request.Dependencies.Engine, options, request.References)
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -75,10 +75,11 @@ func TestExtractPassesReferencesWithoutTreatingThemAsEvidence(t *testing.T) {
|
||||
if _, err := newExtractor(t, client).Extract(context.Background(), req); err != nil {
|
||||
t.Fatalf("Extract() error = %v", err)
|
||||
}
|
||||
inputs := client.requests[0].Inputs
|
||||
if string(inputs["glossary"].Content) != "Old Mill: abandoned granary" || strings.Contains(string(inputs["transcript"].Content), "abandoned granary") {
|
||||
t.Fatalf("prompt inputs = %#v, want separated reference material", inputs)
|
||||
messageContaining(t, client.requests[0], "Old Mill: abandoned granary")
|
||||
if strings.Contains(messageContaining(t, client.requests[0], string(req.Chunk.Content)).Content, "abandoned granary") {
|
||||
t.Fatal("references became source evidence")
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestExtractDoesNotMutateRequestMaterials(t *testing.T) {
|
||||
@@ -105,11 +106,11 @@ func TestExtractHandlesEmptyOutputAndLocalFailures(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
extractor *Extractor
|
||||
req contracts.TypedExtractionRequest
|
||||
req contracts.TypedExtractionRequest[dnd.LocationRegistry]
|
||||
want string
|
||||
}{
|
||||
{name: "nil extractor", extractor: nilExtractor, req: request, want: "extractor"},
|
||||
{name: "nil client", extractor: &Extractor{}, req: request, want: "LLM client"},
|
||||
{name: "nil client", extractor: &Extractor{}, req: request, want: "native engine"},
|
||||
{name: "preflight", extractor: newExtractor(t, &fakeLocationsLLMClient{}), req: mismatchedSourceInputRequest(request), want: "must match chunk"},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
package locationregistry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
registrycodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/locationregistry"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/locations/identity"
|
||||
shape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/locationregistry/shape"
|
||||
refs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/locationregistry/source_refs"
|
||||
relatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/locationregistry/source_relatedness"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
const ungroundedRegistry = `{"locations":[{"name":"Old Mill","source_refs":[{"start_unit_id":999,"end_unit_id":999}]}]}`
|
||||
const groundedRegistry = `{"locations":[{"name":"Old Mill","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`
|
||||
const emptyRegistry = `{"locations":[]}`
|
||||
|
||||
func assessRegistry(ctx context.Context, request contracts.TypedExtractionRequest[dnd.LocationRegistry], value dnd.LocationRegistry) (contracts.CandidateAssessment, error) {
|
||||
assessment := contracts.CandidateAssessment{}
|
||||
for _, validator := range []contracts.TypedValidator[dnd.LocationRegistry]{
|
||||
shape.New(shape.Options{}), refs.New(refs.Options{}), relatedness.New(relatedness.Options{}),
|
||||
} {
|
||||
result, err := validator.Validate(ctx, contracts.TypedValidationRequest[dnd.LocationRegistry]{Stage: "extract", Source: request.Source, Chunk: request.Chunk, Value: value})
|
||||
if err != nil {
|
||||
return contracts.CandidateAssessment{}, err
|
||||
}
|
||||
decision := contracts.AssessmentRejected
|
||||
if result.Approved {
|
||||
decision = contracts.AssessmentApproved
|
||||
}
|
||||
assessment.Validators = append(assessment.Validators, contracts.ValidatorAssessment{ValidatorName: validator.Name(), Decision: decision, Result: result})
|
||||
}
|
||||
return assessment, nil
|
||||
}
|
||||
|
||||
func TestNativeExtractionCorrectsGroundingWithFixedInputs(t *testing.T) {
|
||||
client := &fakeLocationsLLMClient{responses: []string{ungroundedRegistry, groundedRegistry}}
|
||||
request := extractionRequest()
|
||||
request.References = contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{"glossary": {Items: []contracts.ReferenceItem{{Content: []byte("fixed-reference")}}}}}
|
||||
request.Operation.Controls.MaxAdditionalTurns = 1
|
||||
request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "registry-override"}
|
||||
var prepared []promptkit.PreparedRun
|
||||
request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = append(prepared, value) }
|
||||
assessments := 0
|
||||
request.Operation.Assess = func(ctx context.Context, value dnd.LocationRegistry) (contracts.CandidateAssessment, error) {
|
||||
assessments++
|
||||
assessment, err := assessRegistry(ctx, request, value)
|
||||
if assessments == 1 {
|
||||
if assessment.Accepted() {
|
||||
t.Fatal("unresolvable citation was accepted")
|
||||
}
|
||||
// Changes after capture must not cause source/reference preparation again.
|
||||
request.SourceInput.Content[0] = '!'
|
||||
request.References.Slots["glossary"].Items[0].Content[0] = '!'
|
||||
}
|
||||
return assessment, err
|
||||
}
|
||||
result, err := newExtractor(t, client).Extract(context.Background(), request)
|
||||
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || !result.Outcome.ReuseEligible ||
|
||||
assessments != 2 || len(client.requests) != 2 || len(prepared) != 2 || len(result.Value.Locations) != 1 {
|
||||
t.Fatalf("corrected registry=%#v %v, assessments/provider/prepared=%d/%d/%d", result, err, assessments, len(client.requests), len(prepared))
|
||||
}
|
||||
record := result.Value.Locations[0]
|
||||
if record.Name != "Old Mill" || len(record.SourceRefs) != 1 || record.SourceRefs[0].SourceID != request.Source.ID ||
|
||||
record.SourceRefs[0].StartUnitID != 1 || record.SourceRefs[0].EndUnitID != 1 || record.ID != identity.DeriveID(record.Name, record.SourceRefs) {
|
||||
t.Fatalf("canonical record identity/evidence=%#v", record)
|
||||
}
|
||||
if _, err := registrycodec.New().Encode(result.Value); err != nil {
|
||||
t.Fatalf("durable registry encoding: %v", err)
|
||||
}
|
||||
for i, call := range client.requests {
|
||||
id := PromptID
|
||||
if i > 0 {
|
||||
id = CorrectionPromptID
|
||||
}
|
||||
if prepared[i].PromptID != id || prepared[i].SelectedProfileID != request.LLMProfile || call.Target.Model != "registry-override" || call.Prompt.SessionID != request.SessionID {
|
||||
t.Fatal("native selection/overrides lost")
|
||||
}
|
||||
}
|
||||
initial, revised := client.requests[0].Prompt.Messages, client.requests[1].Prompt.Messages
|
||||
if len(revised) != len(initial)+2 || !reflect.DeepEqual(revised[:len(initial)], initial) ||
|
||||
revised[len(initial)].Role != "assistant" || revised[len(initial)].Content != ungroundedRegistry {
|
||||
t.Fatal("native correction lost fixed inputs or prior response")
|
||||
}
|
||||
messageContaining(t, client.requests[1], "fixed-reference")
|
||||
messageContaining(t, client.requests[1], string(request.Chunk.Content))
|
||||
}
|
||||
|
||||
func TestNativeExtractionDistinguishesEmptyAcceptanceFromFallback(t *testing.T) {
|
||||
failure := errors.New("provider unavailable")
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
mode contracts.ExecutionMode
|
||||
responses []string
|
||||
providerErrors []error
|
||||
wantAssessments, wantRuns int
|
||||
wantError, wantFallback bool
|
||||
}{
|
||||
{"ordinary empty", contracts.ModeStrict, []string{emptyRegistry}, nil, 1, 1, false, false},
|
||||
{"strict rejection", contracts.ModeStrict, []string{ungroundedRegistry, ungroundedRegistry}, nil, 2, 2, true, false},
|
||||
{"permissive rejection", contracts.ModePermissive, []string{ungroundedRegistry, ungroundedRegistry}, nil, 2, 2, false, true},
|
||||
{"provider failure", contracts.ModePermissive, nil, []error{failure}, 0, 1, false, true},
|
||||
{"failed correction", contracts.ModePermissive, []string{ungroundedRegistry}, []error{nil, failure}, 1, 2, false, true},
|
||||
{"structural failure", contracts.ModePermissive, []string{`{"locations":null}`}, nil, 0, 1, false, true},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
client := &fakeLocationsLLMClient{responses: test.responses, errors: test.providerErrors}
|
||||
if test.name == "structural failure" {
|
||||
client.content = []byte(test.responses[0])
|
||||
}
|
||||
request := extractionRequest()
|
||||
request.Operation.Controls.Mode = test.mode
|
||||
request.Operation.Controls.MaxAdditionalTurns = 1
|
||||
assessments, prepared := 0, 0
|
||||
request.Operation.Execution.OnPrepared = func(context.Context, promptkit.PreparedRun) { prepared++ }
|
||||
request.Operation.Assess = func(ctx context.Context, value dnd.LocationRegistry) (contracts.CandidateAssessment, error) {
|
||||
assessments++
|
||||
return assessRegistry(ctx, request, value)
|
||||
}
|
||||
result, err := newExtractor(t, client).Extract(context.Background(), request)
|
||||
if (err != nil) != test.wantError || assessments != test.wantAssessments || prepared != test.wantRuns {
|
||||
t.Fatalf("registry=%#v %v, assessments=%d runs=%d", result, err, assessments, prepared)
|
||||
}
|
||||
if test.wantError {
|
||||
if result.Outcome.Disposition != "" {
|
||||
t.Fatal("strict rejection produced an outcome")
|
||||
}
|
||||
return
|
||||
}
|
||||
if result.Value.Locations == nil || len(result.Value.Locations) != 0 || (result.Outcome.Disposition == contracts.ProducerFallback) != test.wantFallback ||
|
||||
result.Outcome.ReuseEligible == test.wantFallback {
|
||||
t.Fatalf("unqualified empty registry=%#v", result)
|
||||
}
|
||||
if _, err := registrycodec.New().Encode(result.Value); err != nil {
|
||||
t.Fatalf("empty durable registry invalid: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCanceledNativeExtractionCannotDeliverFallback(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
client := &fakeLocationsLLMClient{content: []byte(groundedRegistry), onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }}
|
||||
request := extractionRequest()
|
||||
request.Operation.Controls.Mode = contracts.ModePermissive
|
||||
result, err := newExtractor(t, client).Extract(ctx, request)
|
||||
if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" || len(client.requests) != 1 {
|
||||
t.Fatalf("cancellation=%#v %v", result, err)
|
||||
}
|
||||
}
|
||||
@@ -1,74 +0,0 @@
|
||||
package locationregistry
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"sync"
|
||||
|
||||
rootassets "gitea.maximumdirect.net/eric/notarius/assets"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||
)
|
||||
|
||||
const promptAssetRoot = "assets/prompts"
|
||||
|
||||
var promptAssetManifest = shared.PromptAssetManifest{
|
||||
ModuleDir: PromptID,
|
||||
ModuleFiles: []promptfs.ModulePromptFile{
|
||||
{Name: "prompt.yaml", Path: "prompts/prompt.yaml"},
|
||||
{Name: "instructions.md", Path: "prompts/instructions.md"},
|
||||
},
|
||||
SharedFiles: []string{
|
||||
"common-dnd-system.md",
|
||||
"common-dnd-extraction-evidence.md",
|
||||
"common-dnd-identity.md",
|
||||
"common-dnd-transcript-chunk.md",
|
||||
"common-dnd-references.md",
|
||||
},
|
||||
}
|
||||
|
||||
func moduleAssetFS() (fs.FS, error) {
|
||||
assets, err := fs.Sub(rootassets.FS(), "dnd/location-registry/extract")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scope location extraction assets: %w", err)
|
||||
}
|
||||
return assets, nil
|
||||
}
|
||||
|
||||
func RegisterPromptAssets(registry *llm.AssetRegistry) error {
|
||||
assets, err := moduleAssetFS()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
promptFS, err := promptAssetManifest.PromptFS(assets)
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare location prompt assets: %w", err)
|
||||
}
|
||||
if err := registry.RegisterPromptFS(promptFS, promptAssetRoot); err != nil {
|
||||
return err
|
||||
}
|
||||
schemas, err := fs.Sub(assets, "schemas")
|
||||
if err != nil {
|
||||
return fmt.Errorf("scope location extraction schemas: %w", err)
|
||||
}
|
||||
return registry.RegisterSchemaFS(schemas, ".")
|
||||
}
|
||||
|
||||
func promptAssetMetadata() (string, error) {
|
||||
promptAssetHashOnce.Do(func() {
|
||||
assets, err := moduleAssetFS()
|
||||
if err != nil {
|
||||
promptAssetHashErr = err
|
||||
return
|
||||
}
|
||||
promptAssetHash, promptAssetHashErr = promptAssetManifest.Hash(assets)
|
||||
})
|
||||
return promptAssetHash, promptAssetHashErr
|
||||
}
|
||||
|
||||
var (
|
||||
promptAssetHashOnce sync.Once
|
||||
promptAssetHash string
|
||||
promptAssetHashErr error
|
||||
)
|
||||
@@ -1,39 +0,0 @@
|
||||
package locationregistry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestRegisterPromptAssetsPreparesLocationPrompt(t *testing.T) {
|
||||
registry := llm.NewAssetRegistry()
|
||||
if err := RegisterPromptAssets(registry); err != nil {
|
||||
t.Fatalf("RegisterPromptAssets() error = %v", err)
|
||||
}
|
||||
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
|
||||
"location-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "location-test-model"),
|
||||
})
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "location-test-profile", Inputs: map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.Inline(`{"units":[{"sentinel":"location-transcript"}]}`), "players": promptkit.Inline("location-player"), "party": promptkit.Inline(" "), "glossary": promptkit.Inline(" "),
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("Prepare() error = %v", err)
|
||||
}
|
||||
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_location_registry_llm.v1.json" {
|
||||
t.Fatalf("prepared prompt = %#v, want location prompt identity and schema wiring", prepared)
|
||||
}
|
||||
content := make([]string, len(prepared.Messages))
|
||||
for index, message := range prepared.Messages {
|
||||
content[index] = message.Content
|
||||
}
|
||||
rendered := strings.Join(content, "\n")
|
||||
if !strings.Contains(rendered, "stable proper name or unique in-world designation") || !strings.Contains(rendered, "the room") || !strings.Contains(rendered, "Do not use capitalization as an eligibility test") {
|
||||
t.Fatalf("rendered prompt = %q, want named-or-unique location eligibility rules", rendered)
|
||||
}
|
||||
}
|
||||
@@ -8,17 +8,16 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/locations/identity"
|
||||
)
|
||||
|
||||
func TestModuleRegistrationAndMetadata(t *testing.T) {
|
||||
if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "LLM client") {
|
||||
func TestModuleRegistrationAndNativeDependencies(t *testing.T) {
|
||||
if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "native engine") {
|
||||
t.Fatalf("New(nil) error = %v, want client rejection", err)
|
||||
}
|
||||
if _, err := New(&fakeLocationsLLMClient{}, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one") {
|
||||
if _, err := New(newExtractor(t, &fakeLocationsLLMClient{}).engine, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one") {
|
||||
t.Fatalf("New() error = %v, want reference-set rejection", err)
|
||||
}
|
||||
want := pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1, Requires: []string{"chunks", "source.transcript"}, Provides: []string{"dnd.location_registry"}, ArtifactKind: dnd.LocationRegistryKind, ReferenceSlots: referenceSlots()}
|
||||
want := pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{PromptID, CorrectionPromptID}, Requires: []string{"chunks", "source.transcript"}, Provides: []string{"dnd.location_registry"}, ArtifactKind: dnd.LocationRegistryKind, ReferenceSlots: referenceSlots()}
|
||||
if got := ModuleSpec(); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
|
||||
}
|
||||
@@ -35,19 +34,5 @@ func TestModuleRegistrationAndMetadata(t *testing.T) {
|
||||
if _, err := DecodeOptions(map[string]any{"unknown": true}); err == nil {
|
||||
t.Fatal("DecodeOptions() accepted unknown option")
|
||||
}
|
||||
extractor := newExtractor(t, &fakeLocationsLLMClient{})
|
||||
metadata := extractor.ManifestMetadata()
|
||||
for key, value := range map[string]string{
|
||||
"prompt_id": PromptID, "prompt_version": SchemaVersion,
|
||||
"response_schema_key": string(ResponseSchemaKey), "response_schema_id": ResponseSchemaID,
|
||||
"response_schema_name": ResponseSchemaName, "response_schema_version": SchemaVersion,
|
||||
"identity_policy": identity.Policy, "mapping_policy": mappingPolicy,
|
||||
} {
|
||||
if metadata[key] != value {
|
||||
t.Fatalf("metadata[%q] = %#v, want %q", key, metadata[key], value)
|
||||
}
|
||||
}
|
||||
if got := extractor.CheckpointFingerprints(); len(got) != 4 || got[0].Name != "prompt" || got[1].Name != "response_schema" || got[2].Value != identity.Policy || got[3].Value != mappingPolicy {
|
||||
t.Fatalf("CheckpointFingerprints() = %#v", got)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
package locationregistry
|
||||
|
||||
import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
|
||||
const (
|
||||
PromptID = "dnd.location_registry"
|
||||
ResponseSchemaKey = llm.ResponseSchemaKey("dnd_location_registry_llm")
|
||||
ResponseSchemaID = "notarius.dnd.location_registry.llm"
|
||||
ResponseSchemaName = "notarius_dnd_location_registry_llm_v1"
|
||||
SchemaVersion = "v1"
|
||||
)
|
||||
|
||||
func loadResponseSchema() (llm.ResponseSchema, error) {
|
||||
assets, err := moduleAssetFS()
|
||||
if err != nil {
|
||||
return llm.ResponseSchema{}, err
|
||||
}
|
||||
return llm.LoadResponseSchema(assets, llm.ResponseSchemaDefinition{
|
||||
Key: ResponseSchemaKey,
|
||||
ID: ResponseSchemaID,
|
||||
Version: SchemaVersion,
|
||||
Name: ResponseSchemaName,
|
||||
AssetPath: "schemas/dnd_location_registry_llm.v1.json",
|
||||
})
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
package locationregistry
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLocationResponseSchemaIsPrivateAndStructural(t *testing.T) {
|
||||
schema, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
t.Fatalf("loadResponseSchema() error = %v", err)
|
||||
}
|
||||
if schema.Key != ResponseSchemaKey || schema.ID != ResponseSchemaID || schema.Name != ResponseSchemaName || schema.Version != SchemaVersion || !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) {
|
||||
t.Fatalf("schema = %#v, want location response schema", schema)
|
||||
}
|
||||
var document map[string]any
|
||||
if err := json.Unmarshal(schema.JSONSchema, &document); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
encoded, err := json.Marshal(document)
|
||||
if err != nil || strings.Contains(string(encoded), `"id"`) {
|
||||
t.Fatalf("schema = %s, want no durable ID field", encoded)
|
||||
}
|
||||
}
|
||||
@@ -3,14 +3,18 @@ package locationregistry
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/assets"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func extractionRequest() contracts.TypedExtractionRequest {
|
||||
func extractionRequest() contracts.TypedExtractionRequest[dnd.LocationRegistry] {
|
||||
doc := sourceDocument()
|
||||
chunk := &source.Chunk{
|
||||
ID: "session-locations:chunk:0", SourceID: doc.ID, Index: 0,
|
||||
@@ -18,10 +22,11 @@ func extractionRequest() contracts.TypedExtractionRequest {
|
||||
Content: []byte(`{"units":[1,2,3]}`), MediaType: "application/json",
|
||||
Units: append([]source.SourceUnit(nil), doc.Units...),
|
||||
}
|
||||
return contracts.TypedExtractionRequest{
|
||||
return contracts.TypedExtractionRequest[dnd.LocationRegistry]{
|
||||
Source: doc, Chunk: chunk,
|
||||
SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session-locations.json"),
|
||||
SessionID: "location-session", LLMProfile: "location-profile",
|
||||
Operation: registryOperation(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -37,58 +42,97 @@ func responseSourceRefs(startUnitID, endUnitID int) []locationSourceRefResponse
|
||||
return []locationSourceRefResponse{{StartUnitID: startUnitID, EndUnitID: endUnitID}}
|
||||
}
|
||||
|
||||
func newExtractor(t *testing.T, client contracts.StructuredLLMClient, references ...contracts.ReferenceSet) *Extractor {
|
||||
func registryOperation() contracts.ProducerOperation[dnd.LocationRegistry] {
|
||||
return contracts.ProducerOperation[dnd.LocationRegistry]{
|
||||
Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, Cache: contracts.CacheBypass}, ReuseEligible: true,
|
||||
Assess: func(context.Context, dnd.LocationRegistry) (contracts.CandidateAssessment, error) {
|
||||
return contracts.CandidateAssessment{}, nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func newExtractor(t *testing.T, client promptkit.LLMClient, references ...contracts.ReferenceSet) *Extractor {
|
||||
t.Helper()
|
||||
extractor, err := New(client, Options{}, references...)
|
||||
type settings struct {
|
||||
PromptKit promptkit.Settings `yaml:"promptkit"`
|
||||
}
|
||||
definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit })
|
||||
filesystem := configkit.NewFilesystem("Notarius assets", assets.FS())
|
||||
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("registry fixture", []byte(`promptkit:
|
||||
profiles:
|
||||
- id: location-profile
|
||||
endpoint: https://example.invalid/v1
|
||||
model: registry-test-model
|
||||
api_key_required: false
|
||||
`))}, configkit.LayeredOptions[settings]{Definition: definition, Underlays: []configkit.Underlay{
|
||||
promptkit.BundledOpenRouterUnderlay(),
|
||||
{Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v, want nil", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
extractor, err := New(engine, Options{}, references...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return extractor
|
||||
}
|
||||
|
||||
func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest) contracts.TypedExtractionRequest {
|
||||
func messageContaining(t *testing.T, request promptkit.GenerateRequest, text string) promptkit.RenderedMessage {
|
||||
t.Helper()
|
||||
for _, message := range request.Prompt.Messages {
|
||||
if strings.Contains(message.Content, text) {
|
||||
return message
|
||||
}
|
||||
}
|
||||
t.Fatalf("missing rendered input %q", text)
|
||||
return promptkit.RenderedMessage{}
|
||||
}
|
||||
|
||||
func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest[dnd.LocationRegistry]) contracts.TypedExtractionRequest[dnd.LocationRegistry] {
|
||||
req.SourceInput = contracts.NewLLMInputMaterial("source", "application/json", []byte(`{"different":true}`), "sha256:other", "file:///other.json")
|
||||
return req
|
||||
}
|
||||
|
||||
type fakeLocationsLLMClient struct {
|
||||
response extractionResponse
|
||||
content []byte
|
||||
err error
|
||||
requests []contracts.StructuredCompletionRequest
|
||||
response extractionResponse
|
||||
content []byte
|
||||
responses []string
|
||||
err error
|
||||
errors []error
|
||||
requests []promptkit.GenerateRequest
|
||||
onGenerate func(context.Context, promptkit.GenerateRequest)
|
||||
}
|
||||
|
||||
func (client *fakeLocationsLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
client.requests = append(client.requests, cloneStructuredCompletionRequest(req))
|
||||
if client.err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, client.err
|
||||
func (client *fakeLocationsLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
index := len(client.requests)
|
||||
client.requests = append(client.requests, request)
|
||||
if client.onGenerate != nil {
|
||||
client.onGenerate(ctx, request)
|
||||
}
|
||||
target, ok := out.(*extractionResponse)
|
||||
if !ok {
|
||||
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target")
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if client.err != nil {
|
||||
return nil, client.err
|
||||
}
|
||||
if index < len(client.errors) && client.errors[index] != nil {
|
||||
return nil, client.errors[index]
|
||||
}
|
||||
content := append([]byte(nil), client.content...)
|
||||
if len(content) != 0 {
|
||||
if err := json.Unmarshal(content, target); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
}
|
||||
} else {
|
||||
*target = client.response
|
||||
if index < len(client.responses) {
|
||||
content = []byte(client.responses[index])
|
||||
}
|
||||
if len(content) == 0 {
|
||||
var err error
|
||||
content, err = json.Marshal(client.response)
|
||||
if err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{Content: content}, nil
|
||||
}
|
||||
|
||||
func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest {
|
||||
req.Inputs = req.Inputs.Clone()
|
||||
correction, err := contracts.CloneSemanticCorrection(req.Correction)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
req.Correction = correction
|
||||
return req
|
||||
return &promptkit.GenerateResponse{Content: string(content)}, nil
|
||||
}
|
||||
|
||||
@@ -2,18 +2,22 @@ package npcregistry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"go.maximumdirect.net/promptkit"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||
)
|
||||
|
||||
const Key = "dnd/npc-registry"
|
||||
|
||||
const mappingPolicy = "dnd.npc_registry.extract_mapping.v3"
|
||||
const PromptID = "notarius/dnd.npc_registry"
|
||||
const CorrectionPromptID = "notarius/dnd.npc_registry.repair"
|
||||
|
||||
var requiredCapabilities = []string{
|
||||
"chunks",
|
||||
@@ -36,79 +40,31 @@ func referenceSlots() []contracts.ReferenceSlot {
|
||||
}
|
||||
|
||||
var _ contracts.Extractor[dnd.NPCRegistry] = (*Extractor)(nil)
|
||||
var _ contracts.ManifestMetadataProvider = (*Extractor)(nil)
|
||||
var _ pipeline.CheckpointFingerprintProvider = (*Extractor)(nil)
|
||||
|
||||
type Options struct{}
|
||||
|
||||
type Extractor struct {
|
||||
llm contracts.StructuredLLMClient
|
||||
promptSHA string
|
||||
responseSchemaSHA string
|
||||
}
|
||||
type Extractor struct{ engine *promptkit.Engine }
|
||||
|
||||
func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) {
|
||||
if llmClient == nil {
|
||||
return nil, extractorErrorf("LLM client must not be nil")
|
||||
func New(engine *promptkit.Engine, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) {
|
||||
if engine == nil {
|
||||
return nil, extractorErrorf("native engine must not be nil")
|
||||
}
|
||||
if len(references) > 1 {
|
||||
return nil, extractorErrorf("at most one reference set may be supplied")
|
||||
}
|
||||
promptSHA, err := promptAssetMetadata()
|
||||
if err != nil {
|
||||
return nil, extractorErrorf("load prompt metadata: %w", err)
|
||||
}
|
||||
responseSchema, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
return nil, extractorErrorf("load response schema: %w", err)
|
||||
}
|
||||
return &Extractor{
|
||||
llm: llmClient,
|
||||
promptSHA: promptSHA,
|
||||
responseSchemaSHA: responseSchema.SHA256,
|
||||
}, nil
|
||||
return &Extractor{engine: engine}, nil
|
||||
}
|
||||
|
||||
func (e *Extractor) Key() string { return Key }
|
||||
|
||||
func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot { return referenceSlots() }
|
||||
|
||||
func (e *Extractor) ManifestMetadata() map[string]any {
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
return map[string]any{
|
||||
"prompt_id": PromptID,
|
||||
"prompt_version": SchemaVersion,
|
||||
"prompt_sha256": e.promptSHA,
|
||||
"response_schema_key": string(ResponseSchemaKey),
|
||||
"response_schema_id": ResponseSchemaID,
|
||||
"response_schema_name": ResponseSchemaName,
|
||||
"response_schema_version": SchemaVersion,
|
||||
"response_schema_sha256": e.responseSchemaSHA,
|
||||
"identity_policy": identity.Policy,
|
||||
"mapping_policy": mappingPolicy,
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Extractor) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
return []pipeline.CheckpointFingerprint{
|
||||
{Name: "prompt", Value: e.promptSHA},
|
||||
{Name: "response_schema", Value: e.responseSchemaSHA},
|
||||
{Name: "identity_policy", Value: identity.Policy},
|
||||
{Name: "mapping_policy", Value: mappingPolicy},
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.NPCRegistry], error) {
|
||||
func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest[dnd.NPCRegistry]) (contracts.TypedExtractionResult[dnd.NPCRegistry], error) {
|
||||
if e == nil {
|
||||
return contracts.TypedExtractionResult[dnd.NPCRegistry]{}, extractorErrorf("extractor must not be nil")
|
||||
}
|
||||
if e.llm == nil {
|
||||
return contracts.TypedExtractionResult[dnd.NPCRegistry]{}, extractorErrorf("LLM client must not be nil")
|
||||
if e.engine == nil {
|
||||
return contracts.TypedExtractionResult[dnd.NPCRegistry]{}, extractorErrorf("native engine must not be nil")
|
||||
}
|
||||
sourceInput, err := shared.PrepareChunkExtraction(ctx, req)
|
||||
if err != nil {
|
||||
@@ -116,38 +72,46 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRe
|
||||
}
|
||||
order := shared.NewSourceRefOrder(req.Source)
|
||||
|
||||
var response extractionResponse
|
||||
completion, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||
StageName: Key,
|
||||
PromptID: PromptID,
|
||||
PromptVersion: SchemaVersion,
|
||||
ProfileID: req.LLMProfile,
|
||||
SessionID: req.SessionID,
|
||||
StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts,
|
||||
Correction: req.Correction,
|
||||
Inputs: shared.PromptInputs(sourceInput, req.References),
|
||||
}, &response)
|
||||
if err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.NPCRegistry]{}, extractorErrorf("complete structured output: %w", err)
|
||||
sourceID := req.Source.ID
|
||||
materials := shared.PromptInputs(sourceInput, req.References)
|
||||
inputs := make(map[string]promptkit.ArtifactRef, len(materials))
|
||||
for name, material := range materials {
|
||||
inputs[name] = promptkit.InlineWithURI(material.OriginURI, string(material.Content))
|
||||
}
|
||||
candidate, err := shared.ModelCandidateFromResponse(completion)
|
||||
initial := promptkit.RunRequest{PromptID: PromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID, Inputs: inputs}
|
||||
value, outcome, err := pipeline.RunProducer(ctx, e.engine, req.Operation, initial,
|
||||
func(_ context.Context, result *promptkit.RunResult) (*dnd.NPCRegistry, []string, error) {
|
||||
var response extractionResponse
|
||||
if err := json.Unmarshal([]byte(result.RawOutput), &response); err != nil {
|
||||
return nil, nil, extractorErrorf("decode native output: %w", err)
|
||||
}
|
||||
canonicalizeResponse(&response, order, sourceID)
|
||||
value := canonicalNPCRegistry(response, sourceID)
|
||||
return &value, nil, nil
|
||||
},
|
||||
func(_ context.Context, guidance []string) (promptkit.RunRequest, error) {
|
||||
return promptkit.RunRequest{PromptID: CorrectionPromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID,
|
||||
Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline(strings.Join(guidance, "\n\n"))}}, nil
|
||||
},
|
||||
func(context.Context, *dnd.NPCRegistry) (dnd.NPCRegistry, error) {
|
||||
return dnd.NPCRegistry{NPCs: []dnd.NPC{}}, nil
|
||||
})
|
||||
if err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.NPCRegistry]{}, extractorErrorf("capture model candidate: %w", err)
|
||||
return contracts.TypedExtractionResult[dnd.NPCRegistry]{}, extractorErrorf("produce registry: %w", err)
|
||||
}
|
||||
canonicalizeResponse(&response, order, req.Source.ID)
|
||||
return contracts.TypedExtractionResult[dnd.NPCRegistry]{Value: canonicalNPCRegistry(response, req.Source.ID), ModelCandidate: candidate}, nil
|
||||
return contracts.TypedExtractionResult[dnd.NPCRegistry]{Value: value, Outcome: outcome}, nil
|
||||
}
|
||||
|
||||
func ModuleSpec() pipeline.ModuleSpec {
|
||||
return pipeline.ModuleSpec{
|
||||
Key: Key,
|
||||
Stage: pipeline.StageExtract,
|
||||
ExecutionClass: contracts.ExecutionClassLLMBacked,
|
||||
CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1,
|
||||
Requires: append([]string(nil), requiredCapabilities...),
|
||||
Provides: append([]string(nil), providedCapabilities...),
|
||||
ArtifactKind: dnd.NPCRegistryKind,
|
||||
ReferenceSlots: referenceSlots(),
|
||||
Key: Key,
|
||||
Stage: pipeline.StageExtract,
|
||||
ExecutionClass: contracts.ExecutionClassLLMBacked,
|
||||
Prompts: []string{PromptID, CorrectionPromptID},
|
||||
Requires: append([]string(nil), requiredCapabilities...),
|
||||
Provides: append([]string(nil), providedCapabilities...),
|
||||
ArtifactKind: dnd.NPCRegistryKind,
|
||||
ReferenceSlots: referenceSlots(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -157,7 +121,7 @@ func Register(registry *pipeline.ExtractorRegistry) error {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return New(request.Dependencies.LLM, options, request.References)
|
||||
return New(request.Dependencies.Engine, options, request.References)
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/promptkit"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
@@ -28,7 +30,10 @@ func TestExtractReturnsCanonicalNPCRegistryFromPrivateResponse(t *testing.T) {
|
||||
},
|
||||
}}}
|
||||
|
||||
result, err := newExtractor(t, client).Extract(context.Background(), extractionRequest())
|
||||
callRequest := extractionRequest()
|
||||
var prepared promptkit.PreparedRun
|
||||
callRequest.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = value }
|
||||
result, err := newExtractor(t, client).Extract(context.Background(), callRequest)
|
||||
if err != nil {
|
||||
t.Fatalf("Extract() error = %v, want nil", err)
|
||||
}
|
||||
@@ -43,13 +48,11 @@ func TestExtractReturnsCanonicalNPCRegistryFromPrivateResponse(t *testing.T) {
|
||||
t.Fatalf("LLM calls = %d, want 1", len(client.requests))
|
||||
}
|
||||
request := client.requests[0]
|
||||
if request.StageName != Key || request.PromptID != PromptID || request.PromptVersion != SchemaVersion || request.ProfileID != "profile-npcs" || request.SessionID != "session-123" {
|
||||
t.Fatalf("LLM request identity = %#v", request)
|
||||
}
|
||||
transcript := request.Inputs["transcript"]
|
||||
if transcript.Name != "transcript" || transcript.MediaType != "application/json" || transcript.Digest != "sha256:chunk" || transcript.OriginURI != "file:///session-alpha.json" || string(transcript.Content) != string(extractionRequest().Chunk.Content) {
|
||||
t.Fatalf("transcript input = %#v, want chunk-scoped material", transcript)
|
||||
if prepared.PromptID != PromptID || prepared.SelectedProfileID != "profile-npcs" || request.Prompt.SessionID != "session-123" {
|
||||
t.Fatal("native registry routing lost")
|
||||
}
|
||||
messageContaining(t, request, string(callRequest.Chunk.Content))
|
||||
|
||||
}
|
||||
|
||||
func TestExtractOrdersNPCsBySourcePositionRatherThanUnitID(t *testing.T) {
|
||||
@@ -132,13 +135,15 @@ func TestExtractPassesCampaignReferencesAsPromptInputs(t *testing.T) {
|
||||
if _, err := newExtractor(t, client).Extract(context.Background(), req); err != nil {
|
||||
t.Fatalf("Extract() error = %v, want nil", err)
|
||||
}
|
||||
inputs := client.requests[0].Inputs
|
||||
if string(inputs["players"].Content) != "Dana: Mira" || string(inputs["party"].Content) != "Mira: ranger" || string(inputs["glossary"].Content) != "Greencloak: local title" {
|
||||
t.Fatalf("reference inputs = %#v", inputs)
|
||||
request := client.requests[0]
|
||||
references := messageContaining(t, request, "Dana: Mira").Content
|
||||
if !strings.Contains(references, "Mira: ranger") || !strings.Contains(references, "Greencloak: local title") {
|
||||
t.Fatal("campaign references missing")
|
||||
}
|
||||
if strings.Contains(string(inputs["transcript"].Content), "local title") {
|
||||
t.Fatal("transcript input contains reference content")
|
||||
if strings.Contains(messageContaining(t, request, string(req.Chunk.Content)).Content, "local title") {
|
||||
t.Fatal("references became source evidence")
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestExtractPreservesMalformedCandidatesForValidators(t *testing.T) {
|
||||
@@ -173,38 +178,6 @@ func TestExtractMapsRawSemanticCandidatesWithoutRepair(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractForwardsCorrectionAndOwnsModelCandidate(t *testing.T) {
|
||||
rawResponse := []byte(`{"npcs":[{"name":"Mira Thorn","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`)
|
||||
client := &fakeNPCsLLMClient{content: append([]byte(nil), rawResponse...)}
|
||||
correction, err := contracts.NewSemanticCorrection([]byte(`{"npcs":[]}`), "Retain the transcript-grounded NPC.")
|
||||
if err != nil {
|
||||
t.Fatalf("NewSemanticCorrection() error = %v", err)
|
||||
}
|
||||
request := extractionRequest()
|
||||
request.Correction = correction
|
||||
result, err := newExtractor(t, client).Extract(context.Background(), request)
|
||||
if err != nil {
|
||||
t.Fatalf("Extract() error = %v, want nil", err)
|
||||
}
|
||||
if len(client.requests) != 1 || client.requests[0].Correction == nil ||
|
||||
string(client.requests[0].Correction.AssistantResponse) != `{"npcs":[]}` ||
|
||||
client.requests[0].Correction.UserGuidance != "Retain the transcript-grounded NPC." {
|
||||
t.Fatalf("structured request correction = %#v, want forwarded correction", client.requests)
|
||||
}
|
||||
correction.AssistantResponse[0] = '['
|
||||
if got := string(client.requests[0].Correction.AssistantResponse); got != `{"npcs":[]}` {
|
||||
t.Fatalf("captured correction changed after caller mutation: %q", got)
|
||||
}
|
||||
if result.ModelCandidate == nil || result.ModelCandidate.Protocol != contracts.CorrectionProtocolSingleResponseV1 ||
|
||||
string(result.ModelCandidate.Response) != string(rawResponse) {
|
||||
t.Fatalf("model candidate = %#v, want exact validated response", result.ModelCandidate)
|
||||
}
|
||||
client.content[0] = '['
|
||||
if got := string(result.ModelCandidate.Response); got != string(rawResponse) {
|
||||
t.Fatalf("model candidate changed after provider buffer mutation: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractRetainsLocalErrorContextAndProviderFailures(t *testing.T) {
|
||||
request := extractionRequest()
|
||||
extractor := newExtractor(t, &fakeNPCsLLMClient{response: extractionResponse{NPCs: []npcResponse{}}})
|
||||
@@ -212,11 +185,11 @@ func TestExtractRetainsLocalErrorContextAndProviderFailures(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
extractor *Extractor
|
||||
req contracts.TypedExtractionRequest
|
||||
req contracts.TypedExtractionRequest[dnd.NPCRegistry]
|
||||
want string
|
||||
}{
|
||||
{name: "nil extractor", extractor: nilExtractor, req: request, want: "extractor"},
|
||||
{name: "nil LLM client", extractor: &Extractor{}, req: request, want: "LLM client"},
|
||||
{name: "nil native engine", extractor: &Extractor{}, req: request, want: "native engine"},
|
||||
{name: "wrapped preflight failure", extractor: extractor, req: mismatchedSourceInputRequest(request), want: "must match chunk"},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
package npcregistry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
registrycodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcregistry"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity"
|
||||
shape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/npcregistry/shape"
|
||||
refs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/npcregistry/source_refs"
|
||||
relatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/npcregistry/source_relatedness"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
const ungroundedRegistry = `{"npcs":[{"name":"Mira Thorn","source_refs":[{"start_unit_id":999,"end_unit_id":999}]}]}`
|
||||
const groundedRegistry = `{"npcs":[{"name":"Mira Thorn","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`
|
||||
const emptyRegistry = `{"npcs":[]}`
|
||||
|
||||
func assessRegistry(ctx context.Context, request contracts.TypedExtractionRequest[dnd.NPCRegistry], value dnd.NPCRegistry) (contracts.CandidateAssessment, error) {
|
||||
assessment := contracts.CandidateAssessment{}
|
||||
for _, validator := range []contracts.TypedValidator[dnd.NPCRegistry]{
|
||||
shape.New(shape.Options{}), refs.New(refs.Options{}), relatedness.New(relatedness.Options{}),
|
||||
} {
|
||||
result, err := validator.Validate(ctx, contracts.TypedValidationRequest[dnd.NPCRegistry]{Stage: "extract", Source: request.Source, Chunk: request.Chunk, Value: value})
|
||||
if err != nil {
|
||||
return contracts.CandidateAssessment{}, err
|
||||
}
|
||||
decision := contracts.AssessmentRejected
|
||||
if result.Approved {
|
||||
decision = contracts.AssessmentApproved
|
||||
}
|
||||
assessment.Validators = append(assessment.Validators, contracts.ValidatorAssessment{ValidatorName: validator.Name(), Decision: decision, Result: result})
|
||||
}
|
||||
return assessment, nil
|
||||
}
|
||||
|
||||
func TestNativeExtractionCorrectsGroundingWithFixedInputs(t *testing.T) {
|
||||
client := &fakeNPCsLLMClient{responses: []string{ungroundedRegistry, groundedRegistry}}
|
||||
request := extractionRequest()
|
||||
request.References = contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{"glossary": {Items: []contracts.ReferenceItem{{Content: []byte("fixed-reference")}}}}}
|
||||
request.Operation.Controls.MaxAdditionalTurns = 1
|
||||
request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "registry-override"}
|
||||
var prepared []promptkit.PreparedRun
|
||||
request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = append(prepared, value) }
|
||||
assessments := 0
|
||||
request.Operation.Assess = func(ctx context.Context, value dnd.NPCRegistry) (contracts.CandidateAssessment, error) {
|
||||
assessments++
|
||||
assessment, err := assessRegistry(ctx, request, value)
|
||||
if assessments == 1 {
|
||||
if assessment.Accepted() {
|
||||
t.Fatal("unresolvable citation was accepted")
|
||||
}
|
||||
// Changes after capture must not cause source/reference preparation again.
|
||||
request.SourceInput.Content[0] = '!'
|
||||
request.References.Slots["glossary"].Items[0].Content[0] = '!'
|
||||
}
|
||||
return assessment, err
|
||||
}
|
||||
result, err := newExtractor(t, client).Extract(context.Background(), request)
|
||||
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || !result.Outcome.ReuseEligible ||
|
||||
assessments != 2 || len(client.requests) != 2 || len(prepared) != 2 || len(result.Value.NPCs) != 1 {
|
||||
t.Fatalf("corrected registry=%#v %v, assessments/provider/prepared=%d/%d/%d", result, err, assessments, len(client.requests), len(prepared))
|
||||
}
|
||||
record := result.Value.NPCs[0]
|
||||
if record.Name != "Mira Thorn" || len(record.SourceRefs) != 1 || record.SourceRefs[0].SourceID != request.Source.ID ||
|
||||
record.SourceRefs[0].StartUnitID != 1 || record.SourceRefs[0].EndUnitID != 1 || record.ID != identity.DeriveID(record.Name) {
|
||||
t.Fatalf("canonical record identity/evidence=%#v", record)
|
||||
}
|
||||
if _, err := registrycodec.New().Encode(result.Value); err != nil {
|
||||
t.Fatalf("durable registry encoding: %v", err)
|
||||
}
|
||||
for i, call := range client.requests {
|
||||
id := PromptID
|
||||
if i > 0 {
|
||||
id = CorrectionPromptID
|
||||
}
|
||||
if prepared[i].PromptID != id || prepared[i].SelectedProfileID != request.LLMProfile || call.Target.Model != "registry-override" || call.Prompt.SessionID != request.SessionID {
|
||||
t.Fatal("native selection/overrides lost")
|
||||
}
|
||||
}
|
||||
initial, revised := client.requests[0].Prompt.Messages, client.requests[1].Prompt.Messages
|
||||
if len(revised) != len(initial)+2 || !reflect.DeepEqual(revised[:len(initial)], initial) ||
|
||||
revised[len(initial)].Role != "assistant" || revised[len(initial)].Content != ungroundedRegistry {
|
||||
t.Fatal("native correction lost fixed inputs or prior response")
|
||||
}
|
||||
messageContaining(t, client.requests[1], "fixed-reference")
|
||||
messageContaining(t, client.requests[1], string(request.Chunk.Content))
|
||||
}
|
||||
|
||||
func TestNativeExtractionDistinguishesEmptyAcceptanceFromFallback(t *testing.T) {
|
||||
failure := errors.New("provider unavailable")
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
mode contracts.ExecutionMode
|
||||
responses []string
|
||||
providerErrors []error
|
||||
wantAssessments, wantRuns int
|
||||
wantError, wantFallback bool
|
||||
}{
|
||||
{"ordinary empty", contracts.ModeStrict, []string{emptyRegistry}, nil, 1, 1, false, false},
|
||||
{"strict rejection", contracts.ModeStrict, []string{ungroundedRegistry, ungroundedRegistry}, nil, 2, 2, true, false},
|
||||
{"permissive rejection", contracts.ModePermissive, []string{ungroundedRegistry, ungroundedRegistry}, nil, 2, 2, false, true},
|
||||
{"provider failure", contracts.ModePermissive, nil, []error{failure}, 0, 1, false, true},
|
||||
{"failed correction", contracts.ModePermissive, []string{ungroundedRegistry}, []error{nil, failure}, 1, 2, false, true},
|
||||
{"structural failure", contracts.ModePermissive, []string{`{"npcs":null}`}, nil, 0, 1, false, true},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
client := &fakeNPCsLLMClient{responses: test.responses, errors: test.providerErrors}
|
||||
if test.name == "structural failure" {
|
||||
client.content = []byte(test.responses[0])
|
||||
}
|
||||
request := extractionRequest()
|
||||
request.Operation.Controls.Mode = test.mode
|
||||
request.Operation.Controls.MaxAdditionalTurns = 1
|
||||
assessments, prepared := 0, 0
|
||||
request.Operation.Execution.OnPrepared = func(context.Context, promptkit.PreparedRun) { prepared++ }
|
||||
request.Operation.Assess = func(ctx context.Context, value dnd.NPCRegistry) (contracts.CandidateAssessment, error) {
|
||||
assessments++
|
||||
return assessRegistry(ctx, request, value)
|
||||
}
|
||||
result, err := newExtractor(t, client).Extract(context.Background(), request)
|
||||
if (err != nil) != test.wantError || assessments != test.wantAssessments || prepared != test.wantRuns {
|
||||
t.Fatalf("registry=%#v %v, assessments=%d runs=%d", result, err, assessments, prepared)
|
||||
}
|
||||
if test.wantError {
|
||||
if result.Outcome.Disposition != "" {
|
||||
t.Fatal("strict rejection produced an outcome")
|
||||
}
|
||||
return
|
||||
}
|
||||
if result.Value.NPCs == nil || len(result.Value.NPCs) != 0 || (result.Outcome.Disposition == contracts.ProducerFallback) != test.wantFallback ||
|
||||
result.Outcome.ReuseEligible == test.wantFallback {
|
||||
t.Fatalf("unqualified empty registry=%#v", result)
|
||||
}
|
||||
if _, err := registrycodec.New().Encode(result.Value); err != nil {
|
||||
t.Fatalf("empty durable registry invalid: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCanceledNativeExtractionCannotDeliverFallback(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
client := &fakeNPCsLLMClient{content: []byte(groundedRegistry), onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }}
|
||||
request := extractionRequest()
|
||||
request.Operation.Controls.Mode = contracts.ModePermissive
|
||||
result, err := newExtractor(t, client).Extract(ctx, request)
|
||||
if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" || len(client.requests) != 1 {
|
||||
t.Fatalf("cancellation=%#v %v", result, err)
|
||||
}
|
||||
}
|
||||
@@ -1,74 +0,0 @@
|
||||
package npcregistry
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"sync"
|
||||
|
||||
rootassets "gitea.maximumdirect.net/eric/notarius/assets"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||
)
|
||||
|
||||
const promptAssetRoot = "assets/prompts"
|
||||
|
||||
var promptAssetManifest = shared.PromptAssetManifest{
|
||||
ModuleDir: "dnd.npc_registry",
|
||||
ModuleFiles: []promptfs.ModulePromptFile{
|
||||
{Name: "prompt.yaml", Path: "prompts/prompt.yaml"},
|
||||
{Name: "instructions.md", Path: "prompts/instructions.md"},
|
||||
},
|
||||
SharedFiles: []string{
|
||||
"common-dnd-system.md",
|
||||
"common-dnd-extraction-evidence.md",
|
||||
"common-dnd-identity.md",
|
||||
"common-dnd-transcript-chunk.md",
|
||||
"common-dnd-references.md",
|
||||
},
|
||||
}
|
||||
|
||||
func moduleAssetFS() (fs.FS, error) {
|
||||
assets, err := fs.Sub(rootassets.FS(), "dnd/npc-registry/extract")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scope NPC extraction assets: %w", err)
|
||||
}
|
||||
return assets, nil
|
||||
}
|
||||
|
||||
func RegisterPromptAssets(registry *llm.AssetRegistry) error {
|
||||
assets, err := moduleAssetFS()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
promptFS, err := promptAssetManifest.PromptFS(assets)
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare NPC prompt assets: %w", err)
|
||||
}
|
||||
if err := registry.RegisterPromptFS(promptFS, promptAssetRoot); err != nil {
|
||||
return err
|
||||
}
|
||||
schemas, err := fs.Sub(assets, "schemas")
|
||||
if err != nil {
|
||||
return fmt.Errorf("scope NPC extraction schemas: %w", err)
|
||||
}
|
||||
return registry.RegisterSchemaFS(schemas, ".")
|
||||
}
|
||||
|
||||
func promptAssetMetadata() (string, error) {
|
||||
promptAssetHashOnce.Do(func() {
|
||||
assets, err := moduleAssetFS()
|
||||
if err != nil {
|
||||
promptAssetHashErr = err
|
||||
return
|
||||
}
|
||||
promptAssetHash, promptAssetHashErr = promptAssetManifest.Hash(assets)
|
||||
})
|
||||
return promptAssetHash, promptAssetHashErr
|
||||
}
|
||||
|
||||
var (
|
||||
promptAssetHashOnce sync.Once
|
||||
promptAssetHash string
|
||||
promptAssetHashErr error
|
||||
)
|
||||
@@ -1,55 +0,0 @@
|
||||
package npcregistry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestRegisterPromptAssetsAndPrepareNPCPrompt(t *testing.T) {
|
||||
registry := llm.NewAssetRegistry()
|
||||
if err := RegisterPromptAssets(registry); err != nil {
|
||||
t.Fatalf("RegisterPromptAssets() error = %v, want nil", err)
|
||||
}
|
||||
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
|
||||
"npc-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "npc-test-model"),
|
||||
})
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "npc-test-profile",
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.InlineWithURI("file:///session.json", `{"units":[{"sentinel":"npc-transcript"}]}`),
|
||||
"players": promptkit.Inline("npc-player"),
|
||||
"party": promptkit.Inline(" "),
|
||||
"glossary": promptkit.Inline(" "),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Prepare() error = %v, want nil", err)
|
||||
}
|
||||
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_npc_registry_llm.v1.json" {
|
||||
t.Fatalf("prepared prompt = %#v, want NPC prompt identity and wiring", prepared)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptMetadataAndDiagnosticsDoNotContainRawAssets(t *testing.T) {
|
||||
hash, err := promptAssetMetadata()
|
||||
if err != nil || !strings.HasPrefix(hash, "sha256:") {
|
||||
t.Fatalf("promptAssetMetadata() = %q, %v; want hash", hash, err)
|
||||
}
|
||||
metadata := newExtractor(t, &fakeNPCsLLMClient{}).ManifestMetadata()
|
||||
payload, err := json.Marshal(metadata)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, forbidden := range []string{"common-dnd-system", "dnd_npc_registry_llm.v1.json"} {
|
||||
if strings.Contains(string(payload), forbidden) {
|
||||
t.Fatalf("metadata leaked raw prompt/schema content %q: %s", forbidden, payload)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -10,11 +10,11 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
)
|
||||
|
||||
func TestNewRequiresLLMClientAndRejectsAmbiguousReferences(t *testing.T) {
|
||||
if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "LLM client") {
|
||||
func TestNewRequiresNativeEngineAndRejectsAmbiguousReferences(t *testing.T) {
|
||||
if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "native engine") {
|
||||
t.Fatalf("New(nil) error = %v, want dependency error", err)
|
||||
}
|
||||
if _, err := New(&fakeNPCsLLMClient{}, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one reference set") {
|
||||
if _, err := New(newExtractor(t, &fakeNPCsLLMClient{}).engine, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one reference set") {
|
||||
t.Fatalf("New() error = %v, want reference-set error", err)
|
||||
}
|
||||
if got := newExtractor(t, &fakeNPCsLLMClient{}).Key(); got != Key {
|
||||
@@ -25,13 +25,13 @@ func TestNewRequiresLLMClientAndRejectsAmbiguousReferences(t *testing.T) {
|
||||
func TestModuleSpecAndReferenceSlots(t *testing.T) {
|
||||
got := ModuleSpec()
|
||||
want := pipeline.ModuleSpec{
|
||||
Key: Key,
|
||||
Stage: pipeline.StageExtract,
|
||||
ExecutionClass: contracts.ExecutionClassLLMBacked,
|
||||
CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1,
|
||||
Requires: []string{"chunks", "source.transcript"},
|
||||
Provides: []string{"dnd.npc_registry"},
|
||||
ArtifactKind: dnd.NPCRegistryKind,
|
||||
Key: Key,
|
||||
Stage: pipeline.StageExtract,
|
||||
ExecutionClass: contracts.ExecutionClassLLMBacked,
|
||||
Prompts: []string{PromptID, CorrectionPromptID},
|
||||
Requires: []string{"chunks", "source.transcript"},
|
||||
Provides: []string{"dnd.npc_registry"},
|
||||
ArtifactKind: dnd.NPCRegistryKind,
|
||||
ReferenceSlots: []contracts.ReferenceSlot{
|
||||
{Name: "glossary", Description: "Optional campaign glossary reference material used only for NPC disambiguation.", AcceptedMediaTypes: []string{"application/json", "application/x-yaml", "application/yaml", "text/markdown", "text/plain"}},
|
||||
{Name: "party", Description: "Optional party roster reference material used only for NPC disambiguation.", AcceptedMediaTypes: []string{"application/json", "application/x-yaml", "application/yaml", "text/markdown", "text/plain"}},
|
||||
@@ -42,6 +42,7 @@ func TestModuleSpecAndReferenceSlots(t *testing.T) {
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
|
||||
}
|
||||
got.Prompts[0] = "changed"
|
||||
got.Requires[0] = "changed"
|
||||
got.Provides[0] = "changed"
|
||||
got.ReferenceSlots[0].AcceptedMediaTypes[0] = "changed"
|
||||
@@ -70,34 +71,3 @@ func TestRegisterStoresTypedModuleSpec(t *testing.T) {
|
||||
t.Fatalf("DecodeOptions() error = %v, want strict options error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractorMetadataAndCheckpointIdentity(t *testing.T) {
|
||||
extractor := newExtractor(t, &fakeNPCsLLMClient{})
|
||||
metadata := extractor.ManifestMetadata()
|
||||
for key, want := range map[string]string{
|
||||
"prompt_id": PromptID, "prompt_version": SchemaVersion,
|
||||
"response_schema_key": string(ResponseSchemaKey), "response_schema_id": ResponseSchemaID,
|
||||
"response_schema_name": ResponseSchemaName, "response_schema_version": SchemaVersion,
|
||||
"identity_policy": "dnd.npc_registry.identity.v1",
|
||||
"mapping_policy": mappingPolicy,
|
||||
} {
|
||||
if metadata[key] != want {
|
||||
t.Fatalf("metadata[%q] = %#v, want %q", key, metadata[key], want)
|
||||
}
|
||||
}
|
||||
for _, key := range []string{"prompt_sha256", "response_schema_sha256"} {
|
||||
if value, ok := metadata[key].(string); !ok || !strings.HasPrefix(value, "sha256:") {
|
||||
t.Fatalf("metadata[%q] = %#v, want hash", key, metadata[key])
|
||||
}
|
||||
}
|
||||
fingerprints := extractor.CheckpointFingerprints()
|
||||
want := map[string]string{"prompt": metadata["prompt_sha256"].(string), "response_schema": metadata["response_schema_sha256"].(string), "identity_policy": "dnd.npc_registry.identity.v1", "mapping_policy": mappingPolicy}
|
||||
if len(fingerprints) != len(want) {
|
||||
t.Fatalf("CheckpointFingerprints() = %#v, want %d entries", fingerprints, len(want))
|
||||
}
|
||||
for _, fingerprint := range fingerprints {
|
||||
if fingerprint.Value != want[fingerprint.Name] {
|
||||
t.Fatalf("fingerprint %q = %q, want %#v", fingerprint.Name, fingerprint.Value, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
package npcregistry
|
||||
|
||||
import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
|
||||
const (
|
||||
PromptID = "dnd.npc_registry"
|
||||
ResponseSchemaKey = llm.ResponseSchemaKey("dnd_npc_registry_llm")
|
||||
ResponseSchemaID = "notarius.dnd.npc_registry.llm"
|
||||
ResponseSchemaName = "notarius_dnd_npc_registry_llm_v1"
|
||||
SchemaVersion = "v1"
|
||||
)
|
||||
|
||||
func loadResponseSchema() (llm.ResponseSchema, error) {
|
||||
assets, err := moduleAssetFS()
|
||||
if err != nil {
|
||||
return llm.ResponseSchema{}, err
|
||||
}
|
||||
return llm.LoadResponseSchema(assets, llm.ResponseSchemaDefinition{
|
||||
Key: ResponseSchemaKey,
|
||||
ID: ResponseSchemaID,
|
||||
Version: SchemaVersion,
|
||||
Name: ResponseSchemaName,
|
||||
AssetPath: "schemas/dnd_npc_registry_llm.v1.json",
|
||||
})
|
||||
}
|
||||
@@ -1,138 +0,0 @@
|
||||
package npcregistry
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/santhosh-tekuri/jsonschema/v6"
|
||||
)
|
||||
|
||||
func TestLoadResponseSchemaUsesPrivateNPCSchema(t *testing.T) {
|
||||
schema, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
|
||||
}
|
||||
if schema.Key != ResponseSchemaKey || schema.ID != ResponseSchemaID || schema.Version != SchemaVersion || schema.Name != ResponseSchemaName || !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) {
|
||||
t.Fatalf("schema = %#v, want private NPC schema identity", schema)
|
||||
}
|
||||
valid := map[string]any{"npcs": []any{map[string]any{
|
||||
"name": "Mira Thorn",
|
||||
"source_refs": []any{map[string]any{"start_unit_id": 1, "end_unit_id": 2}},
|
||||
}}}
|
||||
validJSON, err := json.Marshal(valid)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := validateJSONSchema(validJSON, schema.JSONSchema); err != nil {
|
||||
t.Fatalf("valid private NPC response rejected: %v", err)
|
||||
}
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
response map[string]any
|
||||
valid bool
|
||||
}{
|
||||
{
|
||||
name: "semantic blanks and empty collections",
|
||||
response: map[string]any{"npcs": []any{map[string]any{
|
||||
"name": "", "source_refs": []any{},
|
||||
}}},
|
||||
valid: true,
|
||||
},
|
||||
{
|
||||
name: "nonpositive unit candidates",
|
||||
response: map[string]any{"npcs": []any{map[string]any{
|
||||
"name": "Mira Thorn",
|
||||
"source_refs": []any{map[string]any{"start_unit_id": 0, "end_unit_id": -1}},
|
||||
}}},
|
||||
valid: true,
|
||||
},
|
||||
{
|
||||
name: "missing required field",
|
||||
response: map[string]any{"npcs": []any{map[string]any{
|
||||
"source_refs": []any{},
|
||||
}}},
|
||||
},
|
||||
{
|
||||
name: "unknown field",
|
||||
response: map[string]any{"npcs": []any{map[string]any{
|
||||
"name": "Mira Thorn", "source_refs": []any{}, "id": "assigned later",
|
||||
}}},
|
||||
},
|
||||
{
|
||||
name: "wrong field type",
|
||||
response: map[string]any{"npcs": []any{map[string]any{
|
||||
"name": 7, "source_refs": []any{},
|
||||
}}},
|
||||
},
|
||||
{
|
||||
name: "noninteger source identifier",
|
||||
response: map[string]any{"npcs": []any{map[string]any{
|
||||
"name": "Mira Thorn",
|
||||
"source_refs": []any{map[string]any{"start_unit_id": 1.5, "end_unit_id": 2}},
|
||||
}}},
|
||||
},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
content, err := json.Marshal(test.response)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = validateJSONSchema(content, schema.JSONSchema)
|
||||
if (err == nil) != test.valid {
|
||||
t.Fatalf("validateJSONSchema() error = %v, want valid=%t", err, test.valid)
|
||||
}
|
||||
})
|
||||
}
|
||||
if err := validateJSONSchema([]byte(`{"npcs":`), schema.JSONSchema); err == nil {
|
||||
t.Fatal("validateJSONSchema() error = nil, want malformed JSON rejected")
|
||||
}
|
||||
withID := map[string]any{"npcs": []any{map[string]any{
|
||||
"name": "Mira Thorn", "id": "assigned-later",
|
||||
"source_refs": []any{map[string]any{"start_unit_id": 1, "end_unit_id": 2}},
|
||||
}}}
|
||||
withIDJSON, err := json.Marshal(withID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := validateJSONSchema(withIDJSON, schema.JSONSchema); err == nil {
|
||||
t.Fatal("private schema accepted framework-assigned id")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResponseSchemaIsMutationSafeAndDiagnosticsRedactContent(t *testing.T) {
|
||||
first, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first.JSONSchema[0] = '['
|
||||
second, err := loadResponseSchema()
|
||||
if err != nil || !json.Valid(second.JSONSchema) || bytes.Equal(first.JSONSchema, second.JSONSchema) {
|
||||
t.Fatalf("second schema = %s, %v; want defensive copy", second.JSONSchema, err)
|
||||
}
|
||||
diagnostics := second.DiagnosticsMap()
|
||||
if _, ok := diagnostics["json_schema"]; ok {
|
||||
t.Fatalf("schema diagnostics included raw content: %#v", diagnostics)
|
||||
}
|
||||
}
|
||||
|
||||
func validateJSONSchema(instanceContent, schemaContent []byte) error {
|
||||
instance, err := jsonschema.UnmarshalJSON(bytes.NewReader(instanceContent))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
schemaDocument, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaContent))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
compiler := jsonschema.NewCompiler()
|
||||
if err := compiler.AddResource("schema.json", schemaDocument); err != nil {
|
||||
return err
|
||||
}
|
||||
compiled, err := compiler.Compile("schema.json")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return compiled.Validate(instance)
|
||||
}
|
||||
@@ -3,14 +3,18 @@ package npcregistry
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/assets"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func extractionRequest() contracts.TypedExtractionRequest {
|
||||
func extractionRequest() contracts.TypedExtractionRequest[dnd.NPCRegistry] {
|
||||
doc := sourceDocument()
|
||||
chunk := &source.Chunk{
|
||||
ID: "session-alpha:chunk:0",
|
||||
@@ -22,12 +26,13 @@ func extractionRequest() contracts.TypedExtractionRequest {
|
||||
Units: append([]source.SourceUnit(nil), doc.Units...),
|
||||
Metadata: map[string]any{"ignored": "chunk metadata"},
|
||||
}
|
||||
return contracts.TypedExtractionRequest{
|
||||
return contracts.TypedExtractionRequest[dnd.NPCRegistry]{
|
||||
Source: doc,
|
||||
Chunk: chunk,
|
||||
SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session-alpha.json"),
|
||||
SessionID: "session-123",
|
||||
LLMProfile: "profile-npcs",
|
||||
Operation: registryOperation(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -49,67 +54,97 @@ func responseSourceRefs(startUnitID, endUnitID int) []npcSourceRefResponse {
|
||||
return []npcSourceRefResponse{{StartUnitID: startUnitID, EndUnitID: endUnitID}}
|
||||
}
|
||||
|
||||
func newExtractor(t *testing.T, client contracts.StructuredLLMClient, references ...contracts.ReferenceSet) *Extractor {
|
||||
func registryOperation() contracts.ProducerOperation[dnd.NPCRegistry] {
|
||||
return contracts.ProducerOperation[dnd.NPCRegistry]{
|
||||
Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, Cache: contracts.CacheBypass}, ReuseEligible: true,
|
||||
Assess: func(context.Context, dnd.NPCRegistry) (contracts.CandidateAssessment, error) {
|
||||
return contracts.CandidateAssessment{}, nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func newExtractor(t *testing.T, client promptkit.LLMClient, references ...contracts.ReferenceSet) *Extractor {
|
||||
t.Helper()
|
||||
extractor, err := New(client, Options{}, references...)
|
||||
type settings struct {
|
||||
PromptKit promptkit.Settings `yaml:"promptkit"`
|
||||
}
|
||||
definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit })
|
||||
filesystem := configkit.NewFilesystem("Notarius assets", assets.FS())
|
||||
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("registry fixture", []byte(`promptkit:
|
||||
profiles:
|
||||
- id: profile-npcs
|
||||
endpoint: https://example.invalid/v1
|
||||
model: registry-test-model
|
||||
api_key_required: false
|
||||
`))}, configkit.LayeredOptions[settings]{Definition: definition, Underlays: []configkit.Underlay{
|
||||
promptkit.BundledOpenRouterUnderlay(),
|
||||
{Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v, want nil", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
extractor, err := New(engine, Options{}, references...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return extractor
|
||||
}
|
||||
|
||||
func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest) contracts.TypedExtractionRequest {
|
||||
func messageContaining(t *testing.T, request promptkit.GenerateRequest, text string) promptkit.RenderedMessage {
|
||||
t.Helper()
|
||||
for _, message := range request.Prompt.Messages {
|
||||
if strings.Contains(message.Content, text) {
|
||||
return message
|
||||
}
|
||||
}
|
||||
t.Fatalf("missing rendered input %q", text)
|
||||
return promptkit.RenderedMessage{}
|
||||
}
|
||||
|
||||
func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest[dnd.NPCRegistry]) contracts.TypedExtractionRequest[dnd.NPCRegistry] {
|
||||
req.SourceInput = contracts.NewLLMInputMaterial("source", "application/json", []byte(`{"different":true}`), "sha256:other", "file:///other.json")
|
||||
return req
|
||||
}
|
||||
|
||||
func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest {
|
||||
req.Inputs = req.Inputs.Clone()
|
||||
correction, err := contracts.CloneSemanticCorrection(req.Correction)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
req.Correction = correction
|
||||
if len(req.Vars) == 0 {
|
||||
req.Vars = nil
|
||||
return req
|
||||
}
|
||||
vars := make(map[string]any, len(req.Vars))
|
||||
for key, value := range req.Vars {
|
||||
vars[key] = value
|
||||
}
|
||||
req.Vars = vars
|
||||
return req
|
||||
}
|
||||
|
||||
type fakeNPCsLLMClient struct {
|
||||
response extractionResponse
|
||||
content []byte
|
||||
err error
|
||||
requests []contracts.StructuredCompletionRequest
|
||||
response extractionResponse
|
||||
content []byte
|
||||
responses []string
|
||||
err error
|
||||
errors []error
|
||||
requests []promptkit.GenerateRequest
|
||||
onGenerate func(context.Context, promptkit.GenerateRequest)
|
||||
}
|
||||
|
||||
func (client *fakeNPCsLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
client.requests = append(client.requests, cloneStructuredCompletionRequest(req))
|
||||
if client.err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, client.err
|
||||
func (client *fakeNPCsLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
index := len(client.requests)
|
||||
client.requests = append(client.requests, request)
|
||||
if client.onGenerate != nil {
|
||||
client.onGenerate(ctx, request)
|
||||
}
|
||||
target, ok := out.(*extractionResponse)
|
||||
if !ok {
|
||||
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target")
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if client.err != nil {
|
||||
return nil, client.err
|
||||
}
|
||||
if index < len(client.errors) && client.errors[index] != nil {
|
||||
return nil, client.errors[index]
|
||||
}
|
||||
content := append([]byte(nil), client.content...)
|
||||
if len(content) != 0 {
|
||||
if err := json.Unmarshal(content, target); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
}
|
||||
} else {
|
||||
*target = client.response
|
||||
if index < len(client.responses) {
|
||||
content = []byte(client.responses[index])
|
||||
}
|
||||
if len(content) == 0 {
|
||||
var err error
|
||||
content, err = json.Marshal(client.response)
|
||||
if err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{Content: content}, nil
|
||||
return &promptkit.GenerateResponse{Content: string(content)}, nil
|
||||
}
|
||||
|
||||
@@ -41,7 +41,7 @@ func TestValidatorApprovesAndRegisters(t *testing.T) {
|
||||
if err := Register(registry); err != nil {
|
||||
t.Fatalf("Register() error = %v", err)
|
||||
}
|
||||
if got, ok := registry.Spec(Key); !ok || got != Spec() || got.ExecutionClass != contracts.ExecutionClassDeterministic {
|
||||
if got, ok := registry.Spec(Key); !ok || !reflect.DeepEqual(got, Spec()) || got.ExecutionClass != contracts.ExecutionClassDeterministic {
|
||||
t.Fatalf("registered spec = %#v, ok = %t", got, ok)
|
||||
}
|
||||
if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil {
|
||||
|
||||
Reference in New Issue
Block a user