Exercise spell and NPC integration through native providers and published artifacts

This commit is contained in:
2026-11-22 04:39:55 +00:00
parent 20c0975d9d
commit 0ad73ff575
9 changed files with 219 additions and 261 deletions

View File

@@ -1699,6 +1699,18 @@ Seriatim package tests and the unchanged production import-boundary suite.
The remaining legacy integration files still prevent running that whole package;
their completion/checkpoint fixture migration is not finished.
Migrated the older spell and NPC registry integration fixtures to native
provider requests, real JSON lane publication, and version 6 test configurations.
Kept source ordering/identity, capability failures, reference inputs, malformed
candidate rejection, and consolidation coverage. NPC reconciliation now tests
explicit strict failure, permissive final safe groups with one qualified fallback,
and correction to a complete proposal. Shared production registries remain real;
test providers classify the selected native response contract rather than
recreating completion requests or manually decoding model output into modules.
These selected integration tests, vet/race checks, all framework/configuration
tests, and import-boundary tests pass. Remaining combat, location, occurrence,
and grounded-pipeline fixtures still block full integration compilation.
**Goal:** delete the old architecture rather than leave dormant alternatives.
**Inspect/change:** all production/test callers of the feature roadmap's removal

View File

@@ -6,22 +6,19 @@ import (
"fmt"
"os"
"strings"
"sync"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile"
npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcregistry"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/npcregistry"
npcnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/npcregistry"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity"
dndregister "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/register"
npcshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/npcregistry/shape"
npcregistrysourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/npcregistry/source_refs"
genericregister "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/register"
"gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript"
seriatimregister "gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/register"
"go.maximumdirect.net/promptkit"
)
func TestRunnerProcessesSeriatimInputWithProductionDNDNPCPipeline(t *testing.T) {
@@ -57,17 +54,15 @@ func TestRunnerProcessesSeriatimInputWithProductionDNDNPCPipeline(t *testing.T)
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.NormalizeOutputs) != 1 {
t.Fatalf("normalize outputs = %d, want one NPC output; rejected=%#v", len(output.NormalizeOutputs), output.Rejected)
if len(output.OutputFiles) != 1 {
t.Fatalf("output files = %#v, want NPC registry", output.OutputFiles)
}
serialized := output.NormalizeOutputs[0]
if serialized.LaneID != "npc_registry" || serialized.NormalizerKey != npcregistry.Key || serialized.Artifact.Schema.ID != npccodec.SchemaID || serialized.Artifact.Schema.Version != npccodec.SchemaVersion {
t.Fatalf("serialized output = %#v, want durable NPC lane schema", serialized)
}
value, err := npccodec.New().Decode(serialized.Artifact.Content)
file := publishedLane(t, output, "npc_registry")
value, err := npccodec.New().Decode(file.Bytes)
if err != nil {
t.Fatalf("Decode(output) error = %v, want durable NPC payload", err)
t.Fatal(err)
}
if len(value.NPCs) != 2 {
t.Fatalf("NPC output = %#v, want repeated name consolidated and group/PC omitted", value.NPCs)
}
@@ -88,29 +83,12 @@ func TestRunnerProcessesSeriatimInputWithProductionDNDNPCPipeline(t *testing.T)
if !hasNPCWarning(output.Diagnostics, "duplicate_npc_collapsed") {
t.Fatalf("diagnostics = %#v, want name-only consolidation advisory", output.Diagnostics)
}
if output.Manifest.ValidationStatus != "approved" || len(output.Manifest.ArtifactLanes) != 1 {
t.Fatalf("manifest = %#v, want approved NPC lane", output.Manifest)
if file.ContentType != npccodec.MediaType {
t.Fatalf("NPC content type = %q, want %q", file.ContentType, npccodec.MediaType)
}
lane := output.Manifest.ArtifactLanes[0]
if lane.ID != "npc_registry" || lane.Extractor != npcregistry.Key || lane.Merger != pipeline.DefaultMergeModule || lane.Normalizer != npcregistry.Key {
t.Fatalf("manifest lane = %#v, want NPC production composition", lane)
}
normalizerMetadata, ok := lane.Metadata["normalizer"].(map[string]any)
if !ok || normalizerMetadata["identity_policy"] != identity.Policy || normalizerMetadata["normalization_policy"] != npcnormalize.NormalizationPolicy || normalizerMetadata["prompt_id"] != npcnormalize.PromptID || normalizerMetadata["response_schema_id"] != semanticreconcile.ResponseSchemaID {
t.Fatalf("normalizer metadata = %#v, want identity and normalization policies", lane.Metadata)
}
var npcOutputFile *contracts.OutputFile
for index := range output.OutputFiles {
if output.OutputFiles[index].Name == "lanes/npc_registry.json" {
npcOutputFile = &output.OutputFiles[index]
break
}
}
if npcOutputFile == nil || npcOutputFile.ContentType != npccodec.MediaType {
t.Fatalf("output files = %#v, want JSON NPC lane file", output.OutputFiles)
}
if len(client.requests) != 2 || client.requests[0].PromptID != npcregistry.PromptID || client.requests[1].PromptID != npcnormalize.PromptID {
t.Fatalf("LLM requests = %#v, want extraction and normalization prompt requests", client.requests)
if len(client.requests) != 2 || client.extractions != 1 || client.normalizations != 1 {
t.Fatalf("native requests=%d extraction=%d normalization=%d", len(client.requests), client.extractions, client.normalizations)
}
}
@@ -126,54 +104,45 @@ func TestProductionNPCPipelineRoutesSemanticCandidatesToDeterministicValidators(
for _, test := range []struct {
name string
response []byte
reasonCode string
validatorName string
}{
{
name: "blank string",
response: []byte(`{"npcs":[{"name":"","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`),
reasonCode: npcshape.ReasonCode,
validatorName: npcshape.Key,
},
{
name: "empty evidence",
response: []byte(`{"npcs":[{"name":"Mira Thorn","source_refs":[]}]}`),
reasonCode: npcshape.ReasonCode,
validatorName: npcshape.Key,
},
{
name: "nonpositive unit candidate",
response: []byte(`{"npcs":[{"name":"Mira Thorn","source_refs":[{"start_unit_id":0,"end_unit_id":1}]}]}`),
reasonCode: npcregistrysourcerefs.ReasonCode,
validatorName: npcregistrysourcerefs.Key,
},
{
name: "unknown unit candidate",
response: []byte(`{"npcs":[{"name":"Mira Thorn","source_refs":[{"start_unit_id":99,"end_unit_id":99}]}]}`),
reasonCode: npcregistrysourcerefs.ReasonCode,
validatorName: npcregistrysourcerefs.Key,
},
} {
t.Run(test.name, func(t *testing.T) {
client := &fakeNPCProductionLLMClient{rawResponses: [][]byte{test.response, test.response, test.response}}
resolved := effective.ResolvedPipeline
resolved.Steps[0].ArtifactLanes[0].ExtractValidationPolicy.SemanticRejection = pipeline.SemanticRejectionRejectOutput
output, err := runPreparedPipeline(t, registries, resolved, client, pipeline.RunInput{RawInput: readNPCFixture(t)})
if err != nil {
t.Fatalf("Run() error = %v, want non-fatal rejected output", err)
if err == nil || !strings.Contains(err.Error(), test.validatorName) {
t.Fatalf("Run() error = %v, want scoped %s rejection", err, test.validatorName)
}
if len(client.requests) != 3 || len(output.Rejected) != 1 || len(output.NormalizeOutputs) != 0 {
t.Fatalf("LLM requests = %d rejected = %#v normalized = %#v, want exhausted rejection", len(client.requests), output.Rejected, output.NormalizeOutputs)
}
rejection := output.Rejected[0]
if rejection.ReasonCode != test.reasonCode || rejection.ValidatorName != test.validatorName || rejection.AttemptCount != 3 {
t.Fatalf("rejection = %#v, want exhausted %s rejection", rejection, test.validatorName)
if client.extractions != 3 || len(output.OutputFiles) != 0 {
t.Fatalf("native extraction calls=%d files=%#v, want exhausted rejection", client.extractions, output.OutputFiles)
}
})
}
}
func TestProductionNPCNormalizationRetryUsesFinalSafeProposal(t *testing.T) {
func TestProductionNPCNormalizationUsesFinalSafeProposal(t *testing.T) {
registries := productionNPCRegistries(t)
baseResponse := npcProductionResponse{NPCs: []npcProductionRecord{
{Name: "Mira", SourceRefs: []npcProductionSourceRef{{StartUnitID: 1, EndUnitID: 1}}},
@@ -184,20 +153,24 @@ func TestProductionNPCNormalizationRetryUsesFinalSafeProposal(t *testing.T) {
safe := []byte(`{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":2}]}`)
for _, test := range []struct {
name string
retries int
responses [][]byte
wantCalls int
wantExhaustion bool
name string
turns int
mode contracts.ExecutionMode
responses [][]byte
wantCalls int
wantFallback bool
wantError bool
}{
{name: "default retry budget", responses: [][]byte{partial}, wantCalls: 1, wantExhaustion: true},
{name: "later complete proposal", retries: 1, responses: [][]byte{partial, safe}, wantCalls: 2},
{name: "permissive final safe groups", mode: contracts.ModePermissive, responses: [][]byte{partial}, wantCalls: 1, wantFallback: true},
{name: "later complete proposal", mode: contracts.ModeStrict, turns: 1, responses: [][]byte{partial, safe}, wantCalls: 2},
{name: "strict final rejection", mode: contracts.ModeStrict, responses: [][]byte{partial}, wantCalls: 1, wantError: true},
} {
t.Run(test.name, func(t *testing.T) {
cfg := loadNPCPipelineConfig(t)
profile := cfg.Pipelines["dnd-npc-registry-fixture"]
lane := profile.Artifacts["npc_registry"]
lane.Normalize.Retries = test.retries
lane.Normalize.MaxAdditionalTurns = &test.turns
lane.Normalize.Mode = &test.mode
profile.Artifacts["npc_registry"] = lane
cfg.Pipelines["dnd-npc-registry-fixture"] = profile
effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-npc-registry-fixture", Catalog: moduleCatalog(registries)})
@@ -206,14 +179,28 @@ func TestProductionNPCNormalizationRetryUsesFinalSafeProposal(t *testing.T) {
}
client := &fakeNPCProductionLLMClient{response: baseResponse, normalizeResponses: test.responses}
output, err := runPreparedPipeline(t, registries, effective.ResolvedPipeline, client, pipeline.RunInput{RawInput: readNPCFixture(t)})
if err != nil {
t.Fatalf("Run() error = %v", err)
if (err != nil) != test.wantError {
t.Fatalf("Run() error=%v, want error %t", err, test.wantError)
}
if client.requestCount(npcnormalize.PromptID) != test.wantCalls || len(output.NormalizeOutputs) != 1 {
t.Fatalf("normalization calls = %d output = %#v, want %d accepted result", client.requestCount(npcnormalize.PromptID), output.NormalizeOutputs, test.wantCalls)
if client.normalizations != test.wantCalls {
t.Fatalf("normalization calls=%d, want %d", client.normalizations, test.wantCalls)
}
if got := hasWarningReason(output.Diagnostics, npcnormalize.ReasonCodeNPCSemanticReconciliationExhausted); got != test.wantExhaustion {
t.Fatalf("diagnostics = %#v, exhaustion = %t, want %t", output.Diagnostics, got, test.wantExhaustion)
if test.wantError {
if len(output.OutputFiles) != 0 {
t.Fatal("strict failure published files")
}
return
}
wantFallbacks := 0
if test.wantFallback {
wantFallbacks = 1
}
if output.FallbackCount != wantFallbacks || hasWarningReason(output.Diagnostics, npcnormalize.ReasonCodeNPCSemanticReconciliationExhausted) != test.wantFallback {
t.Fatalf("outcome=%#v, want fallback %t", output, test.wantFallback)
}
value, err := npccodec.New().Decode(publishedLane(t, output, "npc_registry").Bytes)
if err != nil || len(value.NPCs) != 2 || value.NPCs[0].Name != "Mira Thorn" || len(value.NPCs[0].SourceRefs) != 2 || value.NPCs[1].Name != "Hooded Guard" {
t.Fatalf("final safe registry=%#v error=%v", value, err)
}
})
}
@@ -237,94 +224,49 @@ type fakeNPCProductionLLMClient struct {
response npcProductionResponse
rawResponses [][]byte
normalizeResponses [][]byte
requests []contracts.StructuredCompletionRequest
requests []promptkit.GenerateRequest
mu sync.Mutex
extractions int
normalizations int
}
func (client *fakeNPCProductionLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
snapshot, err := contracts.CloneStructuredCompletionRequest(req)
if err != nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("clone fake request: %w", err)
func (client *fakeNPCProductionLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
client.mu.Lock()
defer client.mu.Unlock()
if err := ctx.Err(); err != nil {
return nil, err
}
client.requests = append(client.requests, snapshot)
client.requests = append(client.requests, request)
var content []byte
switch req.PromptID {
case npcregistry.PromptID:
if client.rawResponses != nil {
index := client.requestCount(npcregistry.PromptID) - 1
if index >= len(client.rawResponses) {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("missing fake NPC response %d", index)
if nativeSchemaHasProperty(request, "duplicate_groups") {
index := client.normalizations
client.normalizations++
content = []byte(`{"duplicate_groups":[]}`)
if client.normalizeResponses != nil {
if index >= len(client.normalizeResponses) {
return nil, fmt.Errorf("missing NPC normalization response %d", index)
}
content = append([]byte(nil), client.rawResponses[index]...)
content = client.normalizeResponses[index]
}
} else if nativeSchemaHasProperty(request, "npcs") {
index := client.extractions
client.extractions++
if client.rawResponses != nil {
if index >= len(client.rawResponses) {
return nil, fmt.Errorf("missing NPC extraction response %d", index)
}
content = client.rawResponses[index]
} else {
var err error
content, err = json.Marshal(client.response)
if err != nil {
return contracts.StructuredCompletionResponse{}, err
return nil, err
}
}
case npcnormalize.PromptID:
content = []byte(`{"duplicate_groups":[]}`)
if client.normalizeResponses != nil {
index := client.requestCount(npcnormalize.PromptID) - 1
if index >= len(client.normalizeResponses) {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("missing fake NPC normalization response %d", index)
}
content = append([]byte(nil), client.normalizeResponses[index]...)
}
default:
return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected fake NPC prompt %q", req.PromptID)
}
if err := json.Unmarshal(content, out); err != nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate NPC structured target: %w", err)
}
return contracts.StructuredCompletionResponse{Content: content}, nil
}
func (client *fakeNPCProductionLLMClient) requestCount(promptID string) int {
count := 0
for _, request := range client.requests {
if request.PromptID == promptID {
count++
}
}
return count
}
func productionNPCRegistries(t *testing.T) pipeline.Registries {
t.Helper()
registries := pipeline.Registries{
Inputs: pipeline.NewInputAdapterRegistry(),
Chunkers: pipeline.NewChunkerRegistry(),
ArtifactCodecs: pipeline.NewArtifactCodecRegistry(),
ArtifactEvidence: pipeline.NewArtifactEvidenceRegistry(),
Extractors: pipeline.NewExtractorRegistry(),
Mergers: pipeline.NewMergerRegistry(),
Normalizers: pipeline.NewNormalizerRegistry(),
Validators: pipeline.NewValidatorRegistry(),
ValidatorChains: pipeline.NewValidatorChainRegistry(),
Outputs: pipeline.NewOutputEncoderRegistry(),
}
for _, registration := range []struct {
name string
fn func(pipeline.Registries) error
}{
{name: "generic", fn: genericregister.Register},
{name: "seriatim", fn: seriatimregister.Register},
{name: "dnd", fn: dndregister.Register},
} {
if err := registration.fn(registries); err != nil {
t.Fatalf("register %s modules: %v", registration.name, err)
}
}
return registries
}
func moduleCatalog(registries pipeline.Registries) pipeline.ModuleCatalog {
return pipeline.ModuleCatalog{
Inputs: registries.Inputs, Chunkers: registries.Chunkers, ArtifactCodecs: registries.ArtifactCodecs, ArtifactEvidence: registries.ArtifactEvidence,
Extractors: registries.Extractors, Mergers: registries.Mergers, Normalizers: registries.Normalizers,
Validators: registries.Validators, ValidatorChains: registries.ValidatorChains, Outputs: registries.Outputs,
} else {
return nil, fmt.Errorf("unexpected NPC provider output contract")
}
return &promptkit.GenerateResponse{Content: string(content)}, nil
}
func loadNPCPipelineConfig(t *testing.T) config.Config {

View File

@@ -14,6 +14,7 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop"
jsonoutput "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/output/json"
"gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript"
)
@@ -107,7 +108,7 @@ func dndCapabilityCatalog(t *testing.T, inputSpec, extractorSpec pipeline.Module
}
outputs := pipeline.NewOutputEncoderRegistry()
if err := outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: pipeline.DefaultOutputModule, Stage: pipeline.StageOutput, ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.OutputEncoder, error) { return dndSpellsOutput{}, nil }); err != nil {
if err := outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: pipeline.DefaultOutputModule, Stage: pipeline.StageOutput, ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.OutputEncoder, error) { return jsonoutput.New(), nil }); err != nil {
t.Fatalf("register capability output: %v", err)
}
@@ -204,7 +205,7 @@ func dndSpellsTestCatalog(t *testing.T, specs dndSpellsCatalogSpecs) pipeline.Mo
Stage: pipeline.StageOutput,
ExecutionClass: contracts.ExecutionClassDeterministic,
}, func() (contracts.OutputEncoder, error) {
return dndSpellsOutput{}, nil
return jsonoutput.New(), nil
}); err != nil {
t.Fatalf("register output: %v", err)
}
@@ -226,7 +227,7 @@ type configExtractor struct{ key string }
func (extractor configExtractor) Key() string { return extractor.key }
func (configExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (configExtractor) Extract(context.Context, contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.SpellList], error) {
func (configExtractor) Extract(context.Context, contracts.TypedExtractionRequest[dnd.SpellList]) (contracts.TypedExtractionResult[dnd.SpellList], error) {
return contracts.TypedExtractionResult[dnd.SpellList]{}, nil
}
@@ -264,20 +265,6 @@ func (dndSpellsChunker) Plan(ctx context.Context, req contracts.ChunkRequest) (c
}, nil
}
type dndSpellsOutput struct{}
func (dndSpellsOutput) Key() string {
return pipeline.DefaultOutputModule
}
func (dndSpellsOutput) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
return contracts.OutputResult{
Files: []contracts.OutputFile{
{Name: "output.json", ContentType: "application/json", Bytes: []byte(`{"encoded":true}`)},
},
}, nil
}
func withoutCapability(capabilities []string, capability string) []string {
filtered := make([]string, 0, len(capabilities))
for _, candidate := range capabilities {
@@ -290,5 +277,5 @@ func withoutCapability(capabilities []string, capability string) []string {
var (
_ contracts.Chunker = dndSpellsChunker{}
_ contracts.OutputEncoder = dndSpellsOutput{}
_ contracts.OutputEncoder = jsonoutput.New()
)

View File

@@ -4,8 +4,9 @@ import (
"context"
"encoding/json"
"fmt"
"sync"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"go.maximumdirect.net/promptkit"
)
type extractionResponse struct {
@@ -27,16 +28,22 @@ type fakeSpellsLLMClient struct {
response extractionResponse
responses []extractionResponse
rawResponses [][]byte
requests []contracts.StructuredCompletionRequest
requests []promptkit.GenerateRequest
mu sync.Mutex
}
func (client *fakeSpellsLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
client.requests = append(client.requests, cloneStructuredCompletionRequest(req))
func (client *fakeSpellsLLMClient) Generate(ctx context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
client.mu.Lock()
defer client.mu.Unlock()
if err := ctx.Err(); err != nil {
return nil, err
}
client.requests = append(client.requests, req)
var content []byte
if client.rawResponses != nil {
index := len(client.requests) - 1
if index >= len(client.rawResponses) {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("missing fake response %d", index)
return nil, fmt.Errorf("missing fake response %d", index)
}
content = append([]byte(nil), client.rawResponses[index]...)
} else {
@@ -44,20 +51,17 @@ func (client *fakeSpellsLLMClient) CompleteStructured(_ context.Context, req con
if client.responses != nil {
index := len(client.requests) - 1
if index >= len(client.responses) {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("missing fake response %d", index)
return nil, fmt.Errorf("missing fake response %d", index)
}
response = client.responses[index]
}
var err error
content, err = json.Marshal(response)
if err != nil {
return contracts.StructuredCompletionResponse{}, err
return nil, err
}
}
if err := json.Unmarshal(content, out); err != nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate structured target: %w", err)
}
return contracts.StructuredCompletionResponse{Content: content}, nil
return &promptkit.GenerateResponse{Content: string(content)}, nil
}
func responseSourceRefs(startUnitID int, endUnitID int) []spellSourceRefResponse {
@@ -68,22 +72,3 @@ func responseSourceRefs(startUnitID int, endUnitID int) []spellSourceRefResponse
},
}
}
func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest {
req.Inputs = req.Inputs.Clone()
correction, err := contracts.CloneSemanticCorrection(req.Correction)
if err != nil {
panic(err)
}
req.Correction = correction
if len(req.Vars) == 0 {
req.Vars = nil
return req
}
vars := make(map[string]any, len(req.Vars))
for key, value := range req.Vars {
vars[key] = value
}
req.Vars = vars
return req
}

View File

@@ -17,11 +17,12 @@ import (
spellshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/shape"
spellsourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/source_refs"
"gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript"
"go.maximumdirect.net/promptkit"
)
func runPreparedPipeline(t *testing.T, registries pipeline.Registries, resolved pipeline.ResolvedPipeline, llmClient contracts.StructuredLLMClient, input pipeline.RunInput) (pipeline.RunOutput, error) {
func runPreparedPipeline(t *testing.T, registries pipeline.Registries, resolved pipeline.ResolvedPipeline, llmClient promptkit.LLMClient, input pipeline.RunInput) (pipeline.RunOutput, error) {
t.Helper()
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{LLM: llmClient})
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{Engine: nativeTestEngine(t, llmClient)})
if err != nil {
return pipeline.RunOutput{}, err
}
@@ -57,14 +58,13 @@ func TestRunnerProcessesSeriatimInputWithDNDSpellsExtractor(t *testing.T) {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.NormalizeOutputs) != 1 {
t.Fatalf("len(NormalizeOutputs) = %d, want 1", len(output.NormalizeOutputs))
if len(output.OutputFiles) != 1 {
t.Fatalf("len(OutputFiles) = %d, want 1", len(output.OutputFiles))
}
serializedOutput := output.NormalizeOutputs[0]
if serializedOutput.LaneID != "spells" || serializedOutput.Artifact.Schema.ID != spellcodec.SchemaID || serializedOutput.Artifact.Schema.Version != spellcodec.SchemaVersion {
t.Fatalf("serialized output envelope = %#v, want durable spell artifact schema on spells lane", serializedOutput)
if output.OutputFiles[0].Name != "lanes/spells.json" {
t.Fatalf("spell output = %#v", output.OutputFiles)
}
response := decodeRunnerSpellResponse(t, serializedOutput.Artifact.Content)
response := decodeRunnerSpellResponse(t, output.OutputFiles[0].Bytes)
if len(response.SpellCasts) != 2 {
t.Fatalf("len(spell_casts) = %d, want 2", len(response.SpellCasts))
}
@@ -84,31 +84,6 @@ func TestRunnerProcessesSeriatimInputWithDNDSpellsExtractor(t *testing.T) {
}
}
if output.Manifest.InputModule != transcript.Key {
t.Fatalf("manifest input module = %q, want %q", output.Manifest.InputModule, transcript.Key)
}
if output.Manifest.ValidationStatus != "approved" {
t.Fatalf("ValidationStatus = %q, want approved", output.Manifest.ValidationStatus)
}
if len(output.Manifest.ArtifactLanes) != 1 {
t.Fatalf("len(ArtifactLanes) = %d, want 1", len(output.Manifest.ArtifactLanes))
}
lane := output.Manifest.ArtifactLanes[0]
if lane.ID != "spells" || lane.Extractor != spells.Key {
t.Fatalf("manifest lane = %#v, want spells lane with dnd/spells extractor", lane)
}
extractorMetadata, ok := lane.Metadata["extractor"].(map[string]any)
if !ok {
t.Fatalf("manifest lane metadata = %#v, want extractor metadata", lane.Metadata)
}
if extractorMetadata["prompt_id"] != spells.PromptID ||
extractorMetadata["response_schema_key"] != string(spells.ResponseSchemaKey) ||
extractorMetadata["response_schema_name"] != spells.ResponseSchemaName {
t.Fatalf("extractor metadata = %#v, want prompt/schema identifiers", extractorMetadata)
}
if len(output.OutputFiles) != 1 {
t.Fatalf("len(OutputFiles) = %d, want 1", len(output.OutputFiles))
}
if output.OutputFiles[0].ContentType != "application/json" {
t.Fatalf("ContentType = %q, want application/json", output.OutputFiles[0].ContentType)
}
@@ -140,24 +115,17 @@ func TestRunnerPassesPartyAndGlossaryReferencesToDNDSpellsPrompt(t *testing.T) {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.NormalizeOutputs) != 1 {
t.Fatalf("len(NormalizeOutputs) = %d, want 1", len(output.NormalizeOutputs))
}
if len(output.Manifest.References) != 2 {
t.Fatalf("manifest references = %#v, want party and glossary provenance", output.Manifest.References)
if len(output.OutputFiles) != 1 {
t.Fatalf("len(OutputFiles) = %d, want 1", len(output.OutputFiles))
}
if len(llmClient.requests) != 1 {
t.Fatalf("LLM calls = %d, want 1", len(llmClient.requests))
}
request := llmClient.requests[0]
if request.PromptID != spells.PromptID || request.PromptVersion != spells.SchemaVersion {
t.Fatalf("prompt = %q/%q, want %q/%q", request.PromptID, request.PromptVersion, spells.PromptID, spells.SchemaVersion)
}
if got := string(request.Inputs["party"].Content); got != "Aria: party cleric\nBorin: fighter" {
t.Fatalf("party input = %q, want reference text", got)
}
if got := string(request.Inputs["glossary"].Content); got != "Fire Bolt: evocation cantrip" {
t.Fatalf("glossary input = %q, want reference text", got)
for _, value := range []string{"Aria: party cleric\nBorin: fighter", "Fire Bolt: evocation cantrip"} {
if !strings.Contains(nativePromptText(request), value) {
t.Fatalf("rendered prompt omitted %q", value)
}
}
}
@@ -169,7 +137,7 @@ func TestProductionSpellPipelineRoutesSemanticCandidatesToDeterministicValidator
Chunk: pipeline.ModuleBinding{Module: pipeline.DefaultChunkModule, Options: map[string]any{"max_units": 100}},
Artifacts: map[string]pipeline.ArtifactLaneProfile{
"spells": {
Extract: pipeline.ModuleBinding{Module: spells.Key, Retries: 2},
Extract: pipeline.ModuleBinding{Module: spells.Key},
Normalize: pipeline.Binding(spellnormalize.Key),
},
},
@@ -184,48 +152,38 @@ func TestProductionSpellPipelineRoutesSemanticCandidatesToDeterministicValidator
for _, test := range []struct {
name string
response []byte
reasonCode string
validatorName string
}{
{
name: "blank string",
response: []byte(`{"spell_casts":[{"caster":"","spell":"Cure Wounds","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`),
reasonCode: spellshape.ReasonCode,
validatorName: spellshape.Key,
},
{
name: "empty evidence",
response: []byte(`{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","source_refs":[]}]}`),
reasonCode: spellshape.ReasonCode,
validatorName: spellshape.Key,
},
{
name: "nonpositive unit candidate",
response: []byte(`{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","source_refs":[{"start_unit_id":0,"end_unit_id":1}]}]}`),
reasonCode: spellsourcerefs.ReasonCode,
validatorName: spellsourcerefs.Key,
},
{
name: "unknown unit candidate",
response: []byte(`{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","source_refs":[{"start_unit_id":99,"end_unit_id":99}]}]}`),
reasonCode: spellsourcerefs.ReasonCode,
validatorName: spellsourcerefs.Key,
},
} {
t.Run(test.name, func(t *testing.T) {
client := &fakeSpellsLLMClient{rawResponses: [][]byte{test.response, test.response, test.response}}
resolved := effective.ResolvedPipeline
resolved.Steps[0].ArtifactLanes[0].ExtractValidationPolicy.SemanticRejection = pipeline.SemanticRejectionRejectOutput
output, err := runPreparedPipeline(t, registries, resolved, client, pipeline.RunInput{RawInput: readDNDSpellsFixture(t)})
if err != nil {
t.Fatalf("Run() error = %v, want non-fatal rejected output", err)
if err == nil || !strings.Contains(err.Error(), test.validatorName) {
t.Fatalf("Run() error = %v, want scoped %s rejection", err, test.validatorName)
}
if len(client.requests) != 3 || len(output.Rejected) != 1 || len(output.NormalizeOutputs) != 0 {
t.Fatalf("LLM requests = %d rejected = %#v normalized = %#v, want exhausted rejection", len(client.requests), output.Rejected, output.NormalizeOutputs)
}
rejection := output.Rejected[0]
if rejection.ReasonCode != test.reasonCode || rejection.ValidatorName != test.validatorName || rejection.AttemptCount != 3 {
t.Fatalf("rejection = %#v, want exhausted %s rejection", rejection, test.validatorName)
if len(client.requests) != 3 || len(output.OutputFiles) != 0 {
t.Fatalf("requests=%d files=%#v, want exhausted rejection", len(client.requests), output.OutputFiles)
}
})
}

View File

@@ -22,7 +22,12 @@ func nativeTestEngine(t *testing.T, client promptkit.LLMClient, profileIDs ...st
for index, id := range profileIDs {
profiles[index] = map[string]any{"id": id, "endpoint": "https://example.invalid/v1", "model": "integration-test-model", "api_key_required": false}
}
content, err := json.Marshal(map[string]any{"promptkit": map[string]any{"profiles": profiles}})
content, err := json.Marshal(map[string]any{"promptkit": map[string]any{
"profiles": profiles,
"collections": map[string]any{"notarius": map[string]any{"profiles": []map[string]any{
{"id": "dnd-extraction", "endpoint": "https://example.invalid/v1", "model": "integration-test-model", "api_key_required": false},
}}},
}})
if err != nil {
t.Fatal(err)
}
@@ -72,3 +77,21 @@ func nativePromptText(request promptkit.GenerateRequest) string {
}
return strings.Join(content, "\n")
}
func nativeSchemaHasProperty(request promptkit.GenerateRequest, property string) bool {
if request.StructuredOutput == nil || request.StructuredOutput.JSONSchema == nil {
return false
}
content, err := json.Marshal(request.StructuredOutput.JSONSchema.Schema)
if err != nil {
return false
}
var schema struct {
Properties map[string]json.RawMessage `json:"properties"`
}
if json.Unmarshal(content, &schema) != nil {
return false
}
_, found := schema.Properties[property]
return found
}

View File

@@ -0,0 +1,59 @@
package integration_test
import (
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
dndregister "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/register"
genericregister "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/register"
seriatimregister "gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/register"
)
func productionNPCRegistries(t *testing.T) pipeline.Registries {
t.Helper()
registries := pipeline.Registries{
Inputs: pipeline.NewInputAdapterRegistry(),
Chunkers: pipeline.NewChunkerRegistry(),
ArtifactCodecs: pipeline.NewArtifactCodecRegistry(),
ArtifactEvidence: pipeline.NewArtifactEvidenceRegistry(),
Extractors: pipeline.NewExtractorRegistry(),
Mergers: pipeline.NewMergerRegistry(),
Normalizers: pipeline.NewNormalizerRegistry(),
Validators: pipeline.NewValidatorRegistry(),
ValidatorChains: pipeline.NewValidatorChainRegistry(),
Outputs: pipeline.NewOutputEncoderRegistry(),
}
for _, registration := range []struct {
name string
fn func(pipeline.Registries) error
}{
{name: "generic", fn: genericregister.Register},
{name: "seriatim", fn: seriatimregister.Register},
{name: "dnd", fn: dndregister.Register},
} {
if err := registration.fn(registries); err != nil {
t.Fatalf("register %s modules: %v", registration.name, err)
}
}
return registries
}
func moduleCatalog(registries pipeline.Registries) pipeline.ModuleCatalog {
return pipeline.ModuleCatalog{
Inputs: registries.Inputs, Chunkers: registries.Chunkers, ArtifactCodecs: registries.ArtifactCodecs, ArtifactEvidence: registries.ArtifactEvidence,
Extractors: registries.Extractors, Mergers: registries.Mergers, Normalizers: registries.Normalizers,
Validators: registries.Validators, ValidatorChains: registries.ValidatorChains, Outputs: registries.Outputs,
}
}
func publishedLane(t *testing.T, output pipeline.RunOutput, laneID string) contracts.OutputFile {
t.Helper()
for _, file := range output.OutputFiles {
if file.Name == "lanes/"+laneID+".json" {
return file
}
}
t.Fatalf("published files = %#v, missing lane %q", output.OutputFiles, laneID)
return contracts.OutputFile{}
}

View File

@@ -1,10 +1,6 @@
version: 5
version: 6
output:
directory: ./notarius-output
cache:
chunk_plans:
mode: bypass
checkpoints: {}
debug:
directory: ./notarius-debug
pipelines:
@@ -15,6 +11,6 @@ pipelines:
npc_registry:
extract:
module: dnd/npc-registry
retries: 2
max_additional_turns: 2
normalize: {module: dnd/npc-registry}
output: {module: json}

View File

@@ -1,10 +1,6 @@
version: 5
version: 6
output:
directory: ./notarius-output
cache:
chunk_plans:
mode: bypass
checkpoints: {}
debug:
directory: ./notarius-debug
pipelines: