From 0ad73ff575550f3934309d539daef664d54dc7b9 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 22 Nov 2026 04:39:55 +0000 Subject: [PATCH] Exercise spell and NPC integration through native providers and published artifacts --- docs/roadmap/implementation.md | 12 + .../dnd_npc_registry_runner_test.go | 222 +++++++----------- .../integration/dnd_spells_config_test.go | 23 +- .../integration/dnd_spells_helpers_test.go | 45 ++-- .../integration/dnd_spells_runner_test.go | 80 ++----- .../integration/native_provider_test.go | 25 +- .../modules/integration/registries_test.go | 59 +++++ .../testdata/dnd_npc_registry_pipeline.yml | 8 +- .../modules/integration/testdata/pipeline.yml | 6 +- 9 files changed, 219 insertions(+), 261 deletions(-) create mode 100644 internal/modules/integration/registries_test.go diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index a071ad90..cfdfedee 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1699,6 +1699,18 @@ Seriatim package tests and the unchanged production import-boundary suite. The remaining legacy integration files still prevent running that whole package; their completion/checkpoint fixture migration is not finished. +Migrated the older spell and NPC registry integration fixtures to native +provider requests, real JSON lane publication, and version 6 test configurations. +Kept source ordering/identity, capability failures, reference inputs, malformed +candidate rejection, and consolidation coverage. NPC reconciliation now tests +explicit strict failure, permissive final safe groups with one qualified fallback, +and correction to a complete proposal. Shared production registries remain real; +test providers classify the selected native response contract rather than +recreating completion requests or manually decoding model output into modules. +These selected integration tests, vet/race checks, all framework/configuration +tests, and import-boundary tests pass. Remaining combat, location, occurrence, +and grounded-pipeline fixtures still block full integration compilation. + **Goal:** delete the old architecture rather than leave dormant alternatives. **Inspect/change:** all production/test callers of the feature roadmap's removal diff --git a/internal/modules/integration/dnd_npc_registry_runner_test.go b/internal/modules/integration/dnd_npc_registry_runner_test.go index b9fe3267..19bf0b69 100644 --- a/internal/modules/integration/dnd_npc_registry_runner_test.go +++ b/internal/modules/integration/dnd_npc_registry_runner_test.go @@ -6,22 +6,19 @@ import ( "fmt" "os" "strings" + "sync" "testing" "gitea.maximumdirect.net/eric/notarius/internal/core/config" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" - "gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile" npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcregistry" - "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/npcregistry" npcnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/npcregistry" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity" - dndregister "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/register" npcshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/npcregistry/shape" npcregistrysourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/npcregistry/source_refs" - genericregister "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/register" "gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript" - seriatimregister "gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/register" + "go.maximumdirect.net/promptkit" ) func TestRunnerProcessesSeriatimInputWithProductionDNDNPCPipeline(t *testing.T) { @@ -57,17 +54,15 @@ func TestRunnerProcessesSeriatimInputWithProductionDNDNPCPipeline(t *testing.T) if err != nil { t.Fatalf("Run() error = %v, want nil", err) } - if len(output.NormalizeOutputs) != 1 { - t.Fatalf("normalize outputs = %d, want one NPC output; rejected=%#v", len(output.NormalizeOutputs), output.Rejected) + if len(output.OutputFiles) != 1 { + t.Fatalf("output files = %#v, want NPC registry", output.OutputFiles) } - serialized := output.NormalizeOutputs[0] - if serialized.LaneID != "npc_registry" || serialized.NormalizerKey != npcregistry.Key || serialized.Artifact.Schema.ID != npccodec.SchemaID || serialized.Artifact.Schema.Version != npccodec.SchemaVersion { - t.Fatalf("serialized output = %#v, want durable NPC lane schema", serialized) - } - value, err := npccodec.New().Decode(serialized.Artifact.Content) + file := publishedLane(t, output, "npc_registry") + value, err := npccodec.New().Decode(file.Bytes) if err != nil { - t.Fatalf("Decode(output) error = %v, want durable NPC payload", err) + t.Fatal(err) } + if len(value.NPCs) != 2 { t.Fatalf("NPC output = %#v, want repeated name consolidated and group/PC omitted", value.NPCs) } @@ -88,29 +83,12 @@ func TestRunnerProcessesSeriatimInputWithProductionDNDNPCPipeline(t *testing.T) if !hasNPCWarning(output.Diagnostics, "duplicate_npc_collapsed") { t.Fatalf("diagnostics = %#v, want name-only consolidation advisory", output.Diagnostics) } - if output.Manifest.ValidationStatus != "approved" || len(output.Manifest.ArtifactLanes) != 1 { - t.Fatalf("manifest = %#v, want approved NPC lane", output.Manifest) + if file.ContentType != npccodec.MediaType { + t.Fatalf("NPC content type = %q, want %q", file.ContentType, npccodec.MediaType) } - lane := output.Manifest.ArtifactLanes[0] - if lane.ID != "npc_registry" || lane.Extractor != npcregistry.Key || lane.Merger != pipeline.DefaultMergeModule || lane.Normalizer != npcregistry.Key { - t.Fatalf("manifest lane = %#v, want NPC production composition", lane) - } - normalizerMetadata, ok := lane.Metadata["normalizer"].(map[string]any) - if !ok || normalizerMetadata["identity_policy"] != identity.Policy || normalizerMetadata["normalization_policy"] != npcnormalize.NormalizationPolicy || normalizerMetadata["prompt_id"] != npcnormalize.PromptID || normalizerMetadata["response_schema_id"] != semanticreconcile.ResponseSchemaID { - t.Fatalf("normalizer metadata = %#v, want identity and normalization policies", lane.Metadata) - } - var npcOutputFile *contracts.OutputFile - for index := range output.OutputFiles { - if output.OutputFiles[index].Name == "lanes/npc_registry.json" { - npcOutputFile = &output.OutputFiles[index] - break - } - } - if npcOutputFile == nil || npcOutputFile.ContentType != npccodec.MediaType { - t.Fatalf("output files = %#v, want JSON NPC lane file", output.OutputFiles) - } - if len(client.requests) != 2 || client.requests[0].PromptID != npcregistry.PromptID || client.requests[1].PromptID != npcnormalize.PromptID { - t.Fatalf("LLM requests = %#v, want extraction and normalization prompt requests", client.requests) + + if len(client.requests) != 2 || client.extractions != 1 || client.normalizations != 1 { + t.Fatalf("native requests=%d extraction=%d normalization=%d", len(client.requests), client.extractions, client.normalizations) } } @@ -126,54 +104,45 @@ func TestProductionNPCPipelineRoutesSemanticCandidatesToDeterministicValidators( for _, test := range []struct { name string response []byte - reasonCode string validatorName string }{ { name: "blank string", response: []byte(`{"npcs":[{"name":"","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`), - reasonCode: npcshape.ReasonCode, validatorName: npcshape.Key, }, { name: "empty evidence", response: []byte(`{"npcs":[{"name":"Mira Thorn","source_refs":[]}]}`), - reasonCode: npcshape.ReasonCode, validatorName: npcshape.Key, }, { name: "nonpositive unit candidate", response: []byte(`{"npcs":[{"name":"Mira Thorn","source_refs":[{"start_unit_id":0,"end_unit_id":1}]}]}`), - reasonCode: npcregistrysourcerefs.ReasonCode, validatorName: npcregistrysourcerefs.Key, }, { name: "unknown unit candidate", response: []byte(`{"npcs":[{"name":"Mira Thorn","source_refs":[{"start_unit_id":99,"end_unit_id":99}]}]}`), - reasonCode: npcregistrysourcerefs.ReasonCode, validatorName: npcregistrysourcerefs.Key, }, } { t.Run(test.name, func(t *testing.T) { client := &fakeNPCProductionLLMClient{rawResponses: [][]byte{test.response, test.response, test.response}} resolved := effective.ResolvedPipeline - resolved.Steps[0].ArtifactLanes[0].ExtractValidationPolicy.SemanticRejection = pipeline.SemanticRejectionRejectOutput output, err := runPreparedPipeline(t, registries, resolved, client, pipeline.RunInput{RawInput: readNPCFixture(t)}) - if err != nil { - t.Fatalf("Run() error = %v, want non-fatal rejected output", err) + + if err == nil || !strings.Contains(err.Error(), test.validatorName) { + t.Fatalf("Run() error = %v, want scoped %s rejection", err, test.validatorName) } - if len(client.requests) != 3 || len(output.Rejected) != 1 || len(output.NormalizeOutputs) != 0 { - t.Fatalf("LLM requests = %d rejected = %#v normalized = %#v, want exhausted rejection", len(client.requests), output.Rejected, output.NormalizeOutputs) - } - rejection := output.Rejected[0] - if rejection.ReasonCode != test.reasonCode || rejection.ValidatorName != test.validatorName || rejection.AttemptCount != 3 { - t.Fatalf("rejection = %#v, want exhausted %s rejection", rejection, test.validatorName) + if client.extractions != 3 || len(output.OutputFiles) != 0 { + t.Fatalf("native extraction calls=%d files=%#v, want exhausted rejection", client.extractions, output.OutputFiles) } }) } } -func TestProductionNPCNormalizationRetryUsesFinalSafeProposal(t *testing.T) { +func TestProductionNPCNormalizationUsesFinalSafeProposal(t *testing.T) { registries := productionNPCRegistries(t) baseResponse := npcProductionResponse{NPCs: []npcProductionRecord{ {Name: "Mira", SourceRefs: []npcProductionSourceRef{{StartUnitID: 1, EndUnitID: 1}}}, @@ -184,20 +153,24 @@ func TestProductionNPCNormalizationRetryUsesFinalSafeProposal(t *testing.T) { safe := []byte(`{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":2}]}`) for _, test := range []struct { - name string - retries int - responses [][]byte - wantCalls int - wantExhaustion bool + name string + turns int + mode contracts.ExecutionMode + responses [][]byte + wantCalls int + wantFallback bool + wantError bool }{ - {name: "default retry budget", responses: [][]byte{partial}, wantCalls: 1, wantExhaustion: true}, - {name: "later complete proposal", retries: 1, responses: [][]byte{partial, safe}, wantCalls: 2}, + {name: "permissive final safe groups", mode: contracts.ModePermissive, responses: [][]byte{partial}, wantCalls: 1, wantFallback: true}, + {name: "later complete proposal", mode: contracts.ModeStrict, turns: 1, responses: [][]byte{partial, safe}, wantCalls: 2}, + {name: "strict final rejection", mode: contracts.ModeStrict, responses: [][]byte{partial}, wantCalls: 1, wantError: true}, } { t.Run(test.name, func(t *testing.T) { cfg := loadNPCPipelineConfig(t) profile := cfg.Pipelines["dnd-npc-registry-fixture"] lane := profile.Artifacts["npc_registry"] - lane.Normalize.Retries = test.retries + lane.Normalize.MaxAdditionalTurns = &test.turns + lane.Normalize.Mode = &test.mode profile.Artifacts["npc_registry"] = lane cfg.Pipelines["dnd-npc-registry-fixture"] = profile effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-npc-registry-fixture", Catalog: moduleCatalog(registries)}) @@ -206,14 +179,28 @@ func TestProductionNPCNormalizationRetryUsesFinalSafeProposal(t *testing.T) { } client := &fakeNPCProductionLLMClient{response: baseResponse, normalizeResponses: test.responses} output, err := runPreparedPipeline(t, registries, effective.ResolvedPipeline, client, pipeline.RunInput{RawInput: readNPCFixture(t)}) - if err != nil { - t.Fatalf("Run() error = %v", err) + if (err != nil) != test.wantError { + t.Fatalf("Run() error=%v, want error %t", err, test.wantError) } - if client.requestCount(npcnormalize.PromptID) != test.wantCalls || len(output.NormalizeOutputs) != 1 { - t.Fatalf("normalization calls = %d output = %#v, want %d accepted result", client.requestCount(npcnormalize.PromptID), output.NormalizeOutputs, test.wantCalls) + if client.normalizations != test.wantCalls { + t.Fatalf("normalization calls=%d, want %d", client.normalizations, test.wantCalls) } - if got := hasWarningReason(output.Diagnostics, npcnormalize.ReasonCodeNPCSemanticReconciliationExhausted); got != test.wantExhaustion { - t.Fatalf("diagnostics = %#v, exhaustion = %t, want %t", output.Diagnostics, got, test.wantExhaustion) + if test.wantError { + if len(output.OutputFiles) != 0 { + t.Fatal("strict failure published files") + } + return + } + wantFallbacks := 0 + if test.wantFallback { + wantFallbacks = 1 + } + if output.FallbackCount != wantFallbacks || hasWarningReason(output.Diagnostics, npcnormalize.ReasonCodeNPCSemanticReconciliationExhausted) != test.wantFallback { + t.Fatalf("outcome=%#v, want fallback %t", output, test.wantFallback) + } + value, err := npccodec.New().Decode(publishedLane(t, output, "npc_registry").Bytes) + if err != nil || len(value.NPCs) != 2 || value.NPCs[0].Name != "Mira Thorn" || len(value.NPCs[0].SourceRefs) != 2 || value.NPCs[1].Name != "Hooded Guard" { + t.Fatalf("final safe registry=%#v error=%v", value, err) } }) } @@ -237,94 +224,49 @@ type fakeNPCProductionLLMClient struct { response npcProductionResponse rawResponses [][]byte normalizeResponses [][]byte - requests []contracts.StructuredCompletionRequest + requests []promptkit.GenerateRequest + mu sync.Mutex + extractions int + normalizations int } -func (client *fakeNPCProductionLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { - snapshot, err := contracts.CloneStructuredCompletionRequest(req) - if err != nil { - return contracts.StructuredCompletionResponse{}, fmt.Errorf("clone fake request: %w", err) +func (client *fakeNPCProductionLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + client.mu.Lock() + defer client.mu.Unlock() + if err := ctx.Err(); err != nil { + return nil, err } - client.requests = append(client.requests, snapshot) + client.requests = append(client.requests, request) var content []byte - switch req.PromptID { - case npcregistry.PromptID: - if client.rawResponses != nil { - index := client.requestCount(npcregistry.PromptID) - 1 - if index >= len(client.rawResponses) { - return contracts.StructuredCompletionResponse{}, fmt.Errorf("missing fake NPC response %d", index) + if nativeSchemaHasProperty(request, "duplicate_groups") { + index := client.normalizations + client.normalizations++ + content = []byte(`{"duplicate_groups":[]}`) + if client.normalizeResponses != nil { + if index >= len(client.normalizeResponses) { + return nil, fmt.Errorf("missing NPC normalization response %d", index) } - content = append([]byte(nil), client.rawResponses[index]...) + content = client.normalizeResponses[index] + } + } else if nativeSchemaHasProperty(request, "npcs") { + index := client.extractions + client.extractions++ + if client.rawResponses != nil { + if index >= len(client.rawResponses) { + return nil, fmt.Errorf("missing NPC extraction response %d", index) + } + content = client.rawResponses[index] } else { var err error content, err = json.Marshal(client.response) if err != nil { - return contracts.StructuredCompletionResponse{}, err + return nil, err } } - case npcnormalize.PromptID: - content = []byte(`{"duplicate_groups":[]}`) - if client.normalizeResponses != nil { - index := client.requestCount(npcnormalize.PromptID) - 1 - if index >= len(client.normalizeResponses) { - return contracts.StructuredCompletionResponse{}, fmt.Errorf("missing fake NPC normalization response %d", index) - } - content = append([]byte(nil), client.normalizeResponses[index]...) - } - default: - return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected fake NPC prompt %q", req.PromptID) - } - if err := json.Unmarshal(content, out); err != nil { - return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate NPC structured target: %w", err) - } - return contracts.StructuredCompletionResponse{Content: content}, nil -} - -func (client *fakeNPCProductionLLMClient) requestCount(promptID string) int { - count := 0 - for _, request := range client.requests { - if request.PromptID == promptID { - count++ - } - } - return count -} - -func productionNPCRegistries(t *testing.T) pipeline.Registries { - t.Helper() - registries := pipeline.Registries{ - Inputs: pipeline.NewInputAdapterRegistry(), - Chunkers: pipeline.NewChunkerRegistry(), - ArtifactCodecs: pipeline.NewArtifactCodecRegistry(), - ArtifactEvidence: pipeline.NewArtifactEvidenceRegistry(), - Extractors: pipeline.NewExtractorRegistry(), - Mergers: pipeline.NewMergerRegistry(), - Normalizers: pipeline.NewNormalizerRegistry(), - Validators: pipeline.NewValidatorRegistry(), - ValidatorChains: pipeline.NewValidatorChainRegistry(), - Outputs: pipeline.NewOutputEncoderRegistry(), - } - for _, registration := range []struct { - name string - fn func(pipeline.Registries) error - }{ - {name: "generic", fn: genericregister.Register}, - {name: "seriatim", fn: seriatimregister.Register}, - {name: "dnd", fn: dndregister.Register}, - } { - if err := registration.fn(registries); err != nil { - t.Fatalf("register %s modules: %v", registration.name, err) - } - } - return registries -} - -func moduleCatalog(registries pipeline.Registries) pipeline.ModuleCatalog { - return pipeline.ModuleCatalog{ - Inputs: registries.Inputs, Chunkers: registries.Chunkers, ArtifactCodecs: registries.ArtifactCodecs, ArtifactEvidence: registries.ArtifactEvidence, - Extractors: registries.Extractors, Mergers: registries.Mergers, Normalizers: registries.Normalizers, - Validators: registries.Validators, ValidatorChains: registries.ValidatorChains, Outputs: registries.Outputs, + } else { + return nil, fmt.Errorf("unexpected NPC provider output contract") } + return &promptkit.GenerateResponse{Content: string(content)}, nil } func loadNPCPipelineConfig(t *testing.T) config.Config { diff --git a/internal/modules/integration/dnd_spells_config_test.go b/internal/modules/integration/dnd_spells_config_test.go index 0601799b..b5fe21e4 100644 --- a/internal/modules/integration/dnd_spells_config_test.go +++ b/internal/modules/integration/dnd_spells_config_test.go @@ -14,6 +14,7 @@ import ( "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells" "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder" "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop" + jsonoutput "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/output/json" "gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript" ) @@ -107,7 +108,7 @@ func dndCapabilityCatalog(t *testing.T, inputSpec, extractorSpec pipeline.Module } outputs := pipeline.NewOutputEncoderRegistry() - if err := outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: pipeline.DefaultOutputModule, Stage: pipeline.StageOutput, ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.OutputEncoder, error) { return dndSpellsOutput{}, nil }); err != nil { + if err := outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: pipeline.DefaultOutputModule, Stage: pipeline.StageOutput, ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.OutputEncoder, error) { return jsonoutput.New(), nil }); err != nil { t.Fatalf("register capability output: %v", err) } @@ -204,7 +205,7 @@ func dndSpellsTestCatalog(t *testing.T, specs dndSpellsCatalogSpecs) pipeline.Mo Stage: pipeline.StageOutput, ExecutionClass: contracts.ExecutionClassDeterministic, }, func() (contracts.OutputEncoder, error) { - return dndSpellsOutput{}, nil + return jsonoutput.New(), nil }); err != nil { t.Fatalf("register output: %v", err) } @@ -226,7 +227,7 @@ type configExtractor struct{ key string } func (extractor configExtractor) Key() string { return extractor.key } func (configExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil } -func (configExtractor) Extract(context.Context, contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.SpellList], error) { +func (configExtractor) Extract(context.Context, contracts.TypedExtractionRequest[dnd.SpellList]) (contracts.TypedExtractionResult[dnd.SpellList], error) { return contracts.TypedExtractionResult[dnd.SpellList]{}, nil } @@ -264,20 +265,6 @@ func (dndSpellsChunker) Plan(ctx context.Context, req contracts.ChunkRequest) (c }, nil } -type dndSpellsOutput struct{} - -func (dndSpellsOutput) Key() string { - return pipeline.DefaultOutputModule -} - -func (dndSpellsOutput) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) { - return contracts.OutputResult{ - Files: []contracts.OutputFile{ - {Name: "output.json", ContentType: "application/json", Bytes: []byte(`{"encoded":true}`)}, - }, - }, nil -} - func withoutCapability(capabilities []string, capability string) []string { filtered := make([]string, 0, len(capabilities)) for _, candidate := range capabilities { @@ -290,5 +277,5 @@ func withoutCapability(capabilities []string, capability string) []string { var ( _ contracts.Chunker = dndSpellsChunker{} - _ contracts.OutputEncoder = dndSpellsOutput{} + _ contracts.OutputEncoder = jsonoutput.New() ) diff --git a/internal/modules/integration/dnd_spells_helpers_test.go b/internal/modules/integration/dnd_spells_helpers_test.go index 8da67883..62aece5f 100644 --- a/internal/modules/integration/dnd_spells_helpers_test.go +++ b/internal/modules/integration/dnd_spells_helpers_test.go @@ -4,8 +4,9 @@ import ( "context" "encoding/json" "fmt" + "sync" - "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "go.maximumdirect.net/promptkit" ) type extractionResponse struct { @@ -27,16 +28,22 @@ type fakeSpellsLLMClient struct { response extractionResponse responses []extractionResponse rawResponses [][]byte - requests []contracts.StructuredCompletionRequest + requests []promptkit.GenerateRequest + mu sync.Mutex } -func (client *fakeSpellsLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { - client.requests = append(client.requests, cloneStructuredCompletionRequest(req)) +func (client *fakeSpellsLLMClient) Generate(ctx context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + client.mu.Lock() + defer client.mu.Unlock() + if err := ctx.Err(); err != nil { + return nil, err + } + client.requests = append(client.requests, req) var content []byte if client.rawResponses != nil { index := len(client.requests) - 1 if index >= len(client.rawResponses) { - return contracts.StructuredCompletionResponse{}, fmt.Errorf("missing fake response %d", index) + return nil, fmt.Errorf("missing fake response %d", index) } content = append([]byte(nil), client.rawResponses[index]...) } else { @@ -44,20 +51,17 @@ func (client *fakeSpellsLLMClient) CompleteStructured(_ context.Context, req con if client.responses != nil { index := len(client.requests) - 1 if index >= len(client.responses) { - return contracts.StructuredCompletionResponse{}, fmt.Errorf("missing fake response %d", index) + return nil, fmt.Errorf("missing fake response %d", index) } response = client.responses[index] } var err error content, err = json.Marshal(response) if err != nil { - return contracts.StructuredCompletionResponse{}, err + return nil, err } } - if err := json.Unmarshal(content, out); err != nil { - return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate structured target: %w", err) - } - return contracts.StructuredCompletionResponse{Content: content}, nil + return &promptkit.GenerateResponse{Content: string(content)}, nil } func responseSourceRefs(startUnitID int, endUnitID int) []spellSourceRefResponse { @@ -68,22 +72,3 @@ func responseSourceRefs(startUnitID int, endUnitID int) []spellSourceRefResponse }, } } - -func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest { - req.Inputs = req.Inputs.Clone() - correction, err := contracts.CloneSemanticCorrection(req.Correction) - if err != nil { - panic(err) - } - req.Correction = correction - if len(req.Vars) == 0 { - req.Vars = nil - return req - } - vars := make(map[string]any, len(req.Vars)) - for key, value := range req.Vars { - vars[key] = value - } - req.Vars = vars - return req -} diff --git a/internal/modules/integration/dnd_spells_runner_test.go b/internal/modules/integration/dnd_spells_runner_test.go index c5540145..cf7b9b97 100644 --- a/internal/modules/integration/dnd_spells_runner_test.go +++ b/internal/modules/integration/dnd_spells_runner_test.go @@ -17,11 +17,12 @@ import ( spellshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/shape" spellsourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/source_refs" "gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript" + "go.maximumdirect.net/promptkit" ) -func runPreparedPipeline(t *testing.T, registries pipeline.Registries, resolved pipeline.ResolvedPipeline, llmClient contracts.StructuredLLMClient, input pipeline.RunInput) (pipeline.RunOutput, error) { +func runPreparedPipeline(t *testing.T, registries pipeline.Registries, resolved pipeline.ResolvedPipeline, llmClient promptkit.LLMClient, input pipeline.RunInput) (pipeline.RunOutput, error) { t.Helper() - prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{LLM: llmClient}) + prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{Engine: nativeTestEngine(t, llmClient)}) if err != nil { return pipeline.RunOutput{}, err } @@ -57,14 +58,13 @@ func TestRunnerProcessesSeriatimInputWithDNDSpellsExtractor(t *testing.T) { t.Fatalf("Run() error = %v, want nil", err) } - if len(output.NormalizeOutputs) != 1 { - t.Fatalf("len(NormalizeOutputs) = %d, want 1", len(output.NormalizeOutputs)) + if len(output.OutputFiles) != 1 { + t.Fatalf("len(OutputFiles) = %d, want 1", len(output.OutputFiles)) } - serializedOutput := output.NormalizeOutputs[0] - if serializedOutput.LaneID != "spells" || serializedOutput.Artifact.Schema.ID != spellcodec.SchemaID || serializedOutput.Artifact.Schema.Version != spellcodec.SchemaVersion { - t.Fatalf("serialized output envelope = %#v, want durable spell artifact schema on spells lane", serializedOutput) + if output.OutputFiles[0].Name != "lanes/spells.json" { + t.Fatalf("spell output = %#v", output.OutputFiles) } - response := decodeRunnerSpellResponse(t, serializedOutput.Artifact.Content) + response := decodeRunnerSpellResponse(t, output.OutputFiles[0].Bytes) if len(response.SpellCasts) != 2 { t.Fatalf("len(spell_casts) = %d, want 2", len(response.SpellCasts)) } @@ -84,31 +84,6 @@ func TestRunnerProcessesSeriatimInputWithDNDSpellsExtractor(t *testing.T) { } } - if output.Manifest.InputModule != transcript.Key { - t.Fatalf("manifest input module = %q, want %q", output.Manifest.InputModule, transcript.Key) - } - if output.Manifest.ValidationStatus != "approved" { - t.Fatalf("ValidationStatus = %q, want approved", output.Manifest.ValidationStatus) - } - if len(output.Manifest.ArtifactLanes) != 1 { - t.Fatalf("len(ArtifactLanes) = %d, want 1", len(output.Manifest.ArtifactLanes)) - } - lane := output.Manifest.ArtifactLanes[0] - if lane.ID != "spells" || lane.Extractor != spells.Key { - t.Fatalf("manifest lane = %#v, want spells lane with dnd/spells extractor", lane) - } - extractorMetadata, ok := lane.Metadata["extractor"].(map[string]any) - if !ok { - t.Fatalf("manifest lane metadata = %#v, want extractor metadata", lane.Metadata) - } - if extractorMetadata["prompt_id"] != spells.PromptID || - extractorMetadata["response_schema_key"] != string(spells.ResponseSchemaKey) || - extractorMetadata["response_schema_name"] != spells.ResponseSchemaName { - t.Fatalf("extractor metadata = %#v, want prompt/schema identifiers", extractorMetadata) - } - if len(output.OutputFiles) != 1 { - t.Fatalf("len(OutputFiles) = %d, want 1", len(output.OutputFiles)) - } if output.OutputFiles[0].ContentType != "application/json" { t.Fatalf("ContentType = %q, want application/json", output.OutputFiles[0].ContentType) } @@ -140,24 +115,17 @@ func TestRunnerPassesPartyAndGlossaryReferencesToDNDSpellsPrompt(t *testing.T) { t.Fatalf("Run() error = %v, want nil", err) } - if len(output.NormalizeOutputs) != 1 { - t.Fatalf("len(NormalizeOutputs) = %d, want 1", len(output.NormalizeOutputs)) - } - if len(output.Manifest.References) != 2 { - t.Fatalf("manifest references = %#v, want party and glossary provenance", output.Manifest.References) + if len(output.OutputFiles) != 1 { + t.Fatalf("len(OutputFiles) = %d, want 1", len(output.OutputFiles)) } if len(llmClient.requests) != 1 { t.Fatalf("LLM calls = %d, want 1", len(llmClient.requests)) } request := llmClient.requests[0] - if request.PromptID != spells.PromptID || request.PromptVersion != spells.SchemaVersion { - t.Fatalf("prompt = %q/%q, want %q/%q", request.PromptID, request.PromptVersion, spells.PromptID, spells.SchemaVersion) - } - if got := string(request.Inputs["party"].Content); got != "Aria: party cleric\nBorin: fighter" { - t.Fatalf("party input = %q, want reference text", got) - } - if got := string(request.Inputs["glossary"].Content); got != "Fire Bolt: evocation cantrip" { - t.Fatalf("glossary input = %q, want reference text", got) + for _, value := range []string{"Aria: party cleric\nBorin: fighter", "Fire Bolt: evocation cantrip"} { + if !strings.Contains(nativePromptText(request), value) { + t.Fatalf("rendered prompt omitted %q", value) + } } } @@ -169,7 +137,7 @@ func TestProductionSpellPipelineRoutesSemanticCandidatesToDeterministicValidator Chunk: pipeline.ModuleBinding{Module: pipeline.DefaultChunkModule, Options: map[string]any{"max_units": 100}}, Artifacts: map[string]pipeline.ArtifactLaneProfile{ "spells": { - Extract: pipeline.ModuleBinding{Module: spells.Key, Retries: 2}, + Extract: pipeline.ModuleBinding{Module: spells.Key}, Normalize: pipeline.Binding(spellnormalize.Key), }, }, @@ -184,48 +152,38 @@ func TestProductionSpellPipelineRoutesSemanticCandidatesToDeterministicValidator for _, test := range []struct { name string response []byte - reasonCode string validatorName string }{ { name: "blank string", response: []byte(`{"spell_casts":[{"caster":"","spell":"Cure Wounds","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`), - reasonCode: spellshape.ReasonCode, validatorName: spellshape.Key, }, { name: "empty evidence", response: []byte(`{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","source_refs":[]}]}`), - reasonCode: spellshape.ReasonCode, validatorName: spellshape.Key, }, { name: "nonpositive unit candidate", response: []byte(`{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","source_refs":[{"start_unit_id":0,"end_unit_id":1}]}]}`), - reasonCode: spellsourcerefs.ReasonCode, validatorName: spellsourcerefs.Key, }, { name: "unknown unit candidate", response: []byte(`{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","source_refs":[{"start_unit_id":99,"end_unit_id":99}]}]}`), - reasonCode: spellsourcerefs.ReasonCode, validatorName: spellsourcerefs.Key, }, } { t.Run(test.name, func(t *testing.T) { client := &fakeSpellsLLMClient{rawResponses: [][]byte{test.response, test.response, test.response}} resolved := effective.ResolvedPipeline - resolved.Steps[0].ArtifactLanes[0].ExtractValidationPolicy.SemanticRejection = pipeline.SemanticRejectionRejectOutput output, err := runPreparedPipeline(t, registries, resolved, client, pipeline.RunInput{RawInput: readDNDSpellsFixture(t)}) - if err != nil { - t.Fatalf("Run() error = %v, want non-fatal rejected output", err) + if err == nil || !strings.Contains(err.Error(), test.validatorName) { + t.Fatalf("Run() error = %v, want scoped %s rejection", err, test.validatorName) } - if len(client.requests) != 3 || len(output.Rejected) != 1 || len(output.NormalizeOutputs) != 0 { - t.Fatalf("LLM requests = %d rejected = %#v normalized = %#v, want exhausted rejection", len(client.requests), output.Rejected, output.NormalizeOutputs) - } - rejection := output.Rejected[0] - if rejection.ReasonCode != test.reasonCode || rejection.ValidatorName != test.validatorName || rejection.AttemptCount != 3 { - t.Fatalf("rejection = %#v, want exhausted %s rejection", rejection, test.validatorName) + if len(client.requests) != 3 || len(output.OutputFiles) != 0 { + t.Fatalf("requests=%d files=%#v, want exhausted rejection", len(client.requests), output.OutputFiles) } }) } diff --git a/internal/modules/integration/native_provider_test.go b/internal/modules/integration/native_provider_test.go index ce13d4c6..cf29bfd7 100644 --- a/internal/modules/integration/native_provider_test.go +++ b/internal/modules/integration/native_provider_test.go @@ -22,7 +22,12 @@ func nativeTestEngine(t *testing.T, client promptkit.LLMClient, profileIDs ...st for index, id := range profileIDs { profiles[index] = map[string]any{"id": id, "endpoint": "https://example.invalid/v1", "model": "integration-test-model", "api_key_required": false} } - content, err := json.Marshal(map[string]any{"promptkit": map[string]any{"profiles": profiles}}) + content, err := json.Marshal(map[string]any{"promptkit": map[string]any{ + "profiles": profiles, + "collections": map[string]any{"notarius": map[string]any{"profiles": []map[string]any{ + {"id": "dnd-extraction", "endpoint": "https://example.invalid/v1", "model": "integration-test-model", "api_key_required": false}, + }}}, + }}) if err != nil { t.Fatal(err) } @@ -72,3 +77,21 @@ func nativePromptText(request promptkit.GenerateRequest) string { } return strings.Join(content, "\n") } + +func nativeSchemaHasProperty(request promptkit.GenerateRequest, property string) bool { + if request.StructuredOutput == nil || request.StructuredOutput.JSONSchema == nil { + return false + } + content, err := json.Marshal(request.StructuredOutput.JSONSchema.Schema) + if err != nil { + return false + } + var schema struct { + Properties map[string]json.RawMessage `json:"properties"` + } + if json.Unmarshal(content, &schema) != nil { + return false + } + _, found := schema.Properties[property] + return found +} diff --git a/internal/modules/integration/registries_test.go b/internal/modules/integration/registries_test.go new file mode 100644 index 00000000..0f55f1ca --- /dev/null +++ b/internal/modules/integration/registries_test.go @@ -0,0 +1,59 @@ +package integration_test + +import ( + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" + dndregister "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/register" + genericregister "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/register" + seriatimregister "gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/register" +) + +func productionNPCRegistries(t *testing.T) pipeline.Registries { + t.Helper() + registries := pipeline.Registries{ + Inputs: pipeline.NewInputAdapterRegistry(), + Chunkers: pipeline.NewChunkerRegistry(), + ArtifactCodecs: pipeline.NewArtifactCodecRegistry(), + ArtifactEvidence: pipeline.NewArtifactEvidenceRegistry(), + Extractors: pipeline.NewExtractorRegistry(), + Mergers: pipeline.NewMergerRegistry(), + Normalizers: pipeline.NewNormalizerRegistry(), + Validators: pipeline.NewValidatorRegistry(), + ValidatorChains: pipeline.NewValidatorChainRegistry(), + Outputs: pipeline.NewOutputEncoderRegistry(), + } + for _, registration := range []struct { + name string + fn func(pipeline.Registries) error + }{ + {name: "generic", fn: genericregister.Register}, + {name: "seriatim", fn: seriatimregister.Register}, + {name: "dnd", fn: dndregister.Register}, + } { + if err := registration.fn(registries); err != nil { + t.Fatalf("register %s modules: %v", registration.name, err) + } + } + return registries +} + +func moduleCatalog(registries pipeline.Registries) pipeline.ModuleCatalog { + return pipeline.ModuleCatalog{ + Inputs: registries.Inputs, Chunkers: registries.Chunkers, ArtifactCodecs: registries.ArtifactCodecs, ArtifactEvidence: registries.ArtifactEvidence, + Extractors: registries.Extractors, Mergers: registries.Mergers, Normalizers: registries.Normalizers, + Validators: registries.Validators, ValidatorChains: registries.ValidatorChains, Outputs: registries.Outputs, + } +} + +func publishedLane(t *testing.T, output pipeline.RunOutput, laneID string) contracts.OutputFile { + t.Helper() + for _, file := range output.OutputFiles { + if file.Name == "lanes/"+laneID+".json" { + return file + } + } + t.Fatalf("published files = %#v, missing lane %q", output.OutputFiles, laneID) + return contracts.OutputFile{} +} diff --git a/internal/modules/integration/testdata/dnd_npc_registry_pipeline.yml b/internal/modules/integration/testdata/dnd_npc_registry_pipeline.yml index daf12bd7..5ead836b 100644 --- a/internal/modules/integration/testdata/dnd_npc_registry_pipeline.yml +++ b/internal/modules/integration/testdata/dnd_npc_registry_pipeline.yml @@ -1,10 +1,6 @@ -version: 5 +version: 6 output: directory: ./notarius-output -cache: - chunk_plans: - mode: bypass - checkpoints: {} debug: directory: ./notarius-debug pipelines: @@ -15,6 +11,6 @@ pipelines: npc_registry: extract: module: dnd/npc-registry - retries: 2 + max_additional_turns: 2 normalize: {module: dnd/npc-registry} output: {module: json} diff --git a/internal/modules/integration/testdata/pipeline.yml b/internal/modules/integration/testdata/pipeline.yml index 5ba6d809..0fd88728 100644 --- a/internal/modules/integration/testdata/pipeline.yml +++ b/internal/modules/integration/testdata/pipeline.yml @@ -1,10 +1,6 @@ -version: 5 +version: 6 output: directory: ./notarius-output -cache: - chunk_plans: - mode: bypass - checkpoints: {} debug: directory: ./notarius-debug pipelines: