From 562dfad5cdcca6f4b7e3e12843dec86a08b01cd8 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 22 Nov 2026 04:45:39 +0000 Subject: [PATCH] Migrate ordered NPC and location handoff tests to native execution --- docs/roadmap/implementation.md | 14 ++ internal/modules/integration/config_test.go | 8 +- .../dnd_itemoccurrences_runner_test.go | 8 +- .../dnd_location_registry_runner_test.go | 114 ++++------ .../integration/dnd_npc_grounded_test.go | 11 - .../dnd_npc_occurrences_runner_test.go | 205 +++++++++--------- .../integration/native_provider_test.go | 16 ++ .../modules/integration/registries_test.go | 11 + .../testdata/dnd_npc_occurrences_pipeline.yml | 6 +- 9 files changed, 191 insertions(+), 202 deletions(-) diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index cfdfedee..f7d1b064 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1711,6 +1711,20 @@ These selected integration tests, vet/race checks, all framework/configuration tests, and import-boundary tests pass. Remaining combat, location, occurrence, and grounded-pipeline fixtures still block full integration compilation. +NPC-occurrence and location handoff tests now run through native providers and +published lane/evidence files. Retained contextual same-name location selection, +names-only NPC projection, deterministic durable IDs, source chronology, earlier- +producer requirements, and no consumer execution after strict producer rejection. +The multi-chunk semantic NPC fixture now derives evidence from the actual rendered +chunk rather than completion order, and verifies both downstream chunks with the +canonical registry identity under ordinary parallel workers. Removed only the old +generated-reference checkpoint assertion; existing native cache/eligibility tests +own its replacement behavior. Updated occurrence test configurations to version 6 +and made temporary config cleanup test-owned. These selected integration tests, +vet/repeated race checks, framework/configuration tests, and import-boundary tests +pass. Only combat and grounded-pipeline legacy fixtures remain as integration +compile blockers; CLI/dependency cleanup and complete validation are still pending. + **Goal:** delete the old architecture rather than leave dormant alternatives. **Inspect/change:** all production/test callers of the feature roadmap's removal diff --git a/internal/modules/integration/config_test.go b/internal/modules/integration/config_test.go index b27d72a7..4446a4d1 100644 --- a/internal/modules/integration/config_test.go +++ b/internal/modules/integration/config_test.go @@ -26,11 +26,9 @@ func loadIntegrationConfigText(t *testing.T, content string) config.Config { return loadIntegrationConfig(t, path) } -func configFromYAML(content string) (config.Config, error) { - directory, err := os.MkdirTemp("", "notarius-integration-config-") - if err != nil { - return config.Config{}, err - } +func configFromYAML(t *testing.T, content string) (config.Config, error) { + t.Helper() + directory := t.TempDir() path := filepath.Join(directory, "config.yml") if err := os.WriteFile(path, []byte(content), 0o600); err != nil { return config.Config{}, err diff --git a/internal/modules/integration/dnd_itemoccurrences_runner_test.go b/internal/modules/integration/dnd_itemoccurrences_runner_test.go index a90e68be..e4f47bf2 100644 --- a/internal/modules/integration/dnd_itemoccurrences_runner_test.go +++ b/internal/modules/integration/dnd_itemoccurrences_runner_test.go @@ -9,8 +9,8 @@ import ( func TestItemOccurrenceConfigurationRequiresEarlierItemRegistry(t *testing.T) { registries := productionNPCRegistries(t) - cfg, err := configFromYAML(` -version: 5 + cfg, err := configFromYAML(t, ` +version: 6 pipelines: items: input: {module: seriatim} @@ -41,8 +41,8 @@ pipelines: t.Fatalf("Resolve() error = %v", err) } - broken, err := configFromYAML(strings.Replace(` -version: 5 + broken, err := configFromYAML(t, strings.Replace(` +version: 6 pipelines: items: input: {module: seriatim} diff --git a/internal/modules/integration/dnd_location_registry_runner_test.go b/internal/modules/integration/dnd_location_registry_runner_test.go index cc90e220..da3742e7 100644 --- a/internal/modules/integration/dnd_location_registry_runner_test.go +++ b/internal/modules/integration/dnd_location_registry_runner_test.go @@ -10,15 +10,11 @@ import ( "testing" "gitea.maximumdirect.net/eric/notarius/internal/core/config" - "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/evidencecontext" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" - "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" locationoccurrencecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/locationoccurrences" locationcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/locationregistry" - locationoccurrences "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/locationoccurrences" - locations "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/locationregistry" - locationnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/locationregistry" + "go.maximumdirect.net/promptkit" ) func TestLocationRegistryHandoffProducesOccurrencesAndEvidence(t *testing.T) { @@ -29,15 +25,12 @@ func TestLocationRegistryHandoffProducesOccurrencesAndEvidence(t *testing.T) { if err != nil { t.Fatalf("Run() error = %v", err) } - if len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 2 { - t.Fatalf("run outputs = %#v rejected = %#v, want accepted registry and occurrences", output.NormalizeOutputs, output.Rejected) + if len(output.OutputFiles) != 3 { + t.Fatalf("files=%#v, want two lanes and evidence", output.OutputFiles) } - registryOutput := normalizedLane(t, output, "locations") - if registryOutput.Artifact.Kind != dnd.LocationRegistryKind || registryOutput.Artifact.Schema.ID != locationcodec.SchemaID { - t.Fatalf("registry envelope = %#v, want durable location artifact", registryOutput) - } - registry, err := locationcodec.New().Decode(registryOutput.Artifact.Content) + registryOutput := publishedLane(t, output, "locations") + registry, err := locationcodec.New().Decode(registryOutput.Bytes) if err != nil { t.Fatalf("Decode(registry) error = %v", err) } @@ -45,11 +38,8 @@ func TestLocationRegistryHandoffProducesOccurrencesAndEvidence(t *testing.T) { t.Fatalf("registry = %#v, want same-name locations distinguished by source-derived IDs", registry) } - occurrenceOutput := normalizedLane(t, output, "occurrences") - if occurrenceOutput.Artifact.Kind != dnd.LocationOccurrenceListKind || occurrenceOutput.Artifact.Schema.ID != locationoccurrencecodec.SchemaID { - t.Fatalf("occurrence envelope = %#v, want durable occurrence artifact", occurrenceOutput) - } - occurrences, err := locationoccurrencecodec.New().Decode(occurrenceOutput.Artifact.Content) + occurrenceOutput := publishedLane(t, output, "occurrences") + occurrences, err := locationoccurrencecodec.New().Decode(occurrenceOutput.Bytes) if err != nil { t.Fatalf("Decode(occurrences) error = %v", err) } @@ -62,11 +52,16 @@ func TestLocationRegistryHandoffProducesOccurrencesAndEvidence(t *testing.T) { } } - request := client.requestFor(t, locationoccurrences.PromptID) - registryInput := request.Inputs["location_registry"] - if registryInput.MediaType != locationcodec.MediaType || strings.Contains(string(registryInput.Content), "source_refs") || strings.Contains(string(registryInput.Content), registry.Locations[0].ID) || strings.Contains(string(registryInput.Content), registry.Locations[1].ID) || !strings.Contains(string(registryInput.Content), "registry_refs") { - t.Fatalf("occurrence registry input = %#v, want contextual selector projection", registryInput) + request := client.requestFor(t, "occurrences") + + registryInput, err := nativeJSONInput(request, "locations") + if err != nil { + t.Fatal(err) } + if strings.Contains(string(registryInput), "source_refs") || strings.Contains(string(registryInput), registry.Locations[0].ID) || strings.Contains(string(registryInput), registry.Locations[1].ID) || !strings.Contains(string(registryInput), "registry_refs") { + t.Fatalf("registry input=%s, want contextual source-free selectors", registryInput) + } + contextArtifact := outputFileContent(t, output.OutputFiles, "evidence-context.json") evidence, err := evidencecontext.New().Decode(contextArtifact) if err != nil { @@ -92,8 +87,8 @@ func TestLocationOccurrenceConsumerDoesNotRunAfterRejectedRegistry(t *testing.T) if err == nil { t.Fatal("Run() error = nil, want missing accepted location registry") } - if client.requestCount(locationoccurrences.PromptID) != 0 { - t.Fatalf("occurrence requests = %d, want none after rejected registry", client.requestCount(locationoccurrences.PromptID)) + if client.requestCount("occurrences") != 0 { + t.Fatalf("occurrence requests = %d, want none after rejected registry", client.requestCount("occurrences")) } } @@ -106,24 +101,6 @@ func TestLocationOccurrenceConfigurationRequiresEarlierRegistry(t *testing.T) { } } -func TestLocationOccurrenceCheckpointTracksGeneratedRegistry(t *testing.T) { - registries := productionNPCRegistries(t) - checkpoint := newGeneratedReferenceCheckpointLoader() - _, err := runPreparedPipeline(t, registries, resolveLocationPipeline(t, registries), &locationHandoffLLMClient{}, pipeline.RunInput{ - RawInput: readNPCFixture(t), - Checkpoint: checkpoint, - }) - if err != nil { - t.Fatalf("Run() error = %v", err) - } - for _, dependency := range checkpoint.extractDependencies("occurrences") { - if dependency.Name == "generated-reference:location_registry:0" && strings.HasPrefix(dependency.Value, "sha256:") { - return - } - } - t.Fatalf("occurrence checkpoint dependencies = %#v, want generated location registry fingerprint", checkpoint.extractDependencies("occurrences")) -} - func resolveLocationPipeline(t *testing.T, registries pipeline.Registries) pipeline.ResolvedPipeline { t.Helper() cfg := loadIntegrationConfigText(t, locationPipelineConfig) @@ -138,11 +115,7 @@ func resolveLocationPipeline(t *testing.T, registries pipeline.Registries) pipel return resolved } -const locationPipelineConfig = `version: 5 -cache: - chunk_plans: - mode: bypass - checkpoints: {} +const locationPipelineConfig = `version: 6 pipelines: dnd-locations-fixture: input: @@ -175,28 +148,28 @@ pipelines: type locationHandoffLLMClient struct { mu sync.Mutex - requests []contracts.StructuredCompletionRequest + requests []promptkit.GenerateRequest rejectRegistry bool } -func (client *locationHandoffLLMClient) CompleteStructured(ctx context.Context, request contracts.StructuredCompletionRequest, output any) (contracts.StructuredCompletionResponse, error) { +func (client *locationHandoffLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { if err := ctx.Err(); err != nil { - return contracts.StructuredCompletionResponse{}, err + return nil, err } client.mu.Lock() - client.requests = append(client.requests, cloneStructuredCompletionRequest(request)) + client.requests = append(client.requests, request) client.mu.Unlock() var payload any - switch request.PromptID { - case locations.PromptID: + switch { + case nativeSchemaHasProperty(request, "locations"): name := "Moon Gate" if client.rejectRegistry { name = "" } payload = map[string]any{"locations": []any{map[string]any{"name": name, "source_refs": []any{map[string]int{"start_unit_id": 1, "end_unit_id": 1}}}, map[string]any{"name": name, "source_refs": []any{map[string]int{"start_unit_id": 3, "end_unit_id": 3}}}}} - case locationnormalize.PromptID: + case nativeSchemaHasProperty(request, "duplicate_groups"): payload = map[string]any{"duplicate_groups": []any{}} - case locationoccurrences.PromptID: + case nativeSchemaHasProperty(request, "occurrences"): var projection struct { Locations []struct { Name string `json:"name"` @@ -206,48 +179,49 @@ func (client *locationHandoffLLMClient) CompleteStructured(ctx context.Context, } `json:"registry_refs"` } `json:"locations"` } - if err := json.Unmarshal(request.Inputs["location_registry"].Content, &projection); err != nil { - return contracts.StructuredCompletionResponse{}, fmt.Errorf("decode location registry: %w", err) + content, err := nativeJSONInput(request, "locations") + if err != nil { + return nil, err + } + if err := json.Unmarshal(content, &projection); err != nil { + return nil, fmt.Errorf("decode location registry: %w", err) } if len(projection.Locations) != 2 { - return contracts.StructuredCompletionResponse{}, fmt.Errorf("generated location registry has %d locations, want 2", len(projection.Locations)) + return nil, fmt.Errorf("generated location registry has %d locations, want 2", len(projection.Locations)) } payload = map[string]any{"occurrences": []any{map[string]any{"name": projection.Locations[0].Name, "registry_refs": projection.Locations[0].RegistryRefs, "kind": "visited", "source_refs": []any{map[string]int{"start_unit_id": 1, "end_unit_id": 1}}}, map[string]any{"name": projection.Locations[1].Name, "registry_refs": projection.Locations[1].RegistryRefs, "kind": "mentioned", "source_refs": []any{map[string]int{"start_unit_id": 3, "end_unit_id": 3}}}}} default: - return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected prompt %q", request.PromptID) + return nil, fmt.Errorf("unexpected native location output contract") } content, err := json.Marshal(payload) if err != nil { - return contracts.StructuredCompletionResponse{}, err + return nil, err } - if err := json.Unmarshal(content, output); err != nil { - return contracts.StructuredCompletionResponse{}, err - } - return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "location-handoff"}, nil + return &promptkit.GenerateResponse{Content: string(content)}, nil } -func (client *locationHandoffLLMClient) requestFor(t *testing.T, promptID string) contracts.StructuredCompletionRequest { +func (client *locationHandoffLLMClient) requestFor(t *testing.T, property string) promptkit.GenerateRequest { t.Helper() client.mu.Lock() defer client.mu.Unlock() for _, request := range client.requests { - if request.PromptID == promptID { + if nativeSchemaHasProperty(request, property) { return request } } - t.Fatalf("requests = %#v, missing %q", client.requests, promptID) - return contracts.StructuredCompletionRequest{} + t.Fatalf("requests = %#v, missing %q", client.requests, property) + return promptkit.GenerateRequest{} } -func (client *locationHandoffLLMClient) requestCount(promptID string) int { +func (client *locationHandoffLLMClient) requestCount(property string) int { client.mu.Lock() defer client.mu.Unlock() count := 0 for _, request := range client.requests { - if request.PromptID == promptID { + if nativeSchemaHasProperty(request, property) { count++ } } return count } -var _ contracts.StructuredLLMClient = (*locationHandoffLLMClient)(nil) +var _ promptkit.LLMClient = (*locationHandoffLLMClient)(nil) diff --git a/internal/modules/integration/dnd_npc_grounded_test.go b/internal/modules/integration/dnd_npc_grounded_test.go index 20c44d86..e4c4ed47 100644 --- a/internal/modules/integration/dnd_npc_grounded_test.go +++ b/internal/modules/integration/dnd_npc_grounded_test.go @@ -482,17 +482,6 @@ func runGroundedPipelineWithRaw(t *testing.T, configValue config.Config, registr return output } -func outputFileContent(t *testing.T, files []contracts.OutputFile, name string) []byte { - t.Helper() - for _, file := range files { - if file.Name == name { - return append([]byte(nil), file.Bytes...) - } - } - t.Fatalf("output files = %#v, missing %q", files, name) - return nil -} - func normalizedCombatOutput(t *testing.T, output pipeline.RunOutput) dnd.CombatTurnList { t.Helper() for _, serialized := range output.NormalizeOutputs { diff --git a/internal/modules/integration/dnd_npc_occurrences_runner_test.go b/internal/modules/integration/dnd_npc_occurrences_runner_test.go index 6d850014..81cf1cac 100644 --- a/internal/modules/integration/dnd_npc_occurrences_runner_test.go +++ b/internal/modules/integration/dnd_npc_occurrences_runner_test.go @@ -5,20 +5,16 @@ import ( "encoding/json" "fmt" "reflect" - "strings" "sync" "testing" "gitea.maximumdirect.net/eric/notarius/internal/core/config" "gitea.maximumdirect.net/eric/notarius/internal/core/source" - "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" occurrencecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcoccurrences" npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcregistry" - occurrenceextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/npcoccurrences" - "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/npcregistry" - npcnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/npcregistry" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity" + "go.maximumdirect.net/promptkit" ) func TestNPCOccurrencePipelineUsesAcceptedRegistryAndCurrentEvidence(t *testing.T) { @@ -30,24 +26,22 @@ func TestNPCOccurrencePipelineUsesAcceptedRegistryAndCurrentEvidence(t *testing. if err != nil { t.Fatalf("Run() error = %v", err) } - if len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 2 { - t.Fatalf("run outputs = %#v rejected = %#v, want NPC and occurrence artifacts", output.NormalizeOutputs, output.Rejected) + if len(output.OutputFiles) != 2 { + t.Fatalf("output files = %#v, want registry and occurrences", output.OutputFiles) } - request := client.requestFor(t, occurrenceextract.PromptID) + request := client.requestFor(t, "occurrences") wantRegistry := `{"npcs":[{"name":"Mira Thorn"},{"name":"Hooded Guard"}]}` - if got := string(request.Inputs["npc_registry"].Content); got != wantRegistry { - t.Fatalf("occurrence registry input = %s, want names-only projection %s", got, wantRegistry) + registryInput, err := nativeJSONInput(request, "npcs") + if err != nil { + t.Fatal(err) } - if request.Inputs["npc_registry"].MediaType != npccodec.MediaType { - t.Fatalf("occurrence registry media type = %q, want %q", request.Inputs["npc_registry"].MediaType, npccodec.MediaType) + if string(registryInput) != wantRegistry { + t.Fatalf("occurrence registry = %s, want names-only projection %s", registryInput, wantRegistry) } - serialized := normalizedLane(t, output, "occurrences") - if serialized.Artifact.Schema.ID != occurrencecodec.SchemaID || serialized.Artifact.Schema.Version != occurrencecodec.SchemaVersion { - t.Fatalf("occurrence artifact schema = %#v", serialized.Artifact.Schema) - } - occurrences, err := occurrencecodec.New().Decode(serialized.Artifact.Content) + serialized := publishedLane(t, output, "occurrences") + occurrences, err := occurrencecodec.New().Decode(serialized.Bytes) if err != nil { t.Fatalf("Decode(occurrence output) error = %v", err) } @@ -68,7 +62,7 @@ func TestNPCOccurrencePipelineUsesAcceptedRegistryAndCurrentEvidence(t *testing. } var durable map[string]json.RawMessage - if err := json.Unmarshal(serialized.Artifact.Content, &durable); err != nil { + if err := json.Unmarshal(serialized.Bytes, &durable); err != nil { t.Fatalf("unmarshal durable occurrence payload: %v", err) } if len(durable) != 1 || durable["occurrences"] == nil { @@ -90,46 +84,39 @@ func TestSemanticNPCNormalizationCrossesOrderedRegistryHandoff(t *testing.T) { if err != nil || len(warnings) != 0 { t.Fatalf("MaterializeReferences() error = %v warnings = %#v", err, warnings) } - for stepIndex := range resolved.Steps { - for laneIndex := range resolved.Steps[stepIndex].ArtifactLanes { - resolved.Steps[stepIndex].ArtifactLanes[laneIndex].ExtractValidationPolicy.SemanticRejection = pipeline.SemanticRejectionRejectOutput - } - } client := &semanticNPCOccurrenceClient{} raw := []byte(`{"metadata":{"id":"semantic-session","title":"Semantic NPC session"},"segments":[{"id":1,"start":0,"end":1,"speaker":"DM","text":"Mira Thorn enters."},{"id":2,"start":1,"end":2,"speaker":"DM","text":"Mira Thorn, the Greencloak, waves."}]}`) output, err := runPreparedPipeline(t, registries, resolved, client, pipeline.RunInput{RawInput: raw}) if err != nil { t.Fatalf("Run() error = %v", err) } - if client.requestCount(npcregistry.PromptID) != 2 || client.requestCount(npcnormalize.PromptID) != 1 { + if client.requestCount("npcs") != 2 || client.requestCount("duplicate_groups") != 1 { t.Fatalf("prompt requests = %#v, want two extraction calls and one document normalization call", client.requests) } - npcOutput := normalizedLane(t, output, "npc_registry") - npcsValue, err := npccodec.New().Decode(npcOutput.Artifact.Content) + npcOutput := publishedLane(t, output, "npc_registry") + npcsValue, err := npccodec.New().Decode(npcOutput.Bytes) if err != nil { t.Fatal(err) } - if npcOutput.StepID != "identify-npcs" || len(npcsValue.NPCs) != 1 || npcsValue.NPCs[0].Name != "Mira Thorn" || npcsValue.NPCs[0].ID != identity.DeriveID("Mira Thorn") { + if len(npcsValue.NPCs) != 1 || npcsValue.NPCs[0].Name != "Mira Thorn" || npcsValue.NPCs[0].ID != identity.DeriveID("Mira Thorn") { t.Fatalf("NPC output = %#v / %#v, want canonical ordered producer artifact", npcOutput, npcsValue) } if refs := npcsValue.NPCs[0].SourceRefs; !reflect.DeepEqual(refs, []source.SourceRef{{SourceID: "semantic-session", StartUnitID: 1, EndUnitID: 1}, {SourceID: "semantic-session", StartUnitID: 2, EndUnitID: 2}}) { t.Fatalf("NPC evidence = %#v, want original extraction evidence union", refs) } - occurrenceOutput := normalizedLane(t, output, "occurrences") - if occurrenceOutput.StepID != "extract-occurrences" { - t.Fatalf("occurrence output step = %q, want ordered downstream step", occurrenceOutput.StepID) + registryRequest := client.requestFor(t, "occurrences") + registryInput, err := nativeJSONInput(registryRequest, "npcs") + if err != nil || string(registryInput) != `{"npcs":[{"name":"Mira Thorn"}]}` { + t.Fatalf("downstream registry = %s, error=%v", registryInput, err) } - registryRequest := client.requestFor(t, occurrenceextract.PromptID) - if got := string(registryRequest.Inputs["npc_registry"].Content); got != `{"npcs":[{"name":"Mira Thorn"}]}` { - t.Fatalf("downstream registry = %s, want one canonical NPC name", got) + occurrences, err := occurrencecodec.New().Decode(publishedLane(t, output, "occurrences").Bytes) + if err != nil || len(occurrences.Occurrences) != 2 { + t.Fatalf("downstream occurrences=%#v error=%v, want both chunks", occurrences, err) } - manifestContent, err := json.Marshal(output.Manifest) - if err != nil { - t.Fatal(err) - } - for _, forbidden := range []string{"npc:sha256:", "Mira Thorn enters.", "Mira Thorn, the Greencloak, waves."} { - if strings.Contains(string(manifestContent), forbidden) { - t.Fatalf("manifest leaked private identity or transcript content %q: %s", forbidden, manifestContent) + for index, occurrence := range occurrences.Occurrences { + wantRef := source.SourceRef{SourceID: "semantic-session", StartUnitID: index + 1, EndUnitID: index + 1} + if occurrence.Name != "Mira Thorn" || occurrence.NPCID != npcsValue.NPCs[0].ID || !reflect.DeepEqual(occurrence.SourceRefs, []source.SourceRef{wantRef}) { + t.Fatalf("occurrence=%#v, want canonical identity and current chunk evidence %#v", occurrence, wantRef) } } } @@ -138,16 +125,17 @@ func TestNPCOccurrencePipelineSkipsConsumerWhenNPCProducerIsRejected(t *testing. registries := productionNPCRegistries(t) resolved := resolveNPCOccurrencePipeline(t, registries) client := &npcOccurrenceLLMClient{rejectNPCs: true} + resolved.Steps[0].ArtifactLanes[0].ExtractControls.MaxAdditionalTurns = 0 output, err := runPreparedPipeline(t, registries, resolved, client, pipeline.RunInput{RawInput: readNPCFixture(t)}) if err == nil { t.Fatalf("Run() output = %#v, want missing generated NPC producer error", output) } - if client.requestCount(npcregistry.PromptID) != 1 { - t.Fatalf("NPC requests = %d, want rejected producer", client.requestCount(npcregistry.PromptID)) + if client.requestCount("npcs") != 1 { + t.Fatalf("NPC requests = %d, want rejected producer", client.requestCount("npcs")) } - if client.requestCount(occurrenceextract.PromptID) != 0 { - t.Fatalf("occurrence requests = %d, want none after rejected producer", client.requestCount(occurrenceextract.PromptID)) + if client.requestCount("occurrences") != 0 { + t.Fatalf("occurrence requests = %d, want none after rejected producer", client.requestCount("occurrences")) } } @@ -170,17 +158,6 @@ func loadNPCOccurrencePipelineConfig(t *testing.T) config.Config { return loadIntegrationConfig(t, "testdata/dnd_npc_occurrences_pipeline.yml") } -func normalizedLane(t *testing.T, output pipeline.RunOutput, laneID string) contracts.SerializedOutput { - t.Helper() - for _, serialized := range output.NormalizeOutputs { - if serialized.LaneID == laneID { - return serialized - } - } - t.Fatalf("normalized lanes = %#v, missing %q", output.NormalizeOutputs, laneID) - return contracts.SerializedOutput{} -} - func assertOccurrenceEvidence(t *testing.T, references []source.SourceRef) { t.Helper() if len(references) == 0 { @@ -195,21 +172,21 @@ func assertOccurrenceEvidence(t *testing.T, references []source.SourceRef) { type npcOccurrenceLLMClient struct { mu sync.Mutex - requests []contracts.StructuredCompletionRequest + requests []promptkit.GenerateRequest rejectNPCs bool } -func (client *npcOccurrenceLLMClient) CompleteStructured(ctx context.Context, request contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { +func (client *npcOccurrenceLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { if err := ctx.Err(); err != nil { - return contracts.StructuredCompletionResponse{}, err + return nil, err } client.mu.Lock() - client.requests = append(client.requests, cloneStructuredCompletionRequest(request)) + client.requests = append(client.requests, request) client.mu.Unlock() var payload any - switch request.PromptID { - case npcregistry.PromptID: + switch { + case nativeSchemaHasProperty(request, "npcs"): if client.rejectNPCs { payload = map[string]any{"npcs": []any{map[string]any{ "name": "", "source_refs": []any{map[string]int{"start_unit_id": 1, "end_unit_id": 1}}, @@ -220,109 +197,123 @@ func (client *npcOccurrenceLLMClient) CompleteStructured(ctx context.Context, re map[string]any{"name": "Hooded Guard", "source_refs": []any{map[string]int{"start_unit_id": 3, "end_unit_id": 3}}}, }} } - case npcnormalize.PromptID: + case nativeSchemaHasProperty(request, "duplicate_groups"): payload = map[string]any{"duplicate_groups": []any{}} - case occurrenceextract.PromptID: + case nativeSchemaHasProperty(request, "occurrences"): payload = map[string]any{"occurrences": []any{ - map[string]any{"npc_id": identity.DeriveID("Hooded Guard"), "name": "Hooded Guard", "kind": "noncombat_presence", "source_refs": []any{map[string]int{"start_unit_id": 3, "end_unit_id": 3}}}, - map[string]any{"npc_id": identity.DeriveID("Mira Thorn"), "name": "Mira Thorn", "kind": "dialogue", "source_refs": []any{map[string]int{"start_unit_id": 2, "end_unit_id": 2}}}, + map[string]any{"name": "Hooded Guard", "kind": "noncombat_presence", "source_refs": []any{map[string]int{"start_unit_id": 3, "end_unit_id": 3}}}, + map[string]any{"name": "Mira Thorn", "kind": "dialogue", "source_refs": []any{map[string]int{"start_unit_id": 2, "end_unit_id": 2}}}, }} default: - return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected occurrence prompt %q", request.PromptID) + return nil, fmt.Errorf("unexpected native occurrence response contract") } content, err := json.Marshal(payload) if err != nil { - return contracts.StructuredCompletionResponse{}, err + return nil, err } - if err := json.Unmarshal(content, out); err != nil { - return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate occurrence response: %w", err) - } - return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "occurrence-fake"}, nil + return &promptkit.GenerateResponse{Content: string(content)}, nil } -func (client *npcOccurrenceLLMClient) requestFor(t *testing.T, promptID string) contracts.StructuredCompletionRequest { +func (client *npcOccurrenceLLMClient) requestFor(t *testing.T, property string) promptkit.GenerateRequest { t.Helper() client.mu.Lock() defer client.mu.Unlock() for _, request := range client.requests { - if request.PromptID == promptID { + if nativeSchemaHasProperty(request, property) { return request } } - t.Fatalf("requests = %#v, missing prompt %q", client.requests, promptID) - return contracts.StructuredCompletionRequest{} + t.Fatalf("requests = %#v, missing prompt %q", client.requests, property) + return promptkit.GenerateRequest{} } -func (client *npcOccurrenceLLMClient) requestCount(promptID string) int { +func (client *npcOccurrenceLLMClient) requestCount(property string) int { client.mu.Lock() defer client.mu.Unlock() count := 0 for _, request := range client.requests { - if request.PromptID == promptID { + if nativeSchemaHasProperty(request, property) { count++ } } return count } -var _ contracts.StructuredLLMClient = (*npcOccurrenceLLMClient)(nil) +var _ promptkit.LLMClient = (*npcOccurrenceLLMClient)(nil) type semanticNPCOccurrenceClient struct { - requests []contracts.StructuredCompletionRequest - npcCalls int + mu sync.Mutex + requests []promptkit.GenerateRequest } -func (client *semanticNPCOccurrenceClient) CompleteStructured(_ context.Context, request contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { - client.requests = append(client.requests, cloneStructuredCompletionRequest(request)) +func (client *semanticNPCOccurrenceClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + client.mu.Lock() + defer client.mu.Unlock() + if err := ctx.Err(); err != nil { + return nil, err + } + client.requests = append(client.requests, request) var payload any - switch request.PromptID { - case npcregistry.PromptID: - client.npcCalls++ + var unitID int + if nativeSchemaHasProperty(request, "npcs") || nativeSchemaHasProperty(request, "occurrences") { + content, err := nativeJSONInput(request, "units") + if err != nil { + return nil, err + } + var document struct { + Units []struct { + ID int `json:"id"` + } `json:"units"` + } + if err := json.Unmarshal(content, &document); err != nil { + return nil, err + } + if len(document.Units) != 1 || document.Units[0].ID <= 0 { + return nil, fmt.Errorf("semantic fixture requires one source-addressed unit: %s", content) + } + unitID = document.Units[0].ID + } + switch { + case nativeSchemaHasProperty(request, "npcs"): name := "Mira Thorn" - if client.npcCalls == 2 { + if unitID == 2 { name = "Mira Thorn, the Greencloak" } - payload = map[string]any{"npcs": []any{map[string]any{"name": name, "source_refs": []any{map[string]int{"start_unit_id": client.npcCalls, "end_unit_id": client.npcCalls}}}}} - case npcnormalize.PromptID: + payload = map[string]any{"npcs": []any{map[string]any{"name": name, "source_refs": []any{map[string]int{"start_unit_id": unitID, "end_unit_id": unitID}}}}} + case nativeSchemaHasProperty(request, "duplicate_groups"): content := []byte(`{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":1}]}`) - if err := json.Unmarshal(content, out); err != nil { - return contracts.StructuredCompletionResponse{}, err - } - return contracts.StructuredCompletionResponse{Content: content}, nil - case occurrenceextract.PromptID: - payload = map[string]any{"occurrences": []any{map[string]any{"npc_id": identity.DeriveID("Mira Thorn"), "name": "Mira Thorn", "kind": "dialogue", "source_refs": []any{map[string]int{"start_unit_id": 1, "end_unit_id": 1}}}}} + return &promptkit.GenerateResponse{Content: string(content)}, nil + case nativeSchemaHasProperty(request, "occurrences"): + payload = map[string]any{"occurrences": []any{map[string]any{"name": "Mira Thorn", "kind": "dialogue", "source_refs": []any{map[string]int{"start_unit_id": unitID, "end_unit_id": unitID}}}}} default: - return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected semantic pipeline prompt %q", request.PromptID) + return nil, fmt.Errorf("unexpected native occurrence response contract") } content, err := json.Marshal(payload) if err != nil { - return contracts.StructuredCompletionResponse{}, err + return nil, err } - if err := json.Unmarshal(content, out); err != nil { - return contracts.StructuredCompletionResponse{}, err - } - return contracts.StructuredCompletionResponse{Content: content}, nil + return &promptkit.GenerateResponse{Content: string(content)}, nil } -func (client *semanticNPCOccurrenceClient) requestCount(promptID string) int { +func (client *semanticNPCOccurrenceClient) requestCount(property string) int { count := 0 for _, request := range client.requests { - if request.PromptID == promptID { + if nativeSchemaHasProperty(request, property) { count++ } } return count } -func (client *semanticNPCOccurrenceClient) requestFor(t *testing.T, promptID string) contracts.StructuredCompletionRequest { +func (client *semanticNPCOccurrenceClient) requestFor(t *testing.T, property string) promptkit.GenerateRequest { t.Helper() for _, request := range client.requests { - if request.PromptID == promptID { + if nativeSchemaHasProperty(request, property) { return request } } - t.Fatalf("requests = %#v, missing %q", client.requests, promptID) - return contracts.StructuredCompletionRequest{} + t.Fatalf("requests = %#v, missing %q", client.requests, property) + return promptkit.GenerateRequest{} } -var _ contracts.StructuredLLMClient = (*semanticNPCOccurrenceClient)(nil) +var _ promptkit.LLMClient = (*semanticNPCOccurrenceClient)(nil) diff --git a/internal/modules/integration/native_provider_test.go b/internal/modules/integration/native_provider_test.go index cf29bfd7..1f5d7090 100644 --- a/internal/modules/integration/native_provider_test.go +++ b/internal/modules/integration/native_provider_test.go @@ -95,3 +95,19 @@ func nativeSchemaHasProperty(request promptkit.GenerateRequest, property string) _, found := schema.Properties[property] return found } + +func nativeJSONInput(request promptkit.GenerateRequest, field string) ([]byte, error) { + for _, message := range request.Prompt.Messages { + for offset := range len(message.Content) { + if message.Content[offset] != '{' { + continue + } + decoder := json.NewDecoder(strings.NewReader(message.Content[offset:])) + var object map[string]json.RawMessage + if decoder.Decode(&object) == nil && object[field] != nil { + return []byte(message.Content[offset : offset+int(decoder.InputOffset())]), nil + } + } + } + return nil, fmt.Errorf("native request has no JSON input with field %q", field) +} diff --git a/internal/modules/integration/registries_test.go b/internal/modules/integration/registries_test.go index 0f55f1ca..72a406ef 100644 --- a/internal/modules/integration/registries_test.go +++ b/internal/modules/integration/registries_test.go @@ -57,3 +57,14 @@ func publishedLane(t *testing.T, output pipeline.RunOutput, laneID string) contr t.Fatalf("published files = %#v, missing lane %q", output.OutputFiles, laneID) return contracts.OutputFile{} } + +func outputFileContent(t *testing.T, files []contracts.OutputFile, name string) []byte { + t.Helper() + for _, file := range files { + if file.Name == name { + return append([]byte(nil), file.Bytes...) + } + } + t.Fatalf("output files = %#v, missing %q", files, name) + return nil +} diff --git a/internal/modules/integration/testdata/dnd_npc_occurrences_pipeline.yml b/internal/modules/integration/testdata/dnd_npc_occurrences_pipeline.yml index a9270ba6..9db4fdbe 100644 --- a/internal/modules/integration/testdata/dnd_npc_occurrences_pipeline.yml +++ b/internal/modules/integration/testdata/dnd_npc_occurrences_pipeline.yml @@ -1,10 +1,6 @@ -version: 5 +version: 6 output: directory: ./notarius-output -cache: - chunk_plans: - mode: bypass - checkpoints: {} debug: directory: ./notarius-debug pipelines: