Migrate ordered NPC and location handoff tests to native execution

This commit is contained in:
2026-11-22 04:45:39 +00:00
parent 0ad73ff575
commit 562dfad5cd
9 changed files with 191 additions and 202 deletions

View File

@@ -1711,6 +1711,20 @@ These selected integration tests, vet/race checks, all framework/configuration
tests, and import-boundary tests pass. Remaining combat, location, occurrence, tests, and import-boundary tests pass. Remaining combat, location, occurrence,
and grounded-pipeline fixtures still block full integration compilation. and grounded-pipeline fixtures still block full integration compilation.
NPC-occurrence and location handoff tests now run through native providers and
published lane/evidence files. Retained contextual same-name location selection,
names-only NPC projection, deterministic durable IDs, source chronology, earlier-
producer requirements, and no consumer execution after strict producer rejection.
The multi-chunk semantic NPC fixture now derives evidence from the actual rendered
chunk rather than completion order, and verifies both downstream chunks with the
canonical registry identity under ordinary parallel workers. Removed only the old
generated-reference checkpoint assertion; existing native cache/eligibility tests
own its replacement behavior. Updated occurrence test configurations to version 6
and made temporary config cleanup test-owned. These selected integration tests,
vet/repeated race checks, framework/configuration tests, and import-boundary tests
pass. Only combat and grounded-pipeline legacy fixtures remain as integration
compile blockers; CLI/dependency cleanup and complete validation are still pending.
**Goal:** delete the old architecture rather than leave dormant alternatives. **Goal:** delete the old architecture rather than leave dormant alternatives.
**Inspect/change:** all production/test callers of the feature roadmap's removal **Inspect/change:** all production/test callers of the feature roadmap's removal

View File

@@ -26,11 +26,9 @@ func loadIntegrationConfigText(t *testing.T, content string) config.Config {
return loadIntegrationConfig(t, path) return loadIntegrationConfig(t, path)
} }
func configFromYAML(content string) (config.Config, error) { func configFromYAML(t *testing.T, content string) (config.Config, error) {
directory, err := os.MkdirTemp("", "notarius-integration-config-") t.Helper()
if err != nil { directory := t.TempDir()
return config.Config{}, err
}
path := filepath.Join(directory, "config.yml") path := filepath.Join(directory, "config.yml")
if err := os.WriteFile(path, []byte(content), 0o600); err != nil { if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
return config.Config{}, err return config.Config{}, err

View File

@@ -9,8 +9,8 @@ import (
func TestItemOccurrenceConfigurationRequiresEarlierItemRegistry(t *testing.T) { func TestItemOccurrenceConfigurationRequiresEarlierItemRegistry(t *testing.T) {
registries := productionNPCRegistries(t) registries := productionNPCRegistries(t)
cfg, err := configFromYAML(` cfg, err := configFromYAML(t, `
version: 5 version: 6
pipelines: pipelines:
items: items:
input: {module: seriatim} input: {module: seriatim}
@@ -41,8 +41,8 @@ pipelines:
t.Fatalf("Resolve() error = %v", err) t.Fatalf("Resolve() error = %v", err)
} }
broken, err := configFromYAML(strings.Replace(` broken, err := configFromYAML(t, strings.Replace(`
version: 5 version: 6
pipelines: pipelines:
items: items:
input: {module: seriatim} input: {module: seriatim}

View File

@@ -10,15 +10,11 @@ import (
"testing" "testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/config" "gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/evidencecontext" "gitea.maximumdirect.net/eric/notarius/internal/framework/evidencecontext"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
locationoccurrencecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/locationoccurrences" locationoccurrencecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/locationoccurrences"
locationcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/locationregistry" locationcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/locationregistry"
locationoccurrences "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/locationoccurrences" "go.maximumdirect.net/promptkit"
locations "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/locationregistry"
locationnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/locationregistry"
) )
func TestLocationRegistryHandoffProducesOccurrencesAndEvidence(t *testing.T) { func TestLocationRegistryHandoffProducesOccurrencesAndEvidence(t *testing.T) {
@@ -29,15 +25,12 @@ func TestLocationRegistryHandoffProducesOccurrencesAndEvidence(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("Run() error = %v", err) t.Fatalf("Run() error = %v", err)
} }
if len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 2 { if len(output.OutputFiles) != 3 {
t.Fatalf("run outputs = %#v rejected = %#v, want accepted registry and occurrences", output.NormalizeOutputs, output.Rejected) t.Fatalf("files=%#v, want two lanes and evidence", output.OutputFiles)
} }
registryOutput := normalizedLane(t, output, "locations") registryOutput := publishedLane(t, output, "locations")
if registryOutput.Artifact.Kind != dnd.LocationRegistryKind || registryOutput.Artifact.Schema.ID != locationcodec.SchemaID { registry, err := locationcodec.New().Decode(registryOutput.Bytes)
t.Fatalf("registry envelope = %#v, want durable location artifact", registryOutput)
}
registry, err := locationcodec.New().Decode(registryOutput.Artifact.Content)
if err != nil { if err != nil {
t.Fatalf("Decode(registry) error = %v", err) t.Fatalf("Decode(registry) error = %v", err)
} }
@@ -45,11 +38,8 @@ func TestLocationRegistryHandoffProducesOccurrencesAndEvidence(t *testing.T) {
t.Fatalf("registry = %#v, want same-name locations distinguished by source-derived IDs", registry) t.Fatalf("registry = %#v, want same-name locations distinguished by source-derived IDs", registry)
} }
occurrenceOutput := normalizedLane(t, output, "occurrences") occurrenceOutput := publishedLane(t, output, "occurrences")
if occurrenceOutput.Artifact.Kind != dnd.LocationOccurrenceListKind || occurrenceOutput.Artifact.Schema.ID != locationoccurrencecodec.SchemaID { occurrences, err := locationoccurrencecodec.New().Decode(occurrenceOutput.Bytes)
t.Fatalf("occurrence envelope = %#v, want durable occurrence artifact", occurrenceOutput)
}
occurrences, err := locationoccurrencecodec.New().Decode(occurrenceOutput.Artifact.Content)
if err != nil { if err != nil {
t.Fatalf("Decode(occurrences) error = %v", err) t.Fatalf("Decode(occurrences) error = %v", err)
} }
@@ -62,11 +52,16 @@ func TestLocationRegistryHandoffProducesOccurrencesAndEvidence(t *testing.T) {
} }
} }
request := client.requestFor(t, locationoccurrences.PromptID) request := client.requestFor(t, "occurrences")
registryInput := request.Inputs["location_registry"]
if registryInput.MediaType != locationcodec.MediaType || strings.Contains(string(registryInput.Content), "source_refs") || strings.Contains(string(registryInput.Content), registry.Locations[0].ID) || strings.Contains(string(registryInput.Content), registry.Locations[1].ID) || !strings.Contains(string(registryInput.Content), "registry_refs") { registryInput, err := nativeJSONInput(request, "locations")
t.Fatalf("occurrence registry input = %#v, want contextual selector projection", registryInput) if err != nil {
t.Fatal(err)
} }
if strings.Contains(string(registryInput), "source_refs") || strings.Contains(string(registryInput), registry.Locations[0].ID) || strings.Contains(string(registryInput), registry.Locations[1].ID) || !strings.Contains(string(registryInput), "registry_refs") {
t.Fatalf("registry input=%s, want contextual source-free selectors", registryInput)
}
contextArtifact := outputFileContent(t, output.OutputFiles, "evidence-context.json") contextArtifact := outputFileContent(t, output.OutputFiles, "evidence-context.json")
evidence, err := evidencecontext.New().Decode(contextArtifact) evidence, err := evidencecontext.New().Decode(contextArtifact)
if err != nil { if err != nil {
@@ -92,8 +87,8 @@ func TestLocationOccurrenceConsumerDoesNotRunAfterRejectedRegistry(t *testing.T)
if err == nil { if err == nil {
t.Fatal("Run() error = nil, want missing accepted location registry") t.Fatal("Run() error = nil, want missing accepted location registry")
} }
if client.requestCount(locationoccurrences.PromptID) != 0 { if client.requestCount("occurrences") != 0 {
t.Fatalf("occurrence requests = %d, want none after rejected registry", client.requestCount(locationoccurrences.PromptID)) t.Fatalf("occurrence requests = %d, want none after rejected registry", client.requestCount("occurrences"))
} }
} }
@@ -106,24 +101,6 @@ func TestLocationOccurrenceConfigurationRequiresEarlierRegistry(t *testing.T) {
} }
} }
func TestLocationOccurrenceCheckpointTracksGeneratedRegistry(t *testing.T) {
registries := productionNPCRegistries(t)
checkpoint := newGeneratedReferenceCheckpointLoader()
_, err := runPreparedPipeline(t, registries, resolveLocationPipeline(t, registries), &locationHandoffLLMClient{}, pipeline.RunInput{
RawInput: readNPCFixture(t),
Checkpoint: checkpoint,
})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
for _, dependency := range checkpoint.extractDependencies("occurrences") {
if dependency.Name == "generated-reference:location_registry:0" && strings.HasPrefix(dependency.Value, "sha256:") {
return
}
}
t.Fatalf("occurrence checkpoint dependencies = %#v, want generated location registry fingerprint", checkpoint.extractDependencies("occurrences"))
}
func resolveLocationPipeline(t *testing.T, registries pipeline.Registries) pipeline.ResolvedPipeline { func resolveLocationPipeline(t *testing.T, registries pipeline.Registries) pipeline.ResolvedPipeline {
t.Helper() t.Helper()
cfg := loadIntegrationConfigText(t, locationPipelineConfig) cfg := loadIntegrationConfigText(t, locationPipelineConfig)
@@ -138,11 +115,7 @@ func resolveLocationPipeline(t *testing.T, registries pipeline.Registries) pipel
return resolved return resolved
} }
const locationPipelineConfig = `version: 5 const locationPipelineConfig = `version: 6
cache:
chunk_plans:
mode: bypass
checkpoints: {}
pipelines: pipelines:
dnd-locations-fixture: dnd-locations-fixture:
input: input:
@@ -175,28 +148,28 @@ pipelines:
type locationHandoffLLMClient struct { type locationHandoffLLMClient struct {
mu sync.Mutex mu sync.Mutex
requests []contracts.StructuredCompletionRequest requests []promptkit.GenerateRequest
rejectRegistry bool rejectRegistry bool
} }
func (client *locationHandoffLLMClient) CompleteStructured(ctx context.Context, request contracts.StructuredCompletionRequest, output any) (contracts.StructuredCompletionResponse, error) { func (client *locationHandoffLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
if err := ctx.Err(); err != nil { if err := ctx.Err(); err != nil {
return contracts.StructuredCompletionResponse{}, err return nil, err
} }
client.mu.Lock() client.mu.Lock()
client.requests = append(client.requests, cloneStructuredCompletionRequest(request)) client.requests = append(client.requests, request)
client.mu.Unlock() client.mu.Unlock()
var payload any var payload any
switch request.PromptID { switch {
case locations.PromptID: case nativeSchemaHasProperty(request, "locations"):
name := "Moon Gate" name := "Moon Gate"
if client.rejectRegistry { if client.rejectRegistry {
name = "" name = ""
} }
payload = map[string]any{"locations": []any{map[string]any{"name": name, "source_refs": []any{map[string]int{"start_unit_id": 1, "end_unit_id": 1}}}, map[string]any{"name": name, "source_refs": []any{map[string]int{"start_unit_id": 3, "end_unit_id": 3}}}}} payload = map[string]any{"locations": []any{map[string]any{"name": name, "source_refs": []any{map[string]int{"start_unit_id": 1, "end_unit_id": 1}}}, map[string]any{"name": name, "source_refs": []any{map[string]int{"start_unit_id": 3, "end_unit_id": 3}}}}}
case locationnormalize.PromptID: case nativeSchemaHasProperty(request, "duplicate_groups"):
payload = map[string]any{"duplicate_groups": []any{}} payload = map[string]any{"duplicate_groups": []any{}}
case locationoccurrences.PromptID: case nativeSchemaHasProperty(request, "occurrences"):
var projection struct { var projection struct {
Locations []struct { Locations []struct {
Name string `json:"name"` Name string `json:"name"`
@@ -206,48 +179,49 @@ func (client *locationHandoffLLMClient) CompleteStructured(ctx context.Context,
} `json:"registry_refs"` } `json:"registry_refs"`
} `json:"locations"` } `json:"locations"`
} }
if err := json.Unmarshal(request.Inputs["location_registry"].Content, &projection); err != nil { content, err := nativeJSONInput(request, "locations")
return contracts.StructuredCompletionResponse{}, fmt.Errorf("decode location registry: %w", err) if err != nil {
return nil, err
}
if err := json.Unmarshal(content, &projection); err != nil {
return nil, fmt.Errorf("decode location registry: %w", err)
} }
if len(projection.Locations) != 2 { if len(projection.Locations) != 2 {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("generated location registry has %d locations, want 2", len(projection.Locations)) return nil, fmt.Errorf("generated location registry has %d locations, want 2", len(projection.Locations))
} }
payload = map[string]any{"occurrences": []any{map[string]any{"name": projection.Locations[0].Name, "registry_refs": projection.Locations[0].RegistryRefs, "kind": "visited", "source_refs": []any{map[string]int{"start_unit_id": 1, "end_unit_id": 1}}}, map[string]any{"name": projection.Locations[1].Name, "registry_refs": projection.Locations[1].RegistryRefs, "kind": "mentioned", "source_refs": []any{map[string]int{"start_unit_id": 3, "end_unit_id": 3}}}}} payload = map[string]any{"occurrences": []any{map[string]any{"name": projection.Locations[0].Name, "registry_refs": projection.Locations[0].RegistryRefs, "kind": "visited", "source_refs": []any{map[string]int{"start_unit_id": 1, "end_unit_id": 1}}}, map[string]any{"name": projection.Locations[1].Name, "registry_refs": projection.Locations[1].RegistryRefs, "kind": "mentioned", "source_refs": []any{map[string]int{"start_unit_id": 3, "end_unit_id": 3}}}}}
default: default:
return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected prompt %q", request.PromptID) return nil, fmt.Errorf("unexpected native location output contract")
} }
content, err := json.Marshal(payload) content, err := json.Marshal(payload)
if err != nil { if err != nil {
return contracts.StructuredCompletionResponse{}, err return nil, err
} }
if err := json.Unmarshal(content, output); err != nil { return &promptkit.GenerateResponse{Content: string(content)}, nil
return contracts.StructuredCompletionResponse{}, err
}
return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "location-handoff"}, nil
} }
func (client *locationHandoffLLMClient) requestFor(t *testing.T, promptID string) contracts.StructuredCompletionRequest { func (client *locationHandoffLLMClient) requestFor(t *testing.T, property string) promptkit.GenerateRequest {
t.Helper() t.Helper()
client.mu.Lock() client.mu.Lock()
defer client.mu.Unlock() defer client.mu.Unlock()
for _, request := range client.requests { for _, request := range client.requests {
if request.PromptID == promptID { if nativeSchemaHasProperty(request, property) {
return request return request
} }
} }
t.Fatalf("requests = %#v, missing %q", client.requests, promptID) t.Fatalf("requests = %#v, missing %q", client.requests, property)
return contracts.StructuredCompletionRequest{} return promptkit.GenerateRequest{}
} }
func (client *locationHandoffLLMClient) requestCount(promptID string) int { func (client *locationHandoffLLMClient) requestCount(property string) int {
client.mu.Lock() client.mu.Lock()
defer client.mu.Unlock() defer client.mu.Unlock()
count := 0 count := 0
for _, request := range client.requests { for _, request := range client.requests {
if request.PromptID == promptID { if nativeSchemaHasProperty(request, property) {
count++ count++
} }
} }
return count return count
} }
var _ contracts.StructuredLLMClient = (*locationHandoffLLMClient)(nil) var _ promptkit.LLMClient = (*locationHandoffLLMClient)(nil)

View File

@@ -482,17 +482,6 @@ func runGroundedPipelineWithRaw(t *testing.T, configValue config.Config, registr
return output return output
} }
func outputFileContent(t *testing.T, files []contracts.OutputFile, name string) []byte {
t.Helper()
for _, file := range files {
if file.Name == name {
return append([]byte(nil), file.Bytes...)
}
}
t.Fatalf("output files = %#v, missing %q", files, name)
return nil
}
func normalizedCombatOutput(t *testing.T, output pipeline.RunOutput) dnd.CombatTurnList { func normalizedCombatOutput(t *testing.T, output pipeline.RunOutput) dnd.CombatTurnList {
t.Helper() t.Helper()
for _, serialized := range output.NormalizeOutputs { for _, serialized := range output.NormalizeOutputs {

View File

@@ -5,20 +5,16 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"reflect" "reflect"
"strings"
"sync" "sync"
"testing" "testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/config" "gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
occurrencecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcoccurrences" occurrencecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcoccurrences"
npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcregistry" npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcregistry"
occurrenceextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/npcoccurrences"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/npcregistry"
npcnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/npcregistry"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity"
"go.maximumdirect.net/promptkit"
) )
func TestNPCOccurrencePipelineUsesAcceptedRegistryAndCurrentEvidence(t *testing.T) { func TestNPCOccurrencePipelineUsesAcceptedRegistryAndCurrentEvidence(t *testing.T) {
@@ -30,24 +26,22 @@ func TestNPCOccurrencePipelineUsesAcceptedRegistryAndCurrentEvidence(t *testing.
if err != nil { if err != nil {
t.Fatalf("Run() error = %v", err) t.Fatalf("Run() error = %v", err)
} }
if len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 2 { if len(output.OutputFiles) != 2 {
t.Fatalf("run outputs = %#v rejected = %#v, want NPC and occurrence artifacts", output.NormalizeOutputs, output.Rejected) t.Fatalf("output files = %#v, want registry and occurrences", output.OutputFiles)
} }
request := client.requestFor(t, occurrenceextract.PromptID) request := client.requestFor(t, "occurrences")
wantRegistry := `{"npcs":[{"name":"Mira Thorn"},{"name":"Hooded Guard"}]}` wantRegistry := `{"npcs":[{"name":"Mira Thorn"},{"name":"Hooded Guard"}]}`
if got := string(request.Inputs["npc_registry"].Content); got != wantRegistry { registryInput, err := nativeJSONInput(request, "npcs")
t.Fatalf("occurrence registry input = %s, want names-only projection %s", got, wantRegistry) if err != nil {
t.Fatal(err)
} }
if request.Inputs["npc_registry"].MediaType != npccodec.MediaType { if string(registryInput) != wantRegistry {
t.Fatalf("occurrence registry media type = %q, want %q", request.Inputs["npc_registry"].MediaType, npccodec.MediaType) t.Fatalf("occurrence registry = %s, want names-only projection %s", registryInput, wantRegistry)
} }
serialized := normalizedLane(t, output, "occurrences") serialized := publishedLane(t, output, "occurrences")
if serialized.Artifact.Schema.ID != occurrencecodec.SchemaID || serialized.Artifact.Schema.Version != occurrencecodec.SchemaVersion { occurrences, err := occurrencecodec.New().Decode(serialized.Bytes)
t.Fatalf("occurrence artifact schema = %#v", serialized.Artifact.Schema)
}
occurrences, err := occurrencecodec.New().Decode(serialized.Artifact.Content)
if err != nil { if err != nil {
t.Fatalf("Decode(occurrence output) error = %v", err) t.Fatalf("Decode(occurrence output) error = %v", err)
} }
@@ -68,7 +62,7 @@ func TestNPCOccurrencePipelineUsesAcceptedRegistryAndCurrentEvidence(t *testing.
} }
var durable map[string]json.RawMessage var durable map[string]json.RawMessage
if err := json.Unmarshal(serialized.Artifact.Content, &durable); err != nil { if err := json.Unmarshal(serialized.Bytes, &durable); err != nil {
t.Fatalf("unmarshal durable occurrence payload: %v", err) t.Fatalf("unmarshal durable occurrence payload: %v", err)
} }
if len(durable) != 1 || durable["occurrences"] == nil { if len(durable) != 1 || durable["occurrences"] == nil {
@@ -90,46 +84,39 @@ func TestSemanticNPCNormalizationCrossesOrderedRegistryHandoff(t *testing.T) {
if err != nil || len(warnings) != 0 { if err != nil || len(warnings) != 0 {
t.Fatalf("MaterializeReferences() error = %v warnings = %#v", err, warnings) t.Fatalf("MaterializeReferences() error = %v warnings = %#v", err, warnings)
} }
for stepIndex := range resolved.Steps {
for laneIndex := range resolved.Steps[stepIndex].ArtifactLanes {
resolved.Steps[stepIndex].ArtifactLanes[laneIndex].ExtractValidationPolicy.SemanticRejection = pipeline.SemanticRejectionRejectOutput
}
}
client := &semanticNPCOccurrenceClient{} client := &semanticNPCOccurrenceClient{}
raw := []byte(`{"metadata":{"id":"semantic-session","title":"Semantic NPC session"},"segments":[{"id":1,"start":0,"end":1,"speaker":"DM","text":"Mira Thorn enters."},{"id":2,"start":1,"end":2,"speaker":"DM","text":"Mira Thorn, the Greencloak, waves."}]}`) raw := []byte(`{"metadata":{"id":"semantic-session","title":"Semantic NPC session"},"segments":[{"id":1,"start":0,"end":1,"speaker":"DM","text":"Mira Thorn enters."},{"id":2,"start":1,"end":2,"speaker":"DM","text":"Mira Thorn, the Greencloak, waves."}]}`)
output, err := runPreparedPipeline(t, registries, resolved, client, pipeline.RunInput{RawInput: raw}) output, err := runPreparedPipeline(t, registries, resolved, client, pipeline.RunInput{RawInput: raw})
if err != nil { if err != nil {
t.Fatalf("Run() error = %v", err) t.Fatalf("Run() error = %v", err)
} }
if client.requestCount(npcregistry.PromptID) != 2 || client.requestCount(npcnormalize.PromptID) != 1 { if client.requestCount("npcs") != 2 || client.requestCount("duplicate_groups") != 1 {
t.Fatalf("prompt requests = %#v, want two extraction calls and one document normalization call", client.requests) t.Fatalf("prompt requests = %#v, want two extraction calls and one document normalization call", client.requests)
} }
npcOutput := normalizedLane(t, output, "npc_registry") npcOutput := publishedLane(t, output, "npc_registry")
npcsValue, err := npccodec.New().Decode(npcOutput.Artifact.Content) npcsValue, err := npccodec.New().Decode(npcOutput.Bytes)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if npcOutput.StepID != "identify-npcs" || len(npcsValue.NPCs) != 1 || npcsValue.NPCs[0].Name != "Mira Thorn" || npcsValue.NPCs[0].ID != identity.DeriveID("Mira Thorn") { if len(npcsValue.NPCs) != 1 || npcsValue.NPCs[0].Name != "Mira Thorn" || npcsValue.NPCs[0].ID != identity.DeriveID("Mira Thorn") {
t.Fatalf("NPC output = %#v / %#v, want canonical ordered producer artifact", npcOutput, npcsValue) t.Fatalf("NPC output = %#v / %#v, want canonical ordered producer artifact", npcOutput, npcsValue)
} }
if refs := npcsValue.NPCs[0].SourceRefs; !reflect.DeepEqual(refs, []source.SourceRef{{SourceID: "semantic-session", StartUnitID: 1, EndUnitID: 1}, {SourceID: "semantic-session", StartUnitID: 2, EndUnitID: 2}}) { if refs := npcsValue.NPCs[0].SourceRefs; !reflect.DeepEqual(refs, []source.SourceRef{{SourceID: "semantic-session", StartUnitID: 1, EndUnitID: 1}, {SourceID: "semantic-session", StartUnitID: 2, EndUnitID: 2}}) {
t.Fatalf("NPC evidence = %#v, want original extraction evidence union", refs) t.Fatalf("NPC evidence = %#v, want original extraction evidence union", refs)
} }
occurrenceOutput := normalizedLane(t, output, "occurrences") registryRequest := client.requestFor(t, "occurrences")
if occurrenceOutput.StepID != "extract-occurrences" { registryInput, err := nativeJSONInput(registryRequest, "npcs")
t.Fatalf("occurrence output step = %q, want ordered downstream step", occurrenceOutput.StepID) if err != nil || string(registryInput) != `{"npcs":[{"name":"Mira Thorn"}]}` {
t.Fatalf("downstream registry = %s, error=%v", registryInput, err)
} }
registryRequest := client.requestFor(t, occurrenceextract.PromptID) occurrences, err := occurrencecodec.New().Decode(publishedLane(t, output, "occurrences").Bytes)
if got := string(registryRequest.Inputs["npc_registry"].Content); got != `{"npcs":[{"name":"Mira Thorn"}]}` { if err != nil || len(occurrences.Occurrences) != 2 {
t.Fatalf("downstream registry = %s, want one canonical NPC name", got) t.Fatalf("downstream occurrences=%#v error=%v, want both chunks", occurrences, err)
} }
manifestContent, err := json.Marshal(output.Manifest) for index, occurrence := range occurrences.Occurrences {
if err != nil { wantRef := source.SourceRef{SourceID: "semantic-session", StartUnitID: index + 1, EndUnitID: index + 1}
t.Fatal(err) if occurrence.Name != "Mira Thorn" || occurrence.NPCID != npcsValue.NPCs[0].ID || !reflect.DeepEqual(occurrence.SourceRefs, []source.SourceRef{wantRef}) {
} t.Fatalf("occurrence=%#v, want canonical identity and current chunk evidence %#v", occurrence, wantRef)
for _, forbidden := range []string{"npc:sha256:", "Mira Thorn enters.", "Mira Thorn, the Greencloak, waves."} {
if strings.Contains(string(manifestContent), forbidden) {
t.Fatalf("manifest leaked private identity or transcript content %q: %s", forbidden, manifestContent)
} }
} }
} }
@@ -138,16 +125,17 @@ func TestNPCOccurrencePipelineSkipsConsumerWhenNPCProducerIsRejected(t *testing.
registries := productionNPCRegistries(t) registries := productionNPCRegistries(t)
resolved := resolveNPCOccurrencePipeline(t, registries) resolved := resolveNPCOccurrencePipeline(t, registries)
client := &npcOccurrenceLLMClient{rejectNPCs: true} client := &npcOccurrenceLLMClient{rejectNPCs: true}
resolved.Steps[0].ArtifactLanes[0].ExtractControls.MaxAdditionalTurns = 0
output, err := runPreparedPipeline(t, registries, resolved, client, pipeline.RunInput{RawInput: readNPCFixture(t)}) output, err := runPreparedPipeline(t, registries, resolved, client, pipeline.RunInput{RawInput: readNPCFixture(t)})
if err == nil { if err == nil {
t.Fatalf("Run() output = %#v, want missing generated NPC producer error", output) t.Fatalf("Run() output = %#v, want missing generated NPC producer error", output)
} }
if client.requestCount(npcregistry.PromptID) != 1 { if client.requestCount("npcs") != 1 {
t.Fatalf("NPC requests = %d, want rejected producer", client.requestCount(npcregistry.PromptID)) t.Fatalf("NPC requests = %d, want rejected producer", client.requestCount("npcs"))
} }
if client.requestCount(occurrenceextract.PromptID) != 0 { if client.requestCount("occurrences") != 0 {
t.Fatalf("occurrence requests = %d, want none after rejected producer", client.requestCount(occurrenceextract.PromptID)) t.Fatalf("occurrence requests = %d, want none after rejected producer", client.requestCount("occurrences"))
} }
} }
@@ -170,17 +158,6 @@ func loadNPCOccurrencePipelineConfig(t *testing.T) config.Config {
return loadIntegrationConfig(t, "testdata/dnd_npc_occurrences_pipeline.yml") return loadIntegrationConfig(t, "testdata/dnd_npc_occurrences_pipeline.yml")
} }
func normalizedLane(t *testing.T, output pipeline.RunOutput, laneID string) contracts.SerializedOutput {
t.Helper()
for _, serialized := range output.NormalizeOutputs {
if serialized.LaneID == laneID {
return serialized
}
}
t.Fatalf("normalized lanes = %#v, missing %q", output.NormalizeOutputs, laneID)
return contracts.SerializedOutput{}
}
func assertOccurrenceEvidence(t *testing.T, references []source.SourceRef) { func assertOccurrenceEvidence(t *testing.T, references []source.SourceRef) {
t.Helper() t.Helper()
if len(references) == 0 { if len(references) == 0 {
@@ -195,21 +172,21 @@ func assertOccurrenceEvidence(t *testing.T, references []source.SourceRef) {
type npcOccurrenceLLMClient struct { type npcOccurrenceLLMClient struct {
mu sync.Mutex mu sync.Mutex
requests []contracts.StructuredCompletionRequest requests []promptkit.GenerateRequest
rejectNPCs bool rejectNPCs bool
} }
func (client *npcOccurrenceLLMClient) CompleteStructured(ctx context.Context, request contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { func (client *npcOccurrenceLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
if err := ctx.Err(); err != nil { if err := ctx.Err(); err != nil {
return contracts.StructuredCompletionResponse{}, err return nil, err
} }
client.mu.Lock() client.mu.Lock()
client.requests = append(client.requests, cloneStructuredCompletionRequest(request)) client.requests = append(client.requests, request)
client.mu.Unlock() client.mu.Unlock()
var payload any var payload any
switch request.PromptID { switch {
case npcregistry.PromptID: case nativeSchemaHasProperty(request, "npcs"):
if client.rejectNPCs { if client.rejectNPCs {
payload = map[string]any{"npcs": []any{map[string]any{ payload = map[string]any{"npcs": []any{map[string]any{
"name": "", "source_refs": []any{map[string]int{"start_unit_id": 1, "end_unit_id": 1}}, "name": "", "source_refs": []any{map[string]int{"start_unit_id": 1, "end_unit_id": 1}},
@@ -220,109 +197,123 @@ func (client *npcOccurrenceLLMClient) CompleteStructured(ctx context.Context, re
map[string]any{"name": "Hooded Guard", "source_refs": []any{map[string]int{"start_unit_id": 3, "end_unit_id": 3}}}, map[string]any{"name": "Hooded Guard", "source_refs": []any{map[string]int{"start_unit_id": 3, "end_unit_id": 3}}},
}} }}
} }
case npcnormalize.PromptID: case nativeSchemaHasProperty(request, "duplicate_groups"):
payload = map[string]any{"duplicate_groups": []any{}} payload = map[string]any{"duplicate_groups": []any{}}
case occurrenceextract.PromptID: case nativeSchemaHasProperty(request, "occurrences"):
payload = map[string]any{"occurrences": []any{ payload = map[string]any{"occurrences": []any{
map[string]any{"npc_id": identity.DeriveID("Hooded Guard"), "name": "Hooded Guard", "kind": "noncombat_presence", "source_refs": []any{map[string]int{"start_unit_id": 3, "end_unit_id": 3}}}, map[string]any{"name": "Hooded Guard", "kind": "noncombat_presence", "source_refs": []any{map[string]int{"start_unit_id": 3, "end_unit_id": 3}}},
map[string]any{"npc_id": identity.DeriveID("Mira Thorn"), "name": "Mira Thorn", "kind": "dialogue", "source_refs": []any{map[string]int{"start_unit_id": 2, "end_unit_id": 2}}}, map[string]any{"name": "Mira Thorn", "kind": "dialogue", "source_refs": []any{map[string]int{"start_unit_id": 2, "end_unit_id": 2}}},
}} }}
default: default:
return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected occurrence prompt %q", request.PromptID) return nil, fmt.Errorf("unexpected native occurrence response contract")
} }
content, err := json.Marshal(payload) content, err := json.Marshal(payload)
if err != nil { if err != nil {
return contracts.StructuredCompletionResponse{}, err return nil, err
} }
if err := json.Unmarshal(content, out); err != nil { return &promptkit.GenerateResponse{Content: string(content)}, nil
return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate occurrence response: %w", err)
}
return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "occurrence-fake"}, nil
} }
func (client *npcOccurrenceLLMClient) requestFor(t *testing.T, promptID string) contracts.StructuredCompletionRequest { func (client *npcOccurrenceLLMClient) requestFor(t *testing.T, property string) promptkit.GenerateRequest {
t.Helper() t.Helper()
client.mu.Lock() client.mu.Lock()
defer client.mu.Unlock() defer client.mu.Unlock()
for _, request := range client.requests { for _, request := range client.requests {
if request.PromptID == promptID { if nativeSchemaHasProperty(request, property) {
return request return request
} }
} }
t.Fatalf("requests = %#v, missing prompt %q", client.requests, promptID) t.Fatalf("requests = %#v, missing prompt %q", client.requests, property)
return contracts.StructuredCompletionRequest{} return promptkit.GenerateRequest{}
} }
func (client *npcOccurrenceLLMClient) requestCount(promptID string) int { func (client *npcOccurrenceLLMClient) requestCount(property string) int {
client.mu.Lock() client.mu.Lock()
defer client.mu.Unlock() defer client.mu.Unlock()
count := 0 count := 0
for _, request := range client.requests { for _, request := range client.requests {
if request.PromptID == promptID { if nativeSchemaHasProperty(request, property) {
count++ count++
} }
} }
return count return count
} }
var _ contracts.StructuredLLMClient = (*npcOccurrenceLLMClient)(nil) var _ promptkit.LLMClient = (*npcOccurrenceLLMClient)(nil)
type semanticNPCOccurrenceClient struct { type semanticNPCOccurrenceClient struct {
requests []contracts.StructuredCompletionRequest mu sync.Mutex
npcCalls int requests []promptkit.GenerateRequest
} }
func (client *semanticNPCOccurrenceClient) CompleteStructured(_ context.Context, request contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { func (client *semanticNPCOccurrenceClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
client.requests = append(client.requests, cloneStructuredCompletionRequest(request)) client.mu.Lock()
defer client.mu.Unlock()
if err := ctx.Err(); err != nil {
return nil, err
}
client.requests = append(client.requests, request)
var payload any var payload any
switch request.PromptID { var unitID int
case npcregistry.PromptID: if nativeSchemaHasProperty(request, "npcs") || nativeSchemaHasProperty(request, "occurrences") {
client.npcCalls++ content, err := nativeJSONInput(request, "units")
if err != nil {
return nil, err
}
var document struct {
Units []struct {
ID int `json:"id"`
} `json:"units"`
}
if err := json.Unmarshal(content, &document); err != nil {
return nil, err
}
if len(document.Units) != 1 || document.Units[0].ID <= 0 {
return nil, fmt.Errorf("semantic fixture requires one source-addressed unit: %s", content)
}
unitID = document.Units[0].ID
}
switch {
case nativeSchemaHasProperty(request, "npcs"):
name := "Mira Thorn" name := "Mira Thorn"
if client.npcCalls == 2 { if unitID == 2 {
name = "Mira Thorn, the Greencloak" name = "Mira Thorn, the Greencloak"
} }
payload = map[string]any{"npcs": []any{map[string]any{"name": name, "source_refs": []any{map[string]int{"start_unit_id": client.npcCalls, "end_unit_id": client.npcCalls}}}}} payload = map[string]any{"npcs": []any{map[string]any{"name": name, "source_refs": []any{map[string]int{"start_unit_id": unitID, "end_unit_id": unitID}}}}}
case npcnormalize.PromptID: case nativeSchemaHasProperty(request, "duplicate_groups"):
content := []byte(`{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":1}]}`) content := []byte(`{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":1}]}`)
if err := json.Unmarshal(content, out); err != nil { return &promptkit.GenerateResponse{Content: string(content)}, nil
return contracts.StructuredCompletionResponse{}, err case nativeSchemaHasProperty(request, "occurrences"):
} payload = map[string]any{"occurrences": []any{map[string]any{"name": "Mira Thorn", "kind": "dialogue", "source_refs": []any{map[string]int{"start_unit_id": unitID, "end_unit_id": unitID}}}}}
return contracts.StructuredCompletionResponse{Content: content}, nil
case occurrenceextract.PromptID:
payload = map[string]any{"occurrences": []any{map[string]any{"npc_id": identity.DeriveID("Mira Thorn"), "name": "Mira Thorn", "kind": "dialogue", "source_refs": []any{map[string]int{"start_unit_id": 1, "end_unit_id": 1}}}}}
default: default:
return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected semantic pipeline prompt %q", request.PromptID) return nil, fmt.Errorf("unexpected native occurrence response contract")
} }
content, err := json.Marshal(payload) content, err := json.Marshal(payload)
if err != nil { if err != nil {
return contracts.StructuredCompletionResponse{}, err return nil, err
} }
if err := json.Unmarshal(content, out); err != nil { return &promptkit.GenerateResponse{Content: string(content)}, nil
return contracts.StructuredCompletionResponse{}, err
}
return contracts.StructuredCompletionResponse{Content: content}, nil
} }
func (client *semanticNPCOccurrenceClient) requestCount(promptID string) int { func (client *semanticNPCOccurrenceClient) requestCount(property string) int {
count := 0 count := 0
for _, request := range client.requests { for _, request := range client.requests {
if request.PromptID == promptID { if nativeSchemaHasProperty(request, property) {
count++ count++
} }
} }
return count return count
} }
func (client *semanticNPCOccurrenceClient) requestFor(t *testing.T, promptID string) contracts.StructuredCompletionRequest { func (client *semanticNPCOccurrenceClient) requestFor(t *testing.T, property string) promptkit.GenerateRequest {
t.Helper() t.Helper()
for _, request := range client.requests { for _, request := range client.requests {
if request.PromptID == promptID { if nativeSchemaHasProperty(request, property) {
return request return request
} }
} }
t.Fatalf("requests = %#v, missing %q", client.requests, promptID) t.Fatalf("requests = %#v, missing %q", client.requests, property)
return contracts.StructuredCompletionRequest{} return promptkit.GenerateRequest{}
} }
var _ contracts.StructuredLLMClient = (*semanticNPCOccurrenceClient)(nil) var _ promptkit.LLMClient = (*semanticNPCOccurrenceClient)(nil)

View File

@@ -95,3 +95,19 @@ func nativeSchemaHasProperty(request promptkit.GenerateRequest, property string)
_, found := schema.Properties[property] _, found := schema.Properties[property]
return found return found
} }
func nativeJSONInput(request promptkit.GenerateRequest, field string) ([]byte, error) {
for _, message := range request.Prompt.Messages {
for offset := range len(message.Content) {
if message.Content[offset] != '{' {
continue
}
decoder := json.NewDecoder(strings.NewReader(message.Content[offset:]))
var object map[string]json.RawMessage
if decoder.Decode(&object) == nil && object[field] != nil {
return []byte(message.Content[offset : offset+int(decoder.InputOffset())]), nil
}
}
}
return nil, fmt.Errorf("native request has no JSON input with field %q", field)
}

View File

@@ -57,3 +57,14 @@ func publishedLane(t *testing.T, output pipeline.RunOutput, laneID string) contr
t.Fatalf("published files = %#v, missing lane %q", output.OutputFiles, laneID) t.Fatalf("published files = %#v, missing lane %q", output.OutputFiles, laneID)
return contracts.OutputFile{} return contracts.OutputFile{}
} }
func outputFileContent(t *testing.T, files []contracts.OutputFile, name string) []byte {
t.Helper()
for _, file := range files {
if file.Name == name {
return append([]byte(nil), file.Bytes...)
}
}
t.Fatalf("output files = %#v, missing %q", files, name)
return nil
}

View File

@@ -1,10 +1,6 @@
version: 5 version: 6
output: output:
directory: ./notarius-output directory: ./notarius-output
cache:
chunk_plans:
mode: bypass
checkpoints: {}
debug: debug:
directory: ./notarius-debug directory: ./notarius-debug
pipelines: pipelines: