diff --git a/assets/dnd/enemy-events/prompts/correction.md b/assets/dnd/enemy-events/prompts/correction.md new file mode 100644 index 00000000..d7c493ea --- /dev/null +++ b/assets/dnd/enemy-events/prompts/correction.md @@ -0,0 +1,8 @@ +Revise the previous enemy-event list using these findings: + +{{ input "guidance" }} + +Return the complete replacement event list using the original combat transcript +and fixed contextual grounding. Include only directly established engagements and +outcomes, retaining their source evidence and chronology. Return only JSON +matching the original response contract. diff --git a/assets/dnd/enemy-events/prompts/prompt.yaml b/assets/dnd/enemy-events/prompts/prompt.yaml index 08d2e4d4..48dfab6a 100644 --- a/assets/dnd/enemy-events/prompts/prompt.yaml +++ b/assets/dnd/enemy-events/prompts/prompt.yaml @@ -26,7 +26,7 @@ prompts: messages: - role: system content_file: ../../shared/prompts/common-dnd-system.md - - role: user + - role: system content_file: ../../shared/prompts/common-dnd-identity.md - role: user content_file: ../../shared/prompts/common-dnd-references.md @@ -36,13 +36,13 @@ prompts: content_file: ../../shared/prompts/common-dnd-transcript-chunk.md cache_control: type: ephemeral - - role: user + - role: system content_file: ../../shared/prompts/common-dnd-extraction-evidence.md - role: user content_file: ../../shared/prompts/common-dnd-npc-registry.md - role: user content_file: ./combat-grounding.md - - role: user + - role: system content_file: ./instructions.md cache_control: type: ephemeral @@ -50,3 +50,17 @@ prompts: format: json validation_mode: json_schema schema_path: dnd/enemy-events/schemas/dnd_enemy_events_llm.v1.json + + - id: dnd.enemy_events.repair + default_profile: notarius/dnd-extraction + inputs: + - name: guidance + required: true + content_type: text/plain + messages: + - role: user + content_file: ./correction.md + output: + format: json + validation_mode: json_schema + schema_path: dnd/enemy-events/schemas/dnd_enemy_events_llm.v1.json diff --git a/assets/enemy_events_test.go b/assets/enemy_events_test.go new file mode 100644 index 00000000..2317c96b --- /dev/null +++ b/assets/enemy_events_test.go @@ -0,0 +1,151 @@ +package assets_test + +import ( + "context" + "encoding/json" + "io/fs" + "reflect" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/assets" + "go.maximumdirect.net/promptkit" +) + +func TestEnemyResponseSchemaDefinesPrivateStructuralBoundary(t *testing.T) { + schema, err := fs.ReadFile(assets.FS(), "dnd/enemy-events/schemas/dnd_enemy_events_llm.v1.json") + if err != nil { + t.Fatal(err) + } + + valid := validEnemyResponse() + content, err := json.Marshal(valid) + if err != nil { + t.Fatal(err) + } + if err := validateJSONSchema(content, schema); err != nil { + t.Fatalf("valid response rejected: %v", err) + } + semantic := validEnemyResponse() + event := semantic["events"].([]any)[0].(map[string]any) + event["name"] = "" + ref := event["source_refs"].([]any)[0].(map[string]any) + ref["start_unit_id"] = 0 + ref["end_unit_id"] = -1 + content, err = json.Marshal(semantic) + if err != nil || validateJSONSchema(content, schema) != nil { + t.Fatalf("non-categorical validator-owned semantics were rejected: %v", err) + } +} + +func TestEnemyResponseSchemaRejectsInvalidStructure(t *testing.T) { + schema, err := fs.ReadFile(assets.FS(), "dnd/enemy-events/schemas/dnd_enemy_events_llm.v1.json") + if err != nil { + t.Fatal(err) + } + for _, mutate := range []func(map[string]any){ + func(event map[string]any) { delete(event, "name") }, + func(event map[string]any) { event["kind"] = 1 }, + func(event map[string]any) { event["kind"] = "unsupported" }, + func(event map[string]any) { event["unexpected"] = true }, + func(event map[string]any) { event["source_refs"].([]any)[0].(map[string]any)["source_id"] = "session" }, + } { + candidate := validEnemyResponse() + mutate(candidate["events"].([]any)[0].(map[string]any)) + content, err := json.Marshal(candidate) + if err != nil { + t.Fatal(err) + } + if err := validateJSONSchema(content, schema); err == nil { + t.Fatal("private schema accepted structurally invalid response") + } + } + +} + +func TestEnemyResponseSchemaAcceptsEverySupportedKind(t *testing.T) { + schema, err := fs.ReadFile(assets.FS(), "dnd/enemy-events/schemas/dnd_enemy_events_llm.v1.json") + if err != nil { + t.Fatal(err) + } + for _, kind := range []string{"engaged", "killed", "fled", "captured", "incapacitated"} { + candidate := validEnemyResponse() + candidate["events"].([]any)[0].(map[string]any)["kind"] = kind + content, err := json.Marshal(candidate) + if err != nil { + t.Fatal(err) + } + if err := validateJSONSchema(content, schema); err != nil { + t.Fatalf("supported enemy-event kind %q was rejected: %v", kind, err) + } + } +} + +func validEnemyResponse() map[string]any { + return map[string]any{"events": []any{map[string]any{ + "name": "Ashfang", "kind": "engaged", "source_refs": []any{map[string]any{"start_unit_id": 1, "end_unit_id": 1}}, + }}} +} + +func TestEnemyPromptPairRetainsRequiredGroundingAndNativeDefaults(t *testing.T) { + _, engine := loadNativeAssets(t) + initial, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "notarius/dnd.enemy_events", APIKey: "test-only", Inputs: enemyEventPromptInputs()}) + if err != nil { + t.Fatal(err) + } + defer initial.Discard() + correction, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{ + PromptID: "notarius/dnd.enemy_events.repair", APIKey: "test-only", + Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline("Combine the duplicate engagement evidence.")}, + }) + if err != nil { + t.Fatal(err) + } + defer correction.Discard() + first, next := initial.Details(), correction.Details() + if first.SelectedProfileID != "notarius/dnd-extraction" || first.SelectedProfileID != next.SelectedProfileID || first.StructuredOutput == nil || !reflect.DeepEqual(first.OutputContract, next.OutputContract) { + t.Fatal("enemy-event prompt pair lost schema/defaults") + } + var system, all string + for _, message := range first.Messages { + all += message.Content + if message.Role == "system" { + system += message.Content + } + } + for _, rule := range []string{"Emit at most one", "A hostile creature without a registry entry is allowed.", "Do not infer an engagement or outcome", "Transcript units are the only evidence"} { + if !strings.Contains(strings.Join(strings.Fields(system), " "), rule) { + t.Fatalf("missing enduring rule %q", rule) + } + } + for _, marker := range []string{"enemy-transcript", "enemy-player", "enemy-party", "enemy-glossary", "enemy-npc", "enemy-turn", "enemy-opponent"} { + if strings.Count(all, marker) != 1 { + t.Fatalf("fixed grounding repeated: %q", marker) + } + } + if len(next.Messages) != 1 || next.Messages[0].Role != "user" || !strings.Contains(next.Messages[0].Content, "Combine the duplicate engagement evidence.") || strings.Contains(next.Messages[0].Content, "enemy-transcript") { + t.Fatal("correction repeated fixed inputs or omitted findings") + } + for _, name := range []string{"npc_registry", "combat_turns", "npc_occurrences"} { + missing := enemyEventPromptInputs() + delete(missing, name) + handle, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "notarius/dnd.enemy_events", APIKey: "test-only", Inputs: missing}) + if handle != nil { + handle.Discard() + } + if err == nil || !strings.Contains(err.Error(), name) { + t.Fatalf("missing %s input=%v", name, err) + } + } +} +func enemyEventPromptInputs() map[string]promptkit.ArtifactRef { + return map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline(`{"value":"enemy-transcript"}`), + "players": promptkit.Inline("enemy-player"), + "party": promptkit.Inline("enemy-party"), + "glossary": promptkit.Inline("enemy-glossary"), + "npc_registry": promptkit.Inline(`{"npcs":[{"name":"enemy-npc"}]}`), + "combat_turns": promptkit.Inline(`{"combat_turns":[{"actor":"enemy-turn","turn_kind":"turn"}]}`), + "npc_occurrences": promptkit.Inline(`{"npc_occurrences":[{"name":"enemy-opponent","kind":"combat_opponent"}]}`), + } +} diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index fe378729..848503e0 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -992,6 +992,29 @@ invocations, not by obsolete completion-interface mocks. ## Stage 22: Migrate Enemy-Event Extraction +**Status:** Complete. Enemy extraction now uses fixed source/reference projections, +native initial/correction requests, and the shared typed producer helper. Added +same-schema/default-profile correction assets and static dependencies; removed +private completion, schema/asset registration, and manifest/fingerprint metadata. +NPC, scene, combat-turn, and combat-opponent reference compatibility, source-free +grounding, contextual subject names, engagement checks, and source-order mapping +remain unchanged. Ineligible scenes produce directly assessed empty results +without native requests, preserving existing unavailable-classification warnings. +Permissive terminal output is an explicit codec-valid empty event list; accepted +work consuming fallback references retains incoming reuse ineligibility. +Retained grounding/domain regressions with native provider fakes and added +full-chain engagement/source correction, fixed multi-artifact context/history, +run selections, empty predecessor handoffs, strict/permissive exhaustion, +failed continuation, structural failure, and cancellation cases. Added a +leaf-assembled native pipeline test retaining the intrinsic codec boundary even +with explicit empty validator chains, and migrated the existing CLI codec +contract's typed request. Native prompt-pair, required-input, and unchanged-schema +checks run in assets. Asset tests/vet/race checks, focused pipeline tests, and +`git diff --check` pass. Extractor, validator, codec, and assembled CLI package +checks remain blocked at setup by shared legacy helpers importing old ConfigKit's +removed YAML `plugin/limit`; their new runtime tests are not claimed passing. +Scheduled dependency cleanup and final verification must execute those tests. + **Goal:** preserve multi-artifact grounding while changing execution ownership. **Inspect/change:** `internal/modules/dnd/extract/enemyevents`, grounding/mapping diff --git a/internal/cli/assembled_enemy_event_codec_contract_test.go b/internal/cli/assembled_enemy_event_codec_contract_test.go index b1416d4c..0e803745 100644 --- a/internal/cli/assembled_enemy_event_codec_contract_test.go +++ b/internal/cli/assembled_enemy_event_codec_contract_test.go @@ -66,7 +66,7 @@ func (invalidEnemyEventExtractor) Key() string { return invalidEnemyEventExtract func (invalidEnemyEventExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil } -func (invalidEnemyEventExtractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.EnemyEventList], error) { +func (invalidEnemyEventExtractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest[dnd.EnemyEventList]) (contracts.TypedExtractionResult[dnd.EnemyEventList], error) { if err := ctx.Err(); err != nil { return contracts.TypedExtractionResult[dnd.EnemyEventList]{}, err } diff --git a/internal/modules/dnd/extract/enemyevents/extractor.go b/internal/modules/dnd/extract/enemyevents/extractor.go index 731d48be..b826ac4b 100644 --- a/internal/modules/dnd/extract/enemyevents/extractor.go +++ b/internal/modules/dnd/extract/enemyevents/extractor.go @@ -2,7 +2,11 @@ package enemyevents import ( "context" + "encoding/json" "fmt" + "strings" + + "go.maximumdirect.net/promptkit" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" @@ -11,11 +15,9 @@ import ( "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" ) -const ( - Key = "dnd/enemy-events" - mappingPolicy = "dnd.enemy_events.extract_mapping.v2" - sceneGatePolicy = "dnd.enemy_events.scene_gate.v1" -) +const Key = "dnd/enemy-events" +const PromptID = "notarius/dnd.enemy_events" +const CorrectionPromptID = "notarius/dnd.enemy_events.repair" var requiredCapabilities = []string{ "chunks", @@ -27,22 +29,17 @@ var providedCapabilities = []string{ } var _ contracts.Extractor[dnd.EnemyEventList] = (*Extractor)(nil) -var _ contracts.ManifestMetadataProvider = (*Extractor)(nil) -var _ pipeline.CheckpointFingerprintProvider = (*Extractor)(nil) type Options struct{} type Extractor struct { - llm contracts.StructuredLLMClient + engine *promptkit.Engine grounding *groundingResolver - - promptSHA string - responseSchemaSHA string } -func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) { - if llmClient == nil { - return nil, extractorErrorf("LLM client must not be nil") +func New(engine *promptkit.Engine, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) { + if engine == nil { + return nil, extractorErrorf("native engine must not be nil") } if len(references) > 1 { return nil, extractorErrorf("at most one reference set may be supplied") @@ -55,62 +52,19 @@ func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contr if err != nil { return nil, extractorErrorf("prepare grounding: %w", err) } - promptSHA, err := promptAssetMetadata() - if err != nil { - return nil, extractorErrorf("load prompt metadata: %w", err) - } - responseSchema, err := loadResponseSchema() - if err != nil { - return nil, extractorErrorf("load response schema: %w", err) - } - return &Extractor{ - llm: llmClient, - grounding: grounding, - promptSHA: promptSHA, - responseSchemaSHA: responseSchema.SHA256, - }, nil + return &Extractor{engine: engine, grounding: grounding}, nil } func (e *Extractor) Key() string { return Key } func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot { return referenceSlots() } -func (e *Extractor) ManifestMetadata() map[string]any { - if e == nil { - return nil - } - return map[string]any{ - "prompt_id": PromptID, - "prompt_version": SchemaVersion, - "prompt_sha256": e.promptSHA, - "mapping_policy": mappingPolicy, - "scene_gate_policy": sceneGatePolicy, - "response_schema_key": string(ResponseSchemaKey), - "response_schema_id": ResponseSchemaID, - "response_schema_name": ResponseSchemaName, - "response_schema_version": SchemaVersion, - "response_schema_sha256": e.responseSchemaSHA, - } -} - -func (e *Extractor) CheckpointFingerprints() []pipeline.CheckpointFingerprint { - if e == nil { - return nil - } - return []pipeline.CheckpointFingerprint{ - {Name: "prompt", Value: e.promptSHA}, - {Name: "response_schema", Value: e.responseSchemaSHA}, - {Name: "mapping_policy", Value: mappingPolicy}, - {Name: "scene_gate_policy", Value: sceneGatePolicy}, - } -} - -func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.EnemyEventList], error) { +func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest[dnd.EnemyEventList]) (contracts.TypedExtractionResult[dnd.EnemyEventList], error) { if e == nil { return contracts.TypedExtractionResult[dnd.EnemyEventList]{}, extractorErrorf("extractor must not be nil") } - if e.llm == nil { - return contracts.TypedExtractionResult[dnd.EnemyEventList]{}, extractorErrorf("LLM client must not be nil") + if e.engine == nil { + return contracts.TypedExtractionResult[dnd.EnemyEventList]{}, extractorErrorf("native engine must not be nil") } if e.grounding == nil { return contracts.TypedExtractionResult[dnd.EnemyEventList]{}, extractorErrorf("grounding resolver must not be nil") @@ -126,10 +80,10 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRe switch match.State { case sceneregistry.MatchExact: if match.Kind != dnd.SceneKindCombat { - return emptyResult(), nil + return e.withoutModel(ctx, req, emptyResult()) } case sceneregistry.MatchMissing, sceneregistry.MatchMismatched: - return unavailableSceneResult(), nil + return e.withoutModel(ctx, req, unavailableSceneResult()) default: return contracts.TypedExtractionResult[dnd.EnemyEventList]{}, extractorErrorf("unsupported scene eligibility match state %q", match.State) } @@ -137,32 +91,38 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRe if err != nil { return contracts.TypedExtractionResult[dnd.EnemyEventList]{}, extractorErrorf("resolve enemy-event grounding: %w", err) } - inputs := shared.PromptInputs(sourceInput, req.References) - for name, input := range grounding.PromptInputs() { - inputs[name] = input + + order := shared.NewSourceRefOrder(req.Source) + sourceID := req.Source.ID + materials := shared.PromptInputs(sourceInput, req.References) + for name, material := range grounding.PromptInputs() { + materials[name] = material } - var response extractionResponse - completion, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{ - StageName: Key, - PromptID: PromptID, - PromptVersion: SchemaVersion, - ProfileID: req.LLMProfile, - SessionID: req.SessionID, - StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts, - Correction: req.Correction, - Inputs: inputs, - }, &response) + inputs := make(map[string]promptkit.ArtifactRef, len(materials)) + for name, material := range materials { + inputs[name] = promptkit.InlineWithURI(material.OriginURI, string(material.Content)) + } + initial := promptkit.RunRequest{PromptID: PromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID, Inputs: inputs} + value, outcome, err := pipeline.RunProducer(ctx, e.engine, req.Operation, initial, + func(_ context.Context, result *promptkit.RunResult) (*dnd.EnemyEventList, []string, error) { + var response extractionResponse + if err := json.Unmarshal([]byte(result.RawOutput), &response); err != nil { + return nil, nil, extractorErrorf("decode native output: %w", err) + } + value := canonicalEnemyEventList(response, order, sourceID) + return &value, nil, nil + }, + func(_ context.Context, guidance []string) (promptkit.RunRequest, error) { + return promptkit.RunRequest{PromptID: CorrectionPromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID, + Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline(strings.Join(guidance, "\n\n"))}}, nil + }, + func(context.Context, *dnd.EnemyEventList) (dnd.EnemyEventList, error) { + return emptyResult().Value, nil + }) if err != nil { - return contracts.TypedExtractionResult[dnd.EnemyEventList]{}, extractorErrorf("complete structured output: %w", err) + return contracts.TypedExtractionResult[dnd.EnemyEventList]{}, extractorErrorf("produce enemy events: %w", err) } - candidate, err := shared.ModelCandidateFromResponse(completion) - if err != nil { - return contracts.TypedExtractionResult[dnd.EnemyEventList]{}, extractorErrorf("capture model candidate: %w", err) - } - return contracts.TypedExtractionResult[dnd.EnemyEventList]{ - Value: canonicalEnemyEventList(response, shared.NewSourceRefOrder(req.Source), req.Source.ID), - ModelCandidate: candidate, - }, nil + return contracts.TypedExtractionResult[dnd.EnemyEventList]{Value: value, Outcome: outcome}, nil } func emptyResult() contracts.TypedExtractionResult[dnd.EnemyEventList] { @@ -181,16 +141,26 @@ func unavailableSceneResult() contracts.TypedExtractionResult[dnd.EnemyEventList return result } +// withoutModel assesses a deterministic skip without a native request or cache key. +func (e *Extractor) withoutModel(ctx context.Context, req contracts.TypedExtractionRequest[dnd.EnemyEventList], result contracts.TypedExtractionResult[dnd.EnemyEventList]) (contracts.TypedExtractionResult[dnd.EnemyEventList], error) { + value, outcome, err := pipeline.AssessProducer(ctx, req.Operation, result.Value, func(context.Context) (dnd.EnemyEventList, error) { return emptyResult().Value, nil }) + if err != nil { + return contracts.TypedExtractionResult[dnd.EnemyEventList]{}, extractorErrorf("assess skipped enemy extraction: %w", err) + } + result.Value, result.Outcome = value, outcome + return result, nil +} + func ModuleSpec() pipeline.ModuleSpec { return pipeline.ModuleSpec{ - Key: Key, - Stage: pipeline.StageExtract, - ExecutionClass: contracts.ExecutionClassLLMBacked, - CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1, - Requires: append([]string(nil), requiredCapabilities...), - Provides: append([]string(nil), providedCapabilities...), - ArtifactKind: dnd.EnemyEventListKind, - ReferenceSlots: referenceSlots(), + Key: Key, + Stage: pipeline.StageExtract, + ExecutionClass: contracts.ExecutionClassLLMBacked, + Prompts: []string{PromptID, CorrectionPromptID}, + Requires: append([]string(nil), requiredCapabilities...), + Provides: append([]string(nil), providedCapabilities...), + ArtifactKind: dnd.EnemyEventListKind, + ReferenceSlots: referenceSlots(), } } @@ -200,7 +170,7 @@ func Register(registry *pipeline.ExtractorRegistry) error { if err != nil { return nil, err } - return New(request.Dependencies.LLM, options, request.References) + return New(request.Dependencies.Engine, options, request.References) }) } diff --git a/internal/modules/dnd/extract/enemyevents/extractor_test.go b/internal/modules/dnd/extract/enemyevents/extractor_test.go index 5f77266e..147b4727 100644 --- a/internal/modules/dnd/extract/enemyevents/extractor_test.go +++ b/internal/modules/dnd/extract/enemyevents/extractor_test.go @@ -2,12 +2,13 @@ package enemyevents import ( "context" - "encoding/json" "errors" "reflect" "strings" "testing" + "go.maximumdirect.net/promptkit" + "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" @@ -46,14 +47,14 @@ func TestExtractMapsEnemyEventsInSourceOrder(t *testing.T) { t.Fatalf("LLM calls = %d, want 1", len(client.requests)) } request := client.requests[0] - if request.StageName != Key || request.PromptID != PromptID || request.PromptVersion != SchemaVersion || request.ProfileID != "enemy-profile" || request.SessionID != "session-123" { - t.Fatalf("LLM request identity = %#v", request) + if request.Prompt.SessionID != "session-123" { + t.Fatal("session selection lost") } - if got := string(request.Inputs[CombatTurnReferenceSlot].Content); !strings.Contains(got, `"actor":"Ashfang"`) || strings.Contains(got, "source_ref") { - t.Fatalf("combat grounding = %s", got) + if got := string(renderedInput(t, request, CombatTurnReferenceSlot)); !strings.Contains(got, `"actor":"Ashfang"`) || strings.Contains(got, "source_ref") { + t.Fatalf("combat grounding=%s", got) } - if got := string(request.Inputs[NPCOccurrenceReferenceSlot].Content); !strings.Contains(got, `"kind":"combat_opponent"`) || strings.Contains(got, "Aria") { - t.Fatalf("occurrence grounding = %s", got) + if got := string(renderedInput(t, request, NPCOccurrenceReferenceSlot)); !strings.Contains(got, `"kind":"combat_opponent"`) || strings.Contains(got, "Aria") { + t.Fatalf("opponent grounding=%s", got) } } @@ -93,6 +94,13 @@ func TestExtractSkipsModelForIneligibleScenes(t *testing.T) { if test.change != nil { test.change(request.Chunk) } + request.Operation.ReuseEligible = false + prepared, assessed := 0, 0 + request.Operation.Execution.OnPrepared = func(context.Context, promptkit.PreparedRun) { prepared++ } + request.Operation.Assess = func(context.Context, dnd.EnemyEventList) (contracts.CandidateAssessment, error) { + assessed++ + return contracts.CandidateAssessment{}, nil + } result, err := newEnemyExtractor(t, client).Extract(context.Background(), request) if err != nil { t.Fatal(err) @@ -100,6 +108,9 @@ func TestExtractSkipsModelForIneligibleScenes(t *testing.T) { if len(client.requests) != 0 || result.Value.Events == nil || len(result.Value.Events) != 0 { t.Fatalf("result = %#v, calls = %d", result, len(client.requests)) } + if result.Outcome.Disposition != contracts.ProducerAccepted || result.Outcome.ReuseEligible || prepared != 0 || assessed != 1 { + t.Fatalf("deterministic skip=%#v, prepared/assessed=%d/%d", result.Outcome, prepared, assessed) + } if test.wantWarning { if len(result.Diagnostics) != 1 || result.Diagnostics[0].ReasonCode != "scene_classification_unavailable" || result.Diagnostics[0].Disposition != contracts.DiagnosticDispositionWarning || result.Diagnostics[0].Category != contracts.DiagnosticCategoryDegradation || result.Diagnostics[0].Samples[0].Scope != SceneDescriptionReferenceSlot { t.Fatalf("diagnostics = %#v", result.Diagnostics) @@ -115,21 +126,21 @@ func TestExtractReportsRequiredGroundingAndProviderFailures(t *testing.T) { client := &fakeEnemyEventsLLMClient{} request := enemyExtractionRequest(t) request.References = withoutSlot(request.References, NPCRegistryReferenceSlot) - if _, err := (&Extractor{llm: client, grounding: mustGroundingResolver(t, request.References)}).Extract(context.Background(), request); err == nil || !strings.Contains(err.Error(), "NPC registry") { + if _, err := (&Extractor{engine: newEnemyExtractor(t, client).engine, grounding: mustGroundingResolver(t, request.References)}).Extract(context.Background(), request); err == nil || !strings.Contains(err.Error(), "NPC registry") { t.Fatalf("Extract() error = %v, want required grounding context", err) } provider := &fakeEnemyEventsLLMClient{err: errors.New("provider unavailable")} - if _, err := newEnemyExtractor(t, provider).Extract(context.Background(), enemyExtractionRequest(t)); err == nil || !strings.Contains(err.Error(), "complete structured output") || !strings.Contains(err.Error(), "provider unavailable") { + if _, err := newEnemyExtractor(t, provider).Extract(context.Background(), enemyExtractionRequest(t)); err == nil || !strings.Contains(err.Error(), "produce enemy events") || !strings.Contains(err.Error(), "provider unavailable") { t.Fatalf("Extract() error = %v, want provider context", err) } } -func TestConstructorSpecOptionsAndSafeMetadata(t *testing.T) { - if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "LLM client") { +func TestConstructorSpecOptionsAndNativeDependencies(t *testing.T) { + if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "native engine") { t.Fatalf("New(nil) error = %v", err) } - if _, err := New(&fakeEnemyEventsLLMClient{}, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one") { + if _, err := New(newEnemyExtractor(t, &fakeEnemyEventsLLMClient{}).engine, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one") { t.Fatalf("New() error = %v", err) } if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil || !strings.Contains(err.Error(), "unknown option") { @@ -140,7 +151,7 @@ func TestConstructorSpecOptionsAndSafeMetadata(t *testing.T) { first.Requires[0] = "changed" first.ReferenceSlots[0].AcceptedMediaTypes[0] = "changed" second := ModuleSpec() - if second.Requires[0] != "chunks" || second.ArtifactKind != dnd.EnemyEventListKind || second.ExecutionClass != contracts.ExecutionClassLLMBacked || second.CorrectionProtocol != contracts.CorrectionProtocolSingleResponseV1 || second.ReferenceSlots[0].AcceptedMediaTypes[0] == "changed" { + if second.Requires[0] != "chunks" || second.ArtifactKind != dnd.EnemyEventListKind || second.ExecutionClass != contracts.ExecutionClassLLMBacked || !reflect.DeepEqual(second.Prompts, []string{PromptID, CorrectionPromptID}) || second.ReferenceSlots[0].AcceptedMediaTypes[0] == "changed" { t.Fatalf("ModuleSpec() reused mutable state: %#v", second) } registry := pipeline.NewExtractorRegistry() @@ -151,82 +162,4 @@ func TestConstructorSpecOptionsAndSafeMetadata(t *testing.T) { t.Fatalf("registered spec = %#v, %t", spec, ok) } - extractor := newEnemyExtractor(t, &fakeEnemyEventsLLMClient{}, groundingReferences(t, "Ashfang", dnd.SceneKindCombat)) - metadata := extractor.ManifestMetadata() - encoded, err := json.Marshal(metadata) - if err != nil { - t.Fatal(err) - } - if strings.Contains(string(encoded), "Ashfang") || metadata["mapping_policy"] != mappingPolicy || metadata["scene_gate_policy"] != sceneGatePolicy { - t.Fatalf("unsafe or incomplete metadata = %s", encoded) - } - fingerprints := extractor.CheckpointFingerprints() - if len(fingerprints) != 4 || fingerprints[0].Value != metadata["prompt_sha256"] || fingerprints[1].Value != metadata["response_schema_sha256"] || fingerprints[2].Value != mappingPolicy || fingerprints[3].Value != sceneGatePolicy { - t.Fatalf("fingerprints = %#v", fingerprints) - } -} - -func newEnemyExtractor(t *testing.T, client contracts.StructuredLLMClient, references ...contracts.ReferenceSet) *Extractor { - t.Helper() - if len(references) == 0 { - references = []contracts.ReferenceSet{groundingReferences(t, "Ashfang", dnd.SceneKindCombat)} - } - extractor, err := New(client, Options{}, references...) - if err != nil { - t.Fatal(err) - } - return extractor -} - -func mustGroundingResolver(t *testing.T, references contracts.ReferenceSet) *groundingResolver { - t.Helper() - resolver, err := newGroundingResolver(references) - if err != nil { - t.Fatal(err) - } - return resolver -} - -func enemyExtractionRequest(t *testing.T) contracts.TypedExtractionRequest { - t.Helper() - chunk := combatChunk() - chunk.Units = []source.SourceUnit{{ID: 1}, {ID: 2}, {ID: 3}, {ID: 4}} - chunk.MediaType = "application/json" - chunk.Content = []byte(`{"id":"combat-scene","units":[1,2,3,4]}`) - return contracts.TypedExtractionRequest{ - Source: &source.SourceDocument{ID: "combat-session", Units: append([]source.SourceUnit(nil), chunk.Units...)}, - Chunk: chunk, - SourceInput: contracts.NewLLMInputMaterial("source", "application/json", chunk.Content, digest(chunk.Content), "file:///combat-session.json"), - References: groundingReferences(t, "Ashfang", dnd.SceneKindCombat), - LLMProfile: "enemy-profile", - SessionID: "session-123", - } -} - -type fakeEnemyEventsLLMClient struct { - response extractionResponse - err error - requests []contracts.StructuredCompletionRequest -} - -func (client *fakeEnemyEventsLLMClient) CompleteStructured(_ context.Context, request contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { - client.requests = append(client.requests, cloneEnemyRequest(request)) - if client.err != nil { - return contracts.StructuredCompletionResponse{}, client.err - } - target, ok := out.(*extractionResponse) - if !ok { - return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target") - } - *target = client.response - content, err := json.Marshal(client.response) - if err != nil { - return contracts.StructuredCompletionResponse{}, err - } - return contracts.StructuredCompletionResponse{Content: content}, nil -} - -func cloneEnemyRequest(request contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest { - request.Inputs = request.Inputs.Clone() - return request } diff --git a/internal/modules/dnd/extract/enemyevents/native_execution_test.go b/internal/modules/dnd/extract/enemyevents/native_execution_test.go new file mode 100644 index 00000000..587e6845 --- /dev/null +++ b/internal/modules/dnd/extract/enemyevents/native_execution_test.go @@ -0,0 +1,199 @@ +package enemyevents + +import ( + "context" + "errors" + "reflect" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + combatcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/combatturns" + eventcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/enemyevents" + occurrencecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcoccurrences" + npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcregistry" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/enemyevents/engagements" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/enemyevents/shape" + sourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/enemyevents/source_refs" + relatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/enemyevents/source_relatedness" + "go.maximumdirect.net/promptkit" +) + +const duplicateEngagements = `{"events":[{"name":"Ashfang","kind":"engaged","source_refs":[{"start_unit_id":1,"end_unit_id":1}]},{"name":"ASHFANG","kind":"engaged","source_refs":[{"start_unit_id":2,"end_unit_id":2}]}]}` +const invalidEventCitation = `{"events":[{"name":"Ashfang","kind":"engaged","source_refs":[{"start_unit_id":999,"end_unit_id":999}]}]}` +const knownEvents = `{"events":[{"name":"Ashfang","kind":"engaged","source_refs":[{"start_unit_id":2,"end_unit_id":1}]},{"name":"Orcs","kind":"fled","source_refs":[{"start_unit_id":4,"end_unit_id":4}]}]}` + +func bindEnemyAssessment(request *contracts.TypedExtractionRequest[dnd.EnemyEventList]) *[]string { + validators := []contracts.TypedValidator[dnd.EnemyEventList]{ + shape.New(shape.Options{}), sourcerefs.New(sourcerefs.Options{}), + relatedness.New(relatedness.Options{}), engagements.New(engagements.Options{}), + } + var calls []string + request.Operation.Assess = func(ctx context.Context, value dnd.EnemyEventList) (contracts.CandidateAssessment, error) { + var assessment contracts.CandidateAssessment + for _, validator := range validators { + calls = append(calls, validator.Name()) + result, err := validator.Validate(ctx, contracts.TypedValidationRequest[dnd.EnemyEventList]{ + Stage: "extract", Source: request.Source, Chunk: request.Chunk, Value: value, Execution: request.Operation.Execution, + }) + if err != nil { + return contracts.CandidateAssessment{}, err + } + decision := contracts.AssessmentRejected + if result.Approved { + decision = contracts.AssessmentApproved + } + assessment.Validators = append(assessment.Validators, contracts.ValidatorAssessment{ValidatorName: validator.Name(), Decision: decision, Result: result}) + } + return assessment, nil + } + return &calls +} + +func TestEnemyNativeCorrectionKeepsAllGroundingFixedAndSourceFree(t *testing.T) { + for _, first := range []string{duplicateEngagements, invalidEventCitation} { + request := enemyExtractionRequest(t) + request.Operation.Controls.MaxAdditionalTurns = 1 + request.Operation.ReuseEligible = false + request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "enemy-override"} + var prepared []promptkit.PreparedRun + request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = append(prepared, value) } + calls := bindEnemyAssessment(&request) + client := &fakeEnemyEventsLLMClient{responses: []string{first, knownEvents}} + client.onGenerate = func(context.Context, promptkit.GenerateRequest) { + if len(client.requests) == 1 { + for _, name := range []string{NPCRegistryReferenceSlot, SceneDescriptionReferenceSlot, CombatTurnReferenceSlot, NPCOccurrenceReferenceSlot} { + request.References.Slots[name].Items[0].Content[0] = '!' + } + } + } + result, err := newEnemyExtractor(t, client, request.References).Extract(context.Background(), request) + chain := []string{shape.Key, sourcerefs.Key, relatedness.Key, engagements.Key} + if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || result.Outcome.ReuseEligible || + !reflect.DeepEqual(*calls, append(append([]string(nil), chain...), chain...)) || len(result.Value.Events) != 2 || len(client.requests) != 2 || len(prepared) != 2 { + t.Fatalf("enemy correction=%#v %v, validators=%#v native=%d/%d", result, err, *calls, len(client.requests), len(prepared)) + } + if result.Value.Events[0].SourceRefs[0].StartUnitID != 1 || result.Value.Events[0].SourceRefs[0].EndUnitID != 2 || result.Value.Events[1].Name != "Orcs" || result.Value.Events[1].Kind != dnd.EnemyEventKindFled { + t.Fatalf("contextual names/evidence=%#v", result.Value) + } + for i, call := range client.requests { + id := PromptID + if i > 0 { + id = CorrectionPromptID + } + if prepared[i].PromptID != id || prepared[i].SelectedProfileID != request.LLMProfile || call.Target.Model != "enemy-override" || call.Prompt.SessionID != request.SessionID { + t.Fatal("native selection lost") + } + for _, name := range []string{NPCRegistryReferenceSlot, CombatTurnReferenceSlot, NPCOccurrenceReferenceSlot} { + material := string(renderedInput(t, call, name)) + if !strings.Contains(material, "Ashfang") { + t.Fatalf("missing %s grounding", name) + } + for _, forbidden := range []string{"source_ref", "source_id", "start_unit_id", "npc:sha256:", "combat-session", "summary"} { + if strings.Contains(material, forbidden) { + t.Fatalf("%s exposed %q", name, forbidden) + } + } + } + } + initial, revised := client.requests[0].Prompt.Messages, client.requests[1].Prompt.Messages + if len(revised) != len(initial)+2 || !reflect.DeepEqual(initial, revised[:len(initial)]) || revised[len(initial)].Role != "assistant" || revised[len(initial)].Content != first { + t.Fatal("native history changed fixed grounding") + } + guidance := revised[len(initial)+1].Content + if first == duplicateEngagements && !strings.Contains(guidance, "at most one") { + t.Fatal("engagement guidance lost") + } + if first == invalidEventCitation && !strings.Contains(guidance, "source range endpoints") { + t.Fatal("source guidance lost") + } + content, err := eventcodec.New().Encode(result.Value) + if err != nil { + t.Fatal(err) + } + decoded, err := eventcodec.New().Decode(content) + if err != nil || !reflect.DeepEqual(result.Value, decoded) { + t.Fatalf("event wire round trip=%#v %v", decoded, err) + } + } +} + +func TestEnemyTerminalModesAndCancellation(t *testing.T) { + failure := errors.New("provider unavailable") + for _, test := range []struct { + name string + mode contracts.ExecutionMode + response string + failures []error + assessments, runs int + wantError bool + }{ + {"strict engagement rejection", contracts.ModeStrict, duplicateEngagements, nil, 2, 2, true}, + {"permissive engagement rejection", contracts.ModePermissive, duplicateEngagements, nil, 2, 2, false}, + {"initial provider failure", contracts.ModePermissive, "", []error{failure}, 0, 1, false}, + {"failed continuation", contracts.ModePermissive, invalidEventCitation, []error{nil, failure}, 1, 2, false}, + {"structural failure", contracts.ModePermissive, `{"events":null}`, nil, 0, 1, false}, + } { + t.Run(test.name, func(t *testing.T) { + request := enemyExtractionRequest(t) + request.Operation.Controls.Mode = test.mode + request.Operation.Controls.MaxAdditionalTurns = 1 + calls := bindEnemyAssessment(&request) + runs := 0 + request.Operation.Execution.OnPrepared = func(context.Context, promptkit.PreparedRun) { runs++ } + client := &fakeEnemyEventsLLMClient{raw: test.response, errors: test.failures} + result, err := newEnemyExtractor(t, client, request.References).Extract(context.Background(), request) + if (err != nil) != test.wantError || len(*calls) != test.assessments*4 || runs != test.runs { + t.Fatalf("terminal=%#v %v, validators=%#v runs=%d", result, err, *calls, runs) + } + if test.wantError { + if result.Outcome.Disposition != "" { + t.Fatal("strict failure published output") + } + return + } + if result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || result.Value.Events == nil || len(result.Value.Events) != 0 { + t.Fatalf("fallback invented events=%#v", result) + } + if _, err := eventcodec.New().Encode(result.Value); err != nil { + t.Fatal(err) + } + }) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + request := enemyExtractionRequest(t) + request.Operation.Controls.Mode = contracts.ModePermissive + client := &fakeEnemyEventsLLMClient{raw: knownEvents, onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }} + result, err := newEnemyExtractor(t, client).Extract(ctx, request) + if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" { + t.Fatalf("canceled outcome=%#v %v", result, err) + } +} + +func TestEnemyAcceptsEmptyFallbackReferencesWithoutInventingEvents(t *testing.T) { + request := enemyExtractionRequest(t) + npc, err := npccodec.New().Encode(dnd.NPCRegistry{NPCs: []dnd.NPC{}}) + if err != nil { + t.Fatal(err) + } + turns, err := combatcodec.New().Encode(dnd.CombatTurnList{CombatTurns: []dnd.CombatTurn{}}) + if err != nil { + t.Fatal(err) + } + occurrences, err := occurrencecodec.New().Encode(dnd.NPCOccurrenceList{Occurrences: []dnd.NPCOccurrence{}}) + if err != nil { + t.Fatal(err) + } + for name, content := range map[string][]byte{NPCRegistryReferenceSlot: npc, CombatTurnReferenceSlot: turns, NPCOccurrenceReferenceSlot: occurrences} { + request.References = replaceItem(request.References, name, newReferenceItem(name, content)) + } + request.Operation.ReuseEligible = false + bindEnemyAssessment(&request) + client := &fakeEnemyEventsLLMClient{raw: `{"events":[]}`} + result, err := newEnemyExtractor(t, client, contracts.ReferenceSet{}).Extract(context.Background(), request) + if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || result.Outcome.ReuseEligible || result.Value.Events == nil || len(result.Value.Events) != 0 || len(client.requests) != 1 { + t.Fatalf("empty reference handoff=%#v %v", result, err) + } +} diff --git a/internal/modules/dnd/extract/enemyevents/pipeline_test.go b/internal/modules/dnd/extract/enemyevents/pipeline_test.go new file mode 100644 index 00000000..2f5dd9df --- /dev/null +++ b/internal/modules/dnd/extract/enemyevents/pipeline_test.go @@ -0,0 +1,120 @@ +package enemyevents + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/core/source" + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + eventcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/enemyevents" + scenecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/scenedescriptions" + "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/chunk/units" + "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder" + "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop" + "gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript" +) + +type enemyPipelineOutput struct{} + +func (enemyPipelineOutput) Key() string { return "test/enemy-output" } +func (enemyPipelineOutput) Encode(context.Context, contracts.OutputRequest) (contracts.OutputResult, error) { + return contracts.OutputResult{}, nil +} + +func TestNativeEnemyPipelineRetainsIntrinsicCodecBoundaryWithEmptyValidators(t *testing.T) { + for _, name := range []string{"Ashfang", " "} { + t.Run(name, func(t *testing.T) { + client := &fakeEnemyEventsLLMClient{response: extractionResponse{Events: []enemyEventResponse{ + {Name: name, Kind: "engaged", SourceRefs: []enemySourceRefResponse{{StartUnitID: 1, EndUnitID: 1}}}, + }}} + engine := newEnemyExtractor(t, client).engine + registries := pipeline.Registries{ + Inputs: pipeline.NewInputAdapterRegistry(), Chunkers: pipeline.NewChunkerRegistry(), + ArtifactCodecs: pipeline.NewArtifactCodecRegistry(), Extractors: pipeline.NewExtractorRegistry(), + Mergers: pipeline.NewMergerRegistry(), Normalizers: pipeline.NewNormalizerRegistry(), + Validators: pipeline.NewValidatorRegistry(), ValidatorChains: pipeline.NewValidatorChainRegistry(), + Outputs: pipeline.NewOutputEncoderRegistry(), + } + check := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + check(transcript.Register(registries.Inputs)) + check(units.Register(registries.Chunkers)) + check(pipeline.RegisterArtifactCodec(registries.ArtifactCodecs, eventcodec.New())) + check(Register(registries.Extractors)) + check(appendorder.RegisterTyped(registries.Mergers, dnd.EnemyEventListKind, func(values []dnd.EnemyEventList) (dnd.EnemyEventList, error) { + result := dnd.EnemyEventList{Events: []dnd.EnemyEvent{}} + for _, value := range values { + result.Events = append(result.Events, value.Events...) + } + return result, nil + })) + check(noop.RegisterTyped[dnd.EnemyEventList](registries.Normalizers, dnd.EnemyEventListKind)) + check(registries.Outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "test/enemy-output", Stage: pipeline.StageOutput, ExecutionClass: contracts.ExecutionClassDeterministic}, + func() (contracts.OutputEncoder, error) { return enemyPipelineOutput{}, nil })) + references := groundingReferences(t, "Ashfang", dnd.SceneKindCombat) + content, err := scenecodec.New().Encode(dnd.SceneDescriptionList{Scenes: []dnd.SceneDescription{{ + ID: "chunk-000001", SourceRef: source.SourceRef{SourceID: "combat-session", StartUnitID: 1, EndUnitID: 1}, + Kind: dnd.SceneKindCombat, Title: "Combat", Summary: "Ashfang opposes the party.", + }}}) + check(err) + references = replaceItem(references, SceneDescriptionReferenceSlot, newReferenceItem(SceneDescriptionReferenceSlot, content)) + referenceSources := make(map[string]pipeline.ReferenceSource) + root := t.TempDir() + for name, slot := range references.Slots { + path := filepath.Join(root, name+".json") + check(os.WriteFile(path, slot.Items[0].Content, 0600)) + referenceSources[name] = pipeline.ExternalReference(path) + } + profile := pipeline.PipelineProfile{ + ID: "native-enemies", Input: pipeline.Binding(transcript.Key), Chunk: pipeline.Binding(units.Key), Output: pipeline.Binding("test/enemy-output"), + LLMProfile: "enemy-profile", + Artifacts: map[string]pipeline.ArtifactLaneProfile{"enemies": { + Extract: pipeline.ModuleBinding{Module: Key, References: referenceSources, Validators: pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{}}}, + Merge: pipeline.Binding(appendorder.Key), Normalize: pipeline.Binding(noop.Key), + }}, + } + catalog := pipeline.ModuleCatalog{ + Inputs: registries.Inputs, Chunkers: registries.Chunkers, ArtifactCodecs: registries.ArtifactCodecs, + Extractors: registries.Extractors, Mergers: registries.Mergers, Normalizers: registries.Normalizers, + Validators: registries.Validators, ValidatorChains: registries.ValidatorChains, Outputs: registries.Outputs, + } + resolved, err := pipeline.ResolvePipeline(profile, pipeline.ResolveOptions{}, catalog) + check(err) + resolved, _, err = pipeline.MaterializeReferences(resolved, catalog, pipeline.ReferenceMaterializationOptions{}) + check(err) + prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{Engine: engine}) + check(err) + result, err := pipeline.New().Run(context.Background(), pipeline.RunInput{ + Prepared: prepared, SourceID: "combat-session", ExtractWorkers: 1, + RawInput: []byte(`{"segments":[{"id":1,"speaker":"GM","start":0,"end":1,"text":"Ashfang attacks the party."}]}`), + }) + if len(client.requests) != 1 { + t.Fatalf("native calls=%d", len(client.requests)) + } + if strings.TrimSpace(name) == "" { + if err == nil || !strings.Contains(err.Error(), "serialize accepted extract output") || !strings.Contains(err.Error(), "name must not be empty") { + t.Fatalf("intrinsic codec error=%v", err) + } + return + } + check(err) + if result.FallbackCount != 0 || len(result.NormalizeOutputs) != 1 { + t.Fatalf("assembled outcome=%#v", result) + } + value, err := eventcodec.New().Decode(result.NormalizeOutputs[0].Artifact.Content) + check(err) + if len(value.Events) != 1 || value.Events[0].Name != name || value.Events[0].SourceRefs[0].SourceID != "combat-session" { + t.Fatalf("durable events=%#v", value) + } + }) + } +} diff --git a/internal/modules/dnd/extract/enemyevents/prompt_assets.go b/internal/modules/dnd/extract/enemyevents/prompt_assets.go deleted file mode 100644 index befd34b1..00000000 --- a/internal/modules/dnd/extract/enemyevents/prompt_assets.go +++ /dev/null @@ -1,76 +0,0 @@ -package enemyevents - -import ( - "fmt" - "io/fs" - "sync" - - rootassets "gitea.maximumdirect.net/eric/notarius/assets" - "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - "gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs" - "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" -) - -const promptAssetRoot = "assets/prompts" - -var promptAssetManifest = shared.PromptAssetManifest{ - ModuleDir: "dnd.enemy_events", - ModuleFiles: []promptfs.ModulePromptFile{ - {Name: "prompt.yaml", Path: "prompts/prompt.yaml"}, - {Name: "combat-grounding.md", Path: "prompts/combat-grounding.md"}, - {Name: "instructions.md", Path: "prompts/instructions.md"}, - }, - SharedFiles: []string{ - "common-dnd-system.md", - "common-dnd-extraction-evidence.md", - "common-dnd-identity.md", - "common-dnd-transcript-chunk.md", - "common-dnd-references.md", - "common-dnd-npc-registry.md", - }, -} - -func moduleAssetFS() (fs.FS, error) { - assets, err := fs.Sub(rootassets.FS(), "dnd/enemy-events") - if err != nil { - return nil, fmt.Errorf("scope enemy-event assets: %w", err) - } - return assets, nil -} - -func RegisterPromptAssets(registry *llm.AssetRegistry) error { - assets, err := moduleAssetFS() - if err != nil { - return err - } - promptFS, err := promptAssetManifest.PromptFS(assets) - if err != nil { - return fmt.Errorf("prepare enemy-event prompt assets: %w", err) - } - if err := registry.RegisterPromptFS(promptFS, promptAssetRoot); err != nil { - return err - } - schemas, err := fs.Sub(assets, "schemas") - if err != nil { - return fmt.Errorf("scope enemy-event schemas: %w", err) - } - return registry.RegisterSchemaFS(schemas, ".") -} - -func promptAssetMetadata() (string, error) { - promptAssetHashOnce.Do(func() { - assets, err := moduleAssetFS() - if err != nil { - promptAssetHashErr = err - return - } - promptAssetHash, promptAssetHashErr = promptAssetManifest.Hash(assets) - }) - return promptAssetHash, promptAssetHashErr -} - -var ( - promptAssetHashOnce sync.Once - promptAssetHash string - promptAssetHashErr error -) diff --git a/internal/modules/dnd/extract/enemyevents/prompt_assets_test.go b/internal/modules/dnd/extract/enemyevents/prompt_assets_test.go deleted file mode 100644 index 7bd0172c..00000000 --- a/internal/modules/dnd/extract/enemyevents/prompt_assets_test.go +++ /dev/null @@ -1,82 +0,0 @@ -package enemyevents - -import ( - "context" - "io/fs" - "strings" - "testing" - "time" - - "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - "gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture" - "gitea.maximumdirect.net/eric/promptkit" -) - -func TestRegisterPromptAssetsAndPrepareEnemyEventPrompt(t *testing.T) { - registry := llm.NewAssetRegistry() - if err := RegisterPromptAssets(registry); err != nil { - t.Fatal(err) - } - schemaFS, err := registry.SchemaFS() - if err != nil { - t.Fatal(err) - } - if _, err := fs.ReadFile(schemaFS, "dnd_enemy_events_llm.v1.json"); err != nil { - t.Fatalf("response schema asset: %v", err) - } - - prepared := prepareEnemyEventPrompt(t) - if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_enemy_events_llm.v1.json" || prepared.SelectedProfileID != "dnd-extraction" { - t.Fatalf("prepared prompt = %#v", prepared) - } -} - -func TestEnemyEventPromptRequiresGroundingInputs(t *testing.T) { - engine := newEnemyEventPromptEngine(t) - for _, inputName := range []string{"npc_registry", "combat_turns", "npc_occurrences"} { - t.Run(inputName, func(t *testing.T) { - inputs := enemyEventPromptInputs() - delete(inputs, inputName) - _, err := engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: PromptID, PromptVersion: SchemaVersion, Inputs: inputs, - }) - if err == nil || !strings.Contains(err.Error(), inputName) { - t.Fatalf("Prepare() error = %v, want required %q input", err, inputName) - } - }) - } -} - -func prepareEnemyEventPrompt(t *testing.T) *promptkit.PreparedRun { - t.Helper() - prepared, err := newEnemyEventPromptEngine(t).Prepare(context.Background(), promptkit.RunRequest{ - PromptID: PromptID, PromptVersion: SchemaVersion, Inputs: enemyEventPromptInputs(), - }) - if err != nil { - t.Fatal(err) - } - return prepared -} - -func newEnemyEventPromptEngine(t *testing.T) *promptkit.Engine { - t.Helper() - registry := llm.NewAssetRegistry() - if err := RegisterPromptAssets(registry); err != nil { - t.Fatal(err) - } - return promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{ - "dnd-extraction": promptkitfixture.Profile("http://127.0.0.1:1/v1", "enemy-test-model"), - }) -} - -func enemyEventPromptInputs() map[string]promptkit.ArtifactRef { - return map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline(`{"value":"enemy-transcript"}`), - "players": promptkit.Inline("enemy-player"), - "party": promptkit.Inline("enemy-party"), - "glossary": promptkit.Inline("enemy-glossary"), - "npc_registry": promptkit.Inline(`{"npcs":[{"name":"enemy-npc"}]}`), - "combat_turns": promptkit.Inline(`{"combat_turns":[{"actor":"enemy-turn","turn_kind":"turn"}]}`), - "npc_occurrences": promptkit.Inline(`{"npc_occurrences":[{"name":"enemy-opponent","kind":"combat_opponent"}]}`), - } -} diff --git a/internal/modules/dnd/extract/enemyevents/schema.go b/internal/modules/dnd/extract/enemyevents/schema.go deleted file mode 100644 index 211a65e0..00000000 --- a/internal/modules/dnd/extract/enemyevents/schema.go +++ /dev/null @@ -1,25 +0,0 @@ -package enemyevents - -import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - -const ( - PromptID = "dnd.enemy_events" - ResponseSchemaKey = llm.ResponseSchemaKey("dnd_enemy_events_llm") - ResponseSchemaID = "notarius.dnd.enemy_events.llm" - ResponseSchemaName = "notarius_dnd_enemy_events_llm_v1" - SchemaVersion = "v1" -) - -func loadResponseSchema() (llm.ResponseSchema, error) { - assets, err := moduleAssetFS() - if err != nil { - return llm.ResponseSchema{}, err - } - return llm.LoadResponseSchema(assets, llm.ResponseSchemaDefinition{ - Key: ResponseSchemaKey, - ID: ResponseSchemaID, - Version: SchemaVersion, - Name: ResponseSchemaName, - AssetPath: "schemas/dnd_enemy_events_llm.v1.json", - }) -} diff --git a/internal/modules/dnd/extract/enemyevents/schema_test.go b/internal/modules/dnd/extract/enemyevents/schema_test.go deleted file mode 100644 index 9f093eb8..00000000 --- a/internal/modules/dnd/extract/enemyevents/schema_test.go +++ /dev/null @@ -1,117 +0,0 @@ -package enemyevents - -import ( - "bytes" - "encoding/json" - "strings" - "testing" - - "github.com/santhosh-tekuri/jsonschema/v6" - - "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" -) - -func TestResponseSchemaDefinesPrivateStructuralBoundary(t *testing.T) { - schema, err := loadResponseSchema() - if err != nil { - t.Fatal(err) - } - if schema.Key != ResponseSchemaKey || schema.ID != ResponseSchemaID || schema.Name != ResponseSchemaName || schema.Version != SchemaVersion || !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) { - t.Fatalf("schema = %#v", schema) - } - valid := validEnemyResponse() - content, err := json.Marshal(valid) - if err != nil { - t.Fatal(err) - } - if err := validateEnemySchema(content, schema.JSONSchema); err != nil { - t.Fatalf("valid response rejected: %v", err) - } - semantic := validEnemyResponse() - event := semantic["events"].([]any)[0].(map[string]any) - event["name"] = "" - ref := event["source_refs"].([]any)[0].(map[string]any) - ref["start_unit_id"] = 0 - ref["end_unit_id"] = -1 - content, err = json.Marshal(semantic) - if err != nil || validateEnemySchema(content, schema.JSONSchema) != nil { - t.Fatalf("non-categorical validator-owned semantics were rejected: %v", err) - } -} - -func TestResponseSchemaRejectsInvalidStructure(t *testing.T) { - schema, err := loadResponseSchema() - if err != nil { - t.Fatal(err) - } - for _, mutate := range []func(map[string]any){ - func(event map[string]any) { delete(event, "name") }, - func(event map[string]any) { event["kind"] = 1 }, - func(event map[string]any) { event["kind"] = "unsupported" }, - func(event map[string]any) { event["unexpected"] = true }, - func(event map[string]any) { event["source_refs"].([]any)[0].(map[string]any)["source_id"] = "session" }, - } { - candidate := validEnemyResponse() - mutate(candidate["events"].([]any)[0].(map[string]any)) - content, err := json.Marshal(candidate) - if err != nil { - t.Fatal(err) - } - if err := validateEnemySchema(content, schema.JSONSchema); err == nil { - t.Fatal("private schema accepted structurally invalid response") - } - } - first, err := loadResponseSchema() - if err != nil { - t.Fatal(err) - } - first.JSONSchema[0] = '[' - second, err := loadResponseSchema() - if err != nil || !json.Valid(second.JSONSchema) || bytes.Equal(first.JSONSchema, second.JSONSchema) { - t.Fatalf("schema defensive copy = %s, %v", second.JSONSchema, err) - } -} - -func TestResponseSchemaAcceptsEverySupportedKind(t *testing.T) { - schema, err := loadResponseSchema() - if err != nil { - t.Fatal(err) - } - for _, kind := range []string{string(dnd.EnemyEventKindEngaged), string(dnd.EnemyEventKindKilled), string(dnd.EnemyEventKindFled), string(dnd.EnemyEventKindCaptured), string(dnd.EnemyEventKindIncapacitated)} { - candidate := validEnemyResponse() - candidate["events"].([]any)[0].(map[string]any)["kind"] = kind - content, err := json.Marshal(candidate) - if err != nil { - t.Fatal(err) - } - if err := validateEnemySchema(content, schema.JSONSchema); err != nil { - t.Fatalf("supported enemy-event kind %q was rejected: %v", kind, err) - } - } -} - -func validEnemyResponse() map[string]any { - return map[string]any{"events": []any{map[string]any{ - "name": "Ashfang", "kind": "engaged", "source_refs": []any{map[string]any{"start_unit_id": 1, "end_unit_id": 1}}, - }}} -} - -func validateEnemySchema(instanceContent, schemaContent []byte) error { - instance, err := jsonschema.UnmarshalJSON(bytes.NewReader(instanceContent)) - if err != nil { - return err - } - document, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaContent)) - if err != nil { - return err - } - compiler := jsonschema.NewCompiler() - if err := compiler.AddResource("schema.json", document); err != nil { - return err - } - schema, err := compiler.Compile("schema.json") - if err != nil { - return err - } - return schema.Validate(instance) -} diff --git a/internal/modules/dnd/extract/enemyevents/test_helpers_test.go b/internal/modules/dnd/extract/enemyevents/test_helpers_test.go new file mode 100644 index 00000000..548bc787 --- /dev/null +++ b/internal/modules/dnd/extract/enemyevents/test_helpers_test.go @@ -0,0 +1,134 @@ +package enemyevents + +import ( + "context" + "encoding/json" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/assets" + "gitea.maximumdirect.net/eric/notarius/internal/core/source" + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + "go.maximumdirect.net/configkit" + "go.maximumdirect.net/promptkit" +) + +func newEnemyExtractor(t *testing.T, client promptkit.LLMClient, references ...contracts.ReferenceSet) *Extractor { + t.Helper() + if len(references) == 0 { + references = []contracts.ReferenceSet{groundingReferences(t, "Ashfang", dnd.SceneKindCombat)} + } + type settings struct { + PromptKit promptkit.Settings `yaml:"promptkit"` + } + definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit }) + filesystem := configkit.NewFilesystem("Notarius assets", assets.FS()) + result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("occurrence fixture", []byte(`promptkit: + profiles: + - id: enemy-profile + endpoint: https://example.invalid/v1 + model: occurrence-test-model + api_key_required: false +`))}, configkit.LayeredOptions[settings]{Definition: definition, Underlays: []configkit.Underlay{ + promptkit.BundledOpenRouterUnderlay(), + {Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}}, + }}) + if err != nil { + t.Fatal(err) + } + engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client)) + if err != nil { + t.Fatal(err) + } + extractor, err := New(engine, Options{}, references...) + if err != nil { + t.Fatal(err) + } + return extractor +} + +func renderedInput(t *testing.T, request promptkit.GenerateRequest, name string) []byte { + t.Helper() + key := map[string]string{"transcript": "units", "npc_registry": "npcs", "combat_turns": "combat_turns", "npc_occurrences": "npc_occurrences"}[name] + for _, message := range request.Prompt.Messages { + for offset := 0; offset < len(message.Content); offset++ { + if message.Content[offset] != '{' { + continue + } + decoder := json.NewDecoder(strings.NewReader(message.Content[offset:])) + var fields map[string]json.RawMessage + if decoder.Decode(&fields) == nil && fields[key] != nil { + return []byte(message.Content[offset : offset+int(decoder.InputOffset())]) + } + } + } + t.Fatalf("missing rendered %s input", name) + return nil +} + +type fakeEnemyEventsLLMClient struct { + response extractionResponse + responses []string + raw string + err error + errors []error + requests []promptkit.GenerateRequest + onGenerate func(context.Context, promptkit.GenerateRequest) +} + +func (client *fakeEnemyEventsLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + index := len(client.requests) + client.requests = append(client.requests, request) + if client.onGenerate != nil { + client.onGenerate(ctx, request) + } + if err := ctx.Err(); err != nil { + return nil, err + } + if client.err != nil { + return nil, client.err + } + if index < len(client.errors) && client.errors[index] != nil { + return nil, client.errors[index] + } + content := client.raw + if index < len(client.responses) { + content = client.responses[index] + } + if content == "" { + encoded, err := json.Marshal(client.response) + if err != nil { + return nil, err + } + content = string(encoded) + } + return &promptkit.GenerateResponse{Content: content}, nil +} +func mustGroundingResolver(t *testing.T, references contracts.ReferenceSet) *groundingResolver { + t.Helper() + resolver, err := newGroundingResolver(references) + if err != nil { + t.Fatal(err) + } + return resolver +} + +func enemyExtractionRequest(t *testing.T) contracts.TypedExtractionRequest[dnd.EnemyEventList] { + t.Helper() + chunk := combatChunk() + chunk.Units = []source.SourceUnit{{ID: 1}, {ID: 2}, {ID: 3}, {ID: 4}} + chunk.MediaType = "application/json" + chunk.Content = []byte(`{"id":"combat-scene","units":[1,2,3,4]}`) + return contracts.TypedExtractionRequest[dnd.EnemyEventList]{ + Source: &source.SourceDocument{ID: "combat-session", Units: append([]source.SourceUnit(nil), chunk.Units...)}, + Chunk: chunk, + SourceInput: contracts.NewLLMInputMaterial("source", "application/json", chunk.Content, digest(chunk.Content), "file:///combat-session.json"), + References: groundingReferences(t, "Ashfang", dnd.SceneKindCombat), + LLMProfile: "enemy-profile", + SessionID: "session-123", + Operation: contracts.ProducerOperation[dnd.EnemyEventList]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, Cache: contracts.CacheBypass}, ReuseEligible: true, Assess: func(context.Context, dnd.EnemyEventList) (contracts.CandidateAssessment, error) { + return contracts.CandidateAssessment{}, nil + }}, + } +}