From 52a1c96327a6f2d531f66b0a94e5269337b78a49 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 22 Nov 2026 00:34:33 +0000 Subject: [PATCH] Migrate scene planning to native continuation and whole-source fallback --- assets/dnd/scenes/prompts/correction.md | 8 + assets/dnd/scenes/prompts/prompt.yaml | 16 +- assets/scenes_test.go | 184 +++++++++++++++ docs/roadmap/implementation.md | 23 ++ .../pipeline/chunk_plan_operation_test.go | 116 +++++++++ internal/modules/dnd/chunk/scenes/chunker.go | 110 +++++---- .../modules/dnd/chunk/scenes/chunker_test.go | 223 ++++++++---------- .../modules/dnd/chunk/scenes/model_test.go | 21 ++ .../dnd/chunk/scenes/native_execution_test.go | 205 ++++++++++++++++ .../modules/dnd/chunk/scenes/prompt_assets.go | 72 ------ .../dnd/chunk/scenes/prompt_assets_test.go | 138 ----------- internal/modules/dnd/chunk/scenes/schema.go | 25 -- .../modules/dnd/chunk/scenes/schema_test.go | 179 -------------- 13 files changed, 721 insertions(+), 599 deletions(-) create mode 100644 assets/dnd/scenes/prompts/correction.md create mode 100644 assets/scenes_test.go create mode 100644 internal/framework/pipeline/chunk_plan_operation_test.go create mode 100644 internal/modules/dnd/chunk/scenes/model_test.go create mode 100644 internal/modules/dnd/chunk/scenes/native_execution_test.go delete mode 100644 internal/modules/dnd/chunk/scenes/prompt_assets.go delete mode 100644 internal/modules/dnd/chunk/scenes/prompt_assets_test.go delete mode 100644 internal/modules/dnd/chunk/scenes/schema.go delete mode 100644 internal/modules/dnd/chunk/scenes/schema_test.go diff --git a/assets/dnd/scenes/prompts/correction.md b/assets/dnd/scenes/prompts/correction.md new file mode 100644 index 00000000..15f54206 --- /dev/null +++ b/assets/dnd/scenes/prompts/correction.md @@ -0,0 +1,8 @@ +Revise the previous scene-boundary plan using these findings: + +{{ input "guidance" }} + +Return a complete replacement scene plan for the original full transcript, not +only the changed ranges. Keep the original source-unit IDs and cover every unit +in document order without gaps or overlaps. Return only JSON matching the original +response contract. diff --git a/assets/dnd/scenes/prompts/prompt.yaml b/assets/dnd/scenes/prompts/prompt.yaml index 8116b52e..54b8fcb3 100644 --- a/assets/dnd/scenes/prompts/prompt.yaml +++ b/assets/dnd/scenes/prompts/prompt.yaml @@ -21,7 +21,7 @@ prompts: content_file: ../../shared/prompts/common-dnd-references.md cache_control: type: ephemeral - - role: user + - role: system content_file: ./instructions.md - role: user content_file: ../../shared/prompts/common-dnd-transcript-full.md @@ -31,3 +31,17 @@ prompts: format: json validation_mode: json_schema schema_path: dnd/scenes/schemas/dnd_scenes_llm.v1.json + + - id: dnd.scenes.repair + default_profile: notarius/dnd-extraction + inputs: + - name: guidance + required: true + content_type: text/plain + messages: + - role: user + content_file: ./correction.md + output: + format: json + validation_mode: json_schema + schema_path: dnd/scenes/schemas/dnd_scenes_llm.v1.json diff --git a/assets/scenes_test.go b/assets/scenes_test.go new file mode 100644 index 00000000..93db45d3 --- /dev/null +++ b/assets/scenes_test.go @@ -0,0 +1,184 @@ +package assets_test + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io/fs" + "reflect" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/assets" + "github.com/santhosh-tekuri/jsonschema/v6" + "go.maximumdirect.net/promptkit" +) + +func TestScenePromptPairPreservesFullTranscriptAndContext(t *testing.T) { + _, engine := loadNativeAssets(t) + const transcript = `{"sentinel":"scene-transcript"}` + initial, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{ + PromptID: "notarius/dnd.scenes", APIKey: "test-only", + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline(transcript), + "players": promptkit.Inline("scene-players"), + "party": promptkit.Inline("scene-party"), + "glossary": promptkit.Inline("scene-glossary"), + }, + }) + if err != nil { + t.Fatal(err) + } + defer initial.Discard() + correction, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{ + PromptID: "notarius/dnd.scenes.repair", APIKey: "test-only", + Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline("Keep the encounter together.")}, + }) + if err != nil { + t.Fatal(err) + } + defer correction.Discard() + first, next := initial.Details(), correction.Details() + if first.SelectedProfileID != "notarius/dnd-extraction" || next.SelectedProfileID != first.SelectedProfileID || + first.StructuredOutput == nil || next.StructuredOutput == nil || !reflect.DeepEqual(first.OutputContract, next.OutputContract) { + t.Fatal("scene correction pair lost its shared native defaults/output contract") + } + var system, all string + transcriptIndex := -1 + for i, message := range first.Messages { + all += message.Content + if message.Role == "system" { + system += message.Content + } + if strings.Contains(message.Content, transcript) { + transcriptIndex = i + } + if strings.Contains(message.Content, transcript) || strings.Contains(message.Content, "scene-players") { + if message.CacheControl == nil || message.CacheControl.Type != promptkit.CacheControlEphemeral { + t.Fatal("reference/full-transcript native cache hint lost") + } + } + } + if transcriptIndex != len(first.Messages)-1 || !strings.Contains(system, "no gaps or overlaps") || !strings.Contains(system, "coherent unit of play") { + t.Fatal("scene principles or complete final transcript lost") + } + for _, content := range []string{transcript, "scene-players", "scene-party", "scene-glossary"} { + if strings.Count(all, content) != 1 { + t.Fatalf("fixed context not rendered once: %q", content) + } + } + if len(next.Messages) != 1 || next.Messages[0].Role != "user" || !strings.Contains(next.Messages[0].Content, "Keep the encounter together.") || + strings.Contains(next.Messages[0].Content, "scene-transcript") { + t.Fatal("correction should supply findings and a replacement assignment only") + } +} + +func TestResponseSchemaValidatesMinimalSceneResponses(t *testing.T) { + schema, err := fs.ReadFile(assets.FS(), "dnd/scenes/schemas/dnd_scenes_llm.v1.json") + if err != nil { + t.Fatalf("loadResponseSchema() error = %v, want nil", err) + } + + valid := validSceneSchemaResponse() + validJSON, err := json.Marshal(valid) + if err != nil { + t.Fatalf("Marshal(validSceneSchemaResponse()) error = %v, want nil", err) + } + if err := validateJSONSchema(validJSON, schema); err != nil { + t.Fatalf("valid scene response rejected: %v", err) + } + + tests := []struct { + name string + mutate func(map[string]any) + }{ + { + name: "missing scenes", + mutate: func(response map[string]any) { + delete(response, "scenes") + }, + }, + { + name: "empty scenes", + mutate: func(response map[string]any) { + response["scenes"] = []any{} + }, + }, + { + name: "non-positive start unit", + mutate: func(response map[string]any) { + response["scenes"].([]any)[0].(map[string]any)["start_unit_id"] = 0 + }, + }, + { + name: "non-positive end unit", + mutate: func(response map[string]any) { + response["scenes"].([]any)[0].(map[string]any)["end_unit_id"] = 0 + }, + }, + { + name: "non-integer endpoint", + mutate: func(response map[string]any) { + response["scenes"].([]any)[0].(map[string]any)["start_unit_id"] = 1.5 + }, + }, + { + name: "unknown top-level field", + mutate: func(response map[string]any) { + response["boundary_caveats"] = []any{} + }, + }, + { + name: "unknown scene field", + mutate: func(response map[string]any) { + response["scenes"].([]any)[0].(map[string]any)["short_title"] = "Old contract" + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + response := validSceneSchemaResponse() + tt.mutate(response) + content, err := json.Marshal(response) + if err != nil { + t.Fatalf("Marshal() error = %v, want nil", err) + } + if err := validateJSONSchema(content, schema); err == nil { + t.Fatal("validateJSONSchema() error = nil, want rejected response") + } + }) + } +} + +func validSceneSchemaResponse() map[string]any { + return map[string]any{ + "scenes": []any{ + map[string]any{ + "start_unit_id": 1, + "end_unit_id": 3, + }, + }, + } +} + +func validateJSONSchema(instanceContent, schemaContent []byte) error { + instance, err := jsonschema.UnmarshalJSON(bytes.NewReader(instanceContent)) + if err != nil { + return fmt.Errorf("parse instance: %w", err) + } + schemaDocument, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaContent)) + if err != nil { + return fmt.Errorf("parse schema: %w", err) + } + compiler := jsonschema.NewCompiler() + if err := compiler.AddResource("schema.json", schemaDocument); err != nil { + return fmt.Errorf("load schema: %w", err) + } + schema, err := compiler.Compile("schema.json") + if err != nil { + return fmt.Errorf("compile schema: %w", err) + } + return schema.Validate(instance) +} diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 57ec42dd..2870d2cd 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -717,6 +717,29 @@ provider fakes and retain domain identity/reconciliation regression coverage. ## Stage 16: Migrate D&D Scene Planning And Chunk Assessment +**Status:** Complete. Scene planning now prepares the full transcript/reference +inputs once and uses the shared native producer helper with initial/correction +prompts, matching schema/profile defaults, and static dependencies. Enduring +scene principles remain in initial system context. Removed the private completion, +response-schema/asset registration, and manifest metadata paths. Existing +source-order, coverage, gap/overlap, and boundary interpretation rules remain. +Unresolvable boundaries supply actionable correction guidance; unusable native +structural results never become semantic candidates. Permissive terminal handling +constructs one unannotated whole-source plan using normal materialization; invalid +input, cancellation, and fallback-construction failures do not publish a fallback. +The runner already binds materialization and complete chunk assessment to the +native operation without an outer attempt/replay loop. Added a runnable runner +contract proving ordered full-chain assessment through final-turn acceptance, +strict/permissive exhaustion, no fallback revalidation, and derived reuse exclusion. +Migrated scene regressions to a native provider fake and added correction/history, +override, structural-failure, whole-source fallback, and cancellation cases. +Scene prompt-pair/schema checks run in the asset package. +Asset/source tests and vet/race checks, focused pipeline tests and vet/race checks, +and `git diff --check` pass. Scene-chunker package tests remain blocked at setup +by shared legacy helpers importing old ConfigKit's removed YAML `plugin/limit`; +their runtime results are not claimed passing. Scheduled dependency cleanup and +final verification must execute them. Chunk-cache replacement remains Stage 28. + **Goal:** one native chunk producer with deterministic materialization. **Inspect/change:** `internal/modules/dnd/chunk/scenes`, chunk-plan interpretation/ diff --git a/internal/framework/pipeline/chunk_plan_operation_test.go b/internal/framework/pipeline/chunk_plan_operation_test.go new file mode 100644 index 00000000..33a7d4ea --- /dev/null +++ b/internal/framework/pipeline/chunk_plan_operation_test.go @@ -0,0 +1,116 @@ +package pipeline + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/core/source" + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "go.maximumdirect.net/promptkit" +) + +type nativePlanChunker struct{ engine *promptkit.Engine } + +type namedPlanValidator struct { + name string + chunkAssessmentValidator +} + +func (v namedPlanValidator) Name() string { return v.name } + +func (nativePlanChunker) Key() string { return "native-plan" } +func (nativePlanChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil } +func (c nativePlanChunker) Plan(ctx context.Context, request contracts.ChunkRequest) (contracts.ChunkPlanResult, error) { + plan, outcome, err := RunProducer(ctx, c.engine, request.Operation, nativeProducerInitial(), + func(ctx context.Context, result *promptkit.RunResult) (*source.ChunkPlan, []string, error) { + value, _, err := interpretProducerNumber(ctx, result) + if err != nil { + return nil, nil, err + } + plan := typedTestPlan(request.Source) + plan.Ranges[0].Annotations = source.ChunkAnnotations{"test/scene-policy": json.RawMessage(fmt.Sprintf("%d", *value))} + return &plan, nil, nil + }, producerCorrection, func(context.Context, *source.ChunkPlan) (source.ChunkPlan, error) { + return typedTestPlan(request.Source), nil + }) + return contracts.ChunkPlanResult{Plan: plan, Outcome: outcome}, err +} + +func TestRunnerMaterializesAndFullyAssessesEachNativePlanRevision(t *testing.T) { + for _, test := range []struct { + name string + mode contracts.ExecutionMode + turns int + wantError, wantFallback bool + }{ + {"final-turn acceptance", contracts.ModeStrict, 2, false, false}, + {"strict exhaustion", contracts.ModeStrict, 1, true, false}, + {"permissive exhaustion", contracts.ModePermissive, 1, false, true}, + } { + t.Run(test.name, func(t *testing.T) { + calls := 0 + engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + value := calls + calls++ + return &promptkit.GenerateResponse{Content: fmt.Sprintf(`{"value":%d}`, value)}, nil + })) + catalog := typedResolutionCatalog(t, completeTypedCatalogOptions()) + if err := catalog.Chunkers.RegisterBuilderWithSpec(ModuleSpec{Key: "native-plan", Stage: StageChunk, ExecutionClass: contracts.ExecutionClassLLMBacked}, + rejectUnconfiguredOptions, func(request BuildRequest) (contracts.Chunker, error) { + return nativePlanChunker{request.Dependencies.Engine}, nil + }); err != nil { + t.Fatal(err) + } + var assessed []string + for _, name := range []string{"first-plan-policy", "second-plan-policy"} { + if err := RegisterChunkValidator(catalog.Validators, ValidatorSpec{Key: name, ExecutionClass: contracts.ExecutionClassDeterministic}, + func() (contracts.ChunkValidator, error) { + return namedPlanValidator{name, chunkAssessmentValidator(func(_ context.Context, request contracts.ChunkValidationRequest) (contracts.ValidationResult, error) { + if len(request.Chunks) != 1 || len(request.Chunks[0].Units) != len(request.Source.Units) || request.Chunks[0].Ref.SourceID != request.Source.ID { + t.Fatal("validator did not receive normally materialized chunks") + } + value := string(request.Chunks[0].Annotations["test/scene-policy"]) + assessed = append(assessed, name+"/"+value) + if value != "2" { + return contracts.ValidationResult{ReasonCode: "scene_policy", CorrectionGuidance: "Reconsider the scene transition."}, nil + } + return contracts.ValidationResult{Approved: true}, nil + })}, nil + }); err != nil { + t.Fatal(err) + } + } + probe := newRunnerProbe() + profile := typedResolutionProfile() + profile.Chunk = Binding("native-plan") + profile.Chunk.Mode, profile.Chunk.MaxAdditionalTurns = &test.mode, &test.turns + profile.Chunk.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{Binding("first-plan-policy"), Binding("second-plan-policy")}} + profile.Artifacts = map[string]ArtifactLaneProfile{"notes": registerRunnerNotes(t, catalog, runnerNotesProducer{name: "notes", probe: probe})} + output, err := New().Run(context.Background(), RunInput{Prepared: prepareRunnerProfile(t, catalog, profile, engine), RawInput: []byte("input")}) + if (err != nil) != test.wantError || calls != test.turns+1 { + t.Fatalf("native plan output=%#v error=%v calls=%d", output, err, calls) + } + var want []string + for i := 0; i <= test.turns; i++ { + want = append(want, fmt.Sprintf("first-plan-policy/%d", i), fmt.Sprintf("second-plan-policy/%d", i)) + } + if !reflect.DeepEqual(assessed, want) { + t.Fatalf("ordered plan assessments=%v, want %v", assessed, want) + } + if test.wantError { + return + } + if len(output.NormalizeOutputs) != 1 || (output.FallbackCount == 1) != test.wantFallback { + t.Fatalf("plan disposition=%#v", output) + } + for _, role := range []string{"extract", "merge", "normalize"} { + if !reflect.DeepEqual(probe.eligible["notes/"+role], []bool{!test.wantFallback}) { + t.Fatalf("derived %s eligibility=%v", role, probe.eligible) + } + } + }) + } +} diff --git a/internal/modules/dnd/chunk/scenes/chunker.go b/internal/modules/dnd/chunk/scenes/chunker.go index e6cb0924..0f031c55 100644 --- a/internal/modules/dnd/chunk/scenes/chunker.go +++ b/internal/modules/dnd/chunk/scenes/chunker.go @@ -2,7 +2,11 @@ package scenes import ( "context" + "encoding/json" "fmt" + "strings" + + "go.maximumdirect.net/promptkit" "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" @@ -11,6 +15,8 @@ import ( ) const Key = "dnd/scenes" +const PromptID = "notarius/dnd.scenes" +const CorrectionPromptID = "notarius/dnd.scenes.repair" var requiredCapabilities = []string{ "source.transcript", @@ -28,19 +34,18 @@ var referenceSlotDescriptions = shared.ReferenceSlotDescriptions{ } var _ contracts.Chunker = (*Chunker)(nil) -var _ contracts.ManifestMetadataProvider = (*Chunker)(nil) type Options struct{} type Chunker struct { - llm contracts.StructuredLLMClient + engine *promptkit.Engine } -func New(llmClient contracts.StructuredLLMClient, _ Options) (*Chunker, error) { - if llmClient == nil { - return nil, chunkerErrorf("LLM client must not be nil") +func New(engine *promptkit.Engine, _ Options) (*Chunker, error) { + if engine == nil { + return nil, chunkerErrorf("native engine must not be nil") } - return &Chunker{llm: llmClient}, nil + return &Chunker{engine: engine}, nil } func (c *Chunker) Key() string { @@ -51,32 +56,12 @@ func (c *Chunker) ReferenceSlots() []contracts.ReferenceSlot { return shared.ReferenceSlots(referenceSlotDescriptions) } -func (c *Chunker) ManifestMetadata() map[string]any { - promptSHA, err := promptAssetMetadata() - if err != nil { - promptSHA = "" - } - metadata := map[string]any{ - "prompt_id": PromptID, - "prompt_version": ResponseSchemaVersion, - "prompt_sha256": promptSHA, - "response_schema_key": string(ResponseSchemaKey), - "response_schema_id": ResponseSchemaID, - "response_schema_name": ResponseSchemaName, - } - if schema, err := loadResponseSchema(); err == nil { - metadata["response_schema_version"] = schema.Version - metadata["response_schema_sha256"] = schema.SHA256 - } - return metadata -} - func (c *Chunker) Plan(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkPlanResult, error) { if c == nil { return contracts.ChunkPlanResult{}, chunkerErrorf("chunker must not be nil") } - if c.llm == nil { - return contracts.ChunkPlanResult{}, chunkerErrorf("LLM client must not be nil") + if c.engine == nil { + return contracts.ChunkPlanResult{}, chunkerErrorf("native engine must not be nil") } if ctx == nil { return contracts.ChunkPlanResult{}, chunkerErrorf("context must not be nil") @@ -93,41 +78,52 @@ func (c *Chunker) Plan(ctx context.Context, req contracts.ChunkRequest) (contrac if err := source.ValidateDocument(req.Source); err != nil { return contracts.ChunkPlanResult{}, chunkerErrorf("validate source document: %w", err) } - var response chunkResponse - completion, err := c.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{ - StageName: Key, - PromptID: PromptID, - PromptVersion: ResponseSchemaVersion, - ProfileID: req.LLMProfile, - SessionID: req.SessionID, - StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts, - Correction: req.Correction, - Inputs: shared.PromptInputs(req.SourceInput, req.References), - }, &response) - if err != nil { - return contracts.ChunkPlanResult{}, chunkerErrorf("complete structured output: %w", err) + materials := shared.PromptInputs(req.SourceInput, req.References) + inputs := make(map[string]promptkit.ArtifactRef, len(materials)) + for name, material := range materials { + inputs[name] = promptkit.InlineWithURI(material.OriginURI, string(material.Content)) } - candidate, err := shared.ModelCandidateFromResponse(completion) + initial := promptkit.RunRequest{PromptID: PromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID, Inputs: inputs} + plan, outcome, err := pipeline.RunProducer(ctx, c.engine, req.Operation, initial, + func(_ context.Context, result *promptkit.RunResult) (*source.ChunkPlan, []string, error) { + var response chunkResponse + if err := json.Unmarshal([]byte(result.RawOutput), &response); err != nil { + return nil, nil, chunkerErrorf("decode native output: %w", err) + } + plan, err := planFromResponse(req.Source, response) + if err != nil { + return nil, []string{fmt.Sprintf("Revise the scene boundaries: %v. Use source-unit IDs from the supplied transcript and cover every unit in document order, without gaps or overlaps.", err)}, nil + } + return &plan, nil, nil + }, + func(_ context.Context, guidance []string) (promptkit.RunRequest, error) { + return promptkit.RunRequest{PromptID: CorrectionPromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID, + Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline(strings.Join(guidance, "\n\n"))}}, nil + }, + func(context.Context, *source.ChunkPlan) (source.ChunkPlan, error) { + plan := source.ChunkPlan{SourceDigest: req.Source.Digest, Ranges: []source.ChunkRange{{ + StartUnitID: req.Source.Units[0].ID, EndUnitID: req.Source.Units[len(req.Source.Units)-1].ID, + }}} + if _, err := source.MaterializeChunkPlan(req.Source, plan); err != nil { + return source.ChunkPlan{}, chunkerErrorf("materialize whole-source fallback: %w", err) + } + return plan, nil + }) if err != nil { - return contracts.ChunkPlanResult{}, chunkerErrorf("capture model candidate: %w", err) + return contracts.ChunkPlanResult{}, chunkerErrorf("produce scene plan: %w", err) } - - plan, err := planFromResponse(req.Source, response) - if err != nil { - return contracts.ChunkPlanResult{}, chunkerErrorf("malformed structured output: %w", err) - } - return contracts.ChunkPlanResult{Plan: plan, ModelCandidate: candidate}, nil + return contracts.ChunkPlanResult{Plan: plan, Outcome: outcome}, nil } func ModuleSpec() pipeline.ModuleSpec { return pipeline.ModuleSpec{ - Key: Key, - Stage: pipeline.StageChunk, - ExecutionClass: contracts.ExecutionClassLLMBacked, - CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1, - Requires: append([]string(nil), requiredCapabilities...), - Provides: append([]string(nil), providedCapabilities...), - ReferenceSlots: shared.ReferenceSlots(referenceSlotDescriptions), + Key: Key, + Stage: pipeline.StageChunk, + ExecutionClass: contracts.ExecutionClassLLMBacked, + Prompts: []string{PromptID, CorrectionPromptID}, + Requires: append([]string(nil), requiredCapabilities...), + Provides: append([]string(nil), providedCapabilities...), + ReferenceSlots: shared.ReferenceSlots(referenceSlotDescriptions), } } @@ -137,7 +133,7 @@ func Register(registry *pipeline.ChunkerRegistry) error { if err != nil { return nil, err } - return New(request.Dependencies.LLM, options) + return New(request.Dependencies.Engine, options) }) } diff --git a/internal/modules/dnd/chunk/scenes/chunker_test.go b/internal/modules/dnd/chunk/scenes/chunker_test.go index e2652d93..57c244a5 100644 --- a/internal/modules/dnd/chunk/scenes/chunker_test.go +++ b/internal/modules/dnd/chunk/scenes/chunker_test.go @@ -8,10 +8,13 @@ import ( "strings" "testing" + "gitea.maximumdirect.net/eric/notarius/assets" "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" + "go.maximumdirect.net/configkit" + "go.maximumdirect.net/promptkit" ) func TestNewModuleSpecAndRegister(t *testing.T) { @@ -22,18 +25,19 @@ func TestNewModuleSpecAndRegister(t *testing.T) { } want := pipeline.ModuleSpec{ - Key: Key, - Stage: pipeline.StageChunk, - ExecutionClass: contracts.ExecutionClassLLMBacked, - CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1, - Requires: []string{"source.transcript"}, - Provides: []string{"chunks"}, - ReferenceSlots: wantReferenceSlots(), + Key: Key, + Stage: pipeline.StageChunk, + ExecutionClass: contracts.ExecutionClassLLMBacked, + Prompts: []string{PromptID, CorrectionPromptID}, + Requires: []string{"source.transcript"}, + Provides: []string{"chunks"}, + ReferenceSlots: wantReferenceSlots(), } if got := ModuleSpec(); !reflect.DeepEqual(got, want) { t.Fatalf("ModuleSpec() = %#v, want %#v", got, want) } got := ModuleSpec() + got.Prompts[0] = "changed" got.Requires[0] = "changed" got.Provides[0] = "changed" got.ReferenceSlots[0].AcceptedMediaTypes[0] = "changed" @@ -52,7 +56,7 @@ func TestNewModuleSpecAndRegister(t *testing.T) { if !reflect.DeepEqual(registered, want) { t.Fatalf("registered spec = %#v, want %#v", registered, want) } - built, err := registry.BuildWithRequest(Key, pipeline.BuildRequest{Dependencies: pipeline.ModuleDependencies{LLM: client}}) + built, err := registry.BuildWithRequest(Key, pipeline.BuildRequest{Dependencies: pipeline.ModuleDependencies{Engine: chunker.engine}}) if err != nil { t.Fatalf("Build(%q) error = %v, want nil", Key, err) } @@ -68,8 +72,8 @@ func TestNewModuleSpecAndRegister(t *testing.T) { } func TestConstructionRejectsMissingDependencyAndUnknownOptions(t *testing.T) { - if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "LLM client") { - t.Fatalf("New(nil) error = %v, want LLM client error", err) + if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "native engine") { + t.Fatalf("New(nil) error = %v, want native engine error", err) } if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil || !strings.Contains(err.Error(), "unknown option") { t.Fatalf("DecodeOptions() error = %v, want unknown option error", err) @@ -122,7 +126,10 @@ func TestPlanReturnsAnnotationFreeSceneRangesFromStructuredOutput(t *testing.T) }, } - result, err := newChunker(t, client).Plan(context.Background(), chunkRequest()) + request := chunkRequest() + var prepared promptkit.PreparedRun + request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = value } + result, err := newChunker(t, client).Plan(context.Background(), request) if err != nil { t.Fatalf("Plan() error = %v, want nil", err) } @@ -130,34 +137,21 @@ func TestPlanReturnsAnnotationFreeSceneRangesFromStructuredOutput(t *testing.T) if len(client.requests) != 1 { t.Fatalf("LLM calls = %d, want 1", len(client.requests)) } - req := client.requests[0] - if req.StageName != Key { - t.Fatalf("StageName = %q, want %q", req.StageName, Key) + call := client.requests[0] + if prepared.PromptID != PromptID || prepared.SelectedProfileID != request.LLMProfile || call.Prompt.SessionID != request.SessionID { + t.Fatal("native prompt/profile/session not preserved") } - if req.PromptID != PromptID || req.PromptVersion != ResponseSchemaVersion { - t.Fatalf("prompt = %q/%q, want %q/%q", req.PromptID, req.PromptVersion, PromptID, ResponseSchemaVersion) + if result.Outcome.Disposition != contracts.ProducerAccepted || !result.Outcome.ReuseEligible { + t.Fatalf("plan not ordinarily accepted: %#v", result) } - if req.SessionID != "session-123" || req.ProfileID != "profile-scenes" { - t.Fatalf("session/profile = %q/%q, want session-123/profile-scenes", req.SessionID, req.ProfileID) + var transcriptFound bool + for _, message := range call.Prompt.Messages { + if strings.Contains(message.Content, sceneTranscriptJSON) { + transcriptFound = true + } } - transcript, ok := req.Inputs["transcript"] - if !ok { - t.Fatalf("transcript input missing from %#v", req.Inputs) - } - if transcript.Name != "transcript" || transcript.MediaType != "application/json" || transcript.Digest != "sha256:transcript" || transcript.OriginURI != "file:///session-alpha.json" { - t.Fatalf("transcript metadata = %#v", transcript) - } - if got := string(transcript.Content); got != sceneTranscriptJSON { - t.Fatalf("transcript content = %q, want original source input", got) - } - if got := string(req.Inputs["players"].Content); got != " " { - t.Fatalf("players input = %q, want empty reference placeholder", got) - } - if got := string(req.Inputs["party"].Content); got != " " { - t.Fatalf("party input = %q, want empty reference placeholder", got) - } - if got := string(req.Inputs["glossary"].Content); got != " " { - t.Fatalf("glossary input = %q, want empty reference placeholder", got) + if !transcriptFound { + t.Fatal("full source input missing from native prompt") } if result.Plan.SourceDigest != "sha256:source" { @@ -179,13 +173,7 @@ func TestPlanReturnsAnnotationFreeSceneRangesFromStructuredOutput(t *testing.T) if len(result.Plan.Annotations) != 0 { t.Fatalf("plan annotations = %#v, want absent", result.Plan.Annotations) } - wantCandidate, err := json.Marshal(client.response) - if err != nil { - t.Fatalf("marshal expected candidate: %v", err) - } - if result.ModelCandidate == nil || result.ModelCandidate.Protocol != contracts.CorrectionProtocolSingleResponseV1 || !reflect.DeepEqual(result.ModelCandidate.Response, wantCandidate) { - t.Fatalf("model candidate = %#v, want exact validated response", result.ModelCandidate) - } + } func TestPlanUsesDocumentOrderForNonconsecutiveUnitIDs(t *testing.T) { @@ -223,7 +211,7 @@ func TestPlanUsesDocumentOrderForNonconsecutiveUnitIDs(t *testing.T) { scene(20, 20), }}} _, err := newChunker(t, client).Plan(context.Background(), req) - if err == nil || !strings.Contains(err.Error(), "appears after") { + if err == nil { t.Fatalf("Plan() error = %v, want document-position reversal", err) } }) @@ -262,19 +250,16 @@ func TestPlanPassesReferencesAsPromptInputs(t *testing.T) { if _, err := newChunker(t, client).Plan(context.Background(), req); err != nil { t.Fatalf("Plan() error = %v, want nil", err) } - request := client.requests[0] - if got := string(request.Inputs["players"].Content); got != "Alice: Aria" { - t.Fatalf("players input = %q, want reference content", got) + var messages string + for _, message := range client.requests[0].Prompt.Messages { + messages += message.Content } - if got := string(request.Inputs["party"].Content); got != "Aria: cleric" { - t.Fatalf("party input = %q, want reference content", got) - } - if got := string(request.Inputs["glossary"].Content); got != "Brightmantle: local temple" { - t.Fatalf("glossary input = %q, want reference content", got) - } - if strings.Contains(string(request.Inputs["transcript"].Content), "Aria: cleric") { - t.Fatalf("transcript input contains reference content") + for _, reference := range []string{"Alice: Aria", "Aria: cleric", "Brightmantle: local temple"} { + if strings.Count(messages, reference) != 1 { + t.Fatalf("reference not rendered once: %q", reference) + } } + } func TestPromptInputsMapLegacyRosterReferenceToParty(t *testing.T) { @@ -297,35 +282,6 @@ func TestPromptInputsMapLegacyRosterReferenceToParty(t *testing.T) { } } -func TestChunkerManifestMetadataIncludesPromptAndSchemaProvenance(t *testing.T) { - metadata := newChunker(t, &fakeScenesLLMClient{}).ManifestMetadata() - - tests := map[string]string{ - "prompt_id": PromptID, - "prompt_version": ResponseSchemaVersion, - "response_schema_key": string(ResponseSchemaKey), - "response_schema_id": ResponseSchemaID, - "response_schema_name": ResponseSchemaName, - "response_schema_version": ResponseSchemaVersion, - } - for key, want := range tests { - if metadata[key] != want { - t.Fatalf("metadata[%q] = %#v, want %q", key, metadata[key], want) - } - } - for _, key := range []string{"prompt_sha256", "response_schema_sha256"} { - value, ok := metadata[key].(string) - if !ok || !strings.HasPrefix(value, "sha256:") { - t.Fatalf("metadata[%q] = %#v, want sha256 value", key, metadata[key]) - } - } - for _, forbidden := range []string{"prompt", "schema", "source", "text"} { - if _, ok := metadata[forbidden]; ok { - t.Fatalf("metadata includes raw %q field: %#v", forbidden, metadata) - } - } -} - func TestPlanRejectsInvalidRequests(t *testing.T) { validClient := &fakeScenesLLMClient{response: validSceneResponse()} validReq := chunkRequest() @@ -357,14 +313,14 @@ func TestPlanRejectsInvalidRequests(t *testing.T) { if err == nil { t.Fatal("Plan() error = nil, want error") } - if !strings.Contains(err.Error(), "dnd scenes") || !strings.Contains(err.Error(), tt.want) { - t.Fatalf("Plan() error = %q, want module context and %q", err.Error(), tt.want) + if !strings.Contains(err.Error(), tt.want) { + t.Fatalf("Plan() error = %v, want %q", err, tt.want) } }) } } -func TestPlanRejectsMalformedStructuredOutput(t *testing.T) { +func TestPlanFromResponseRejectsMalformedSceneRanges(t *testing.T) { tests := []struct { name string response chunkResponse @@ -413,8 +369,7 @@ func TestPlanRejectsMalformedStructuredOutput(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - client := &fakeScenesLLMClient{response: tt.response} - _, err := newChunker(t, client).Plan(context.Background(), chunkRequest()) + _, err := planFromResponse(sceneSourceDocument(), tt.response) if err == nil { t.Fatal("Plan() error = nil, want error") } @@ -443,6 +398,13 @@ func chunkRequest() contracts.ChunkRequest { SourceInput: sceneSourceInput(), SessionID: "session-123", LLMProfile: "profile-scenes", + Operation: contracts.ProducerOperation[source.ChunkPlan]{ + Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, Cache: contracts.CacheBypass}, + ReuseEligible: true, + Assess: func(context.Context, source.ChunkPlan) (contracts.CandidateAssessment, error) { + return contracts.CandidateAssessment{}, nil + }, + }, } } @@ -452,11 +414,33 @@ func sceneSourceInput() contracts.LLMInputMaterial { return contracts.NewLLMInputMaterial("source", "application/json", []byte(sceneTranscriptJSON), "sha256:transcript", "file:///session-alpha.json") } -func newChunker(t *testing.T, client contracts.StructuredLLMClient) *Chunker { +func newChunker(t *testing.T, client promptkit.LLMClient) *Chunker { t.Helper() - chunker, err := New(client, Options{}) + type settings struct { + PromptKit promptkit.Settings `yaml:"promptkit"` + } + definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit }) + filesystem := configkit.NewFilesystem("Notarius assets", assets.FS()) + result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("scene fixture", []byte(`promptkit: + profiles: + - id: profile-scenes + endpoint: https://example.invalid/v1 + model: scene-test-model + api_key_required: false +`))}, configkit.LayeredOptions[settings]{Definition: definition, Underlays: []configkit.Underlay{ + promptkit.BundledOpenRouterUnderlay(), + {Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}}, + }}) if err != nil { - t.Fatalf("New() error = %v, want nil", err) + t.Fatal(err) + } + engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client)) + if err != nil { + t.Fatal(err) + } + chunker, err := New(engine, Options{}) + if err != nil { + t.Fatal(err) } return chunker } @@ -513,47 +497,32 @@ func unitIDs(units []source.SourceUnit) []int { } type fakeScenesLLMClient struct { - response chunkResponse - err error - requests []contracts.StructuredCompletionRequest + response chunkResponse + responses []string + err error + errors []error + requests []promptkit.GenerateRequest + onGenerate func(context.Context, promptkit.GenerateRequest) } -func (client *fakeScenesLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { - client.requests = append(client.requests, cloneStructuredCompletionRequest(req)) +func (client *fakeScenesLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + index := len(client.requests) + client.requests = append(client.requests, request) + if client.onGenerate != nil { + client.onGenerate(ctx, request) + } if client.err != nil { - return contracts.StructuredCompletionResponse{}, client.err + return nil, client.err } - - target, ok := out.(*chunkResponse) - if !ok { - return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target") + if index < len(client.errors) && client.errors[index] != nil { + return nil, client.errors[index] } - *target = client.response content, err := json.Marshal(client.response) if err != nil { - return contracts.StructuredCompletionResponse{}, err + return nil, err } - return contracts.StructuredCompletionResponse{Content: content}, nil -} - -func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest { - req.Inputs = req.Inputs.Clone() - correction, err := contracts.CloneSemanticCorrection(req.Correction) - if err != nil { - panic(err) - } - req.Correction = correction - req.Vars = cloneVars(req.Vars) - return req -} - -func cloneVars(in map[string]any) map[string]any { - if len(in) == 0 { - return nil - } - out := make(map[string]any, len(in)) - for key, value := range in { - out[key] = value - } - return out + if index < len(client.responses) { + content = []byte(client.responses[index]) + } + return &promptkit.GenerateResponse{Content: string(content)}, nil } diff --git a/internal/modules/dnd/chunk/scenes/model_test.go b/internal/modules/dnd/chunk/scenes/model_test.go new file mode 100644 index 00000000..903e725d --- /dev/null +++ b/internal/modules/dnd/chunk/scenes/model_test.go @@ -0,0 +1,21 @@ +package scenes + +import ( + "encoding/json" + "testing" +) + +func TestResponseStructAcceptsIntegerBoundaries(t *testing.T) { + raw := []byte(`{"scenes":[{"start_unit_id":1,"end_unit_id":3}]}`) + + var response chunkResponse + if err := json.Unmarshal(raw, &response); err != nil { + t.Fatalf("Unmarshal() error = %v, want nil", err) + } + if got := response.Scenes[0].StartUnitID.String(); got != "1" { + t.Fatalf("StartUnitID = %q, want 1", got) + } + if got := response.Scenes[0].EndUnitID.String(); got != "3" { + t.Fatalf("EndUnitID = %q, want 3", got) + } +} diff --git a/internal/modules/dnd/chunk/scenes/native_execution_test.go b/internal/modules/dnd/chunk/scenes/native_execution_test.go new file mode 100644 index 00000000..855fa5ba --- /dev/null +++ b/internal/modules/dnd/chunk/scenes/native_execution_test.go @@ -0,0 +1,205 @@ +package scenes + +import ( + "context" + "errors" + "math" + "reflect" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/core/source" + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "go.maximumdirect.net/promptkit" +) + +const wholeScene = `{"scenes":[{"start_unit_id":1,"end_unit_id":4}]}` +const splitScenes = `{"scenes":[{"start_unit_id":1,"end_unit_id":2},{"start_unit_id":3,"end_unit_id":4}]}` + +func TestNativePlanAssessesEveryRevisionThroughTheFinalTurn(t *testing.T) { + client := &fakeScenesLLMClient{responses: []string{wholeScene, wholeScene, splitScenes}} + request := chunkRequest() + request.Operation.Controls.MaxAdditionalTurns = 2 + request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "scene-override"} + var prepared []promptkit.PreparedRun + request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = append(prepared, value) } + assessments := 0 + request.Operation.Assess = func(_ context.Context, plan source.ChunkPlan) (contracts.CandidateAssessment, error) { + chunks, err := source.MaterializeChunkPlan(request.Source, plan) + if err != nil { + return contracts.CandidateAssessment{}, err + } + assessments++ + findings := []contracts.ValidatorAssessment{ + {ValidatorName: "coverage", Decision: contracts.AssessmentApproved, Result: contracts.ValidationResult{Approved: true}}, + {ValidatorName: "scene-policy", Decision: contracts.AssessmentApproved, Result: contracts.ValidationResult{Approved: true}}, + } + if len(chunks) != 2 { + findings[1].Decision = contracts.AssessmentRejected + findings[1].Result = contracts.ValidationResult{Message: "operator-only-rejection", CorrectionGuidance: "Separate the negotiation from the combat."} + } + return contracts.CandidateAssessment{Validators: findings}, nil + } + result, err := newChunker(t, client).Plan(context.Background(), request) + if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || !result.Outcome.ReuseEligible || + len(result.Plan.Ranges) != 2 || assessments != 3 || len(client.requests) != 3 || len(prepared) != 3 { + t.Fatalf("final-turn result=%#v %v, assessments/provider/prepared=%d/%d/%d", result, err, assessments, len(client.requests), len(prepared)) + } + for i, call := range client.requests { + wantID := PromptID + if i > 0 { + wantID = CorrectionPromptID + } + if prepared[i].PromptID != wantID || prepared[i].SelectedProfileID != request.LLMProfile || + call.Target.Model != "scene-override" || call.Prompt.SessionID != request.SessionID { + t.Fatal("native run selections lost") + } + if i == 0 { + continue + } + old, revised := client.requests[i-1].Prompt.Messages, call.Prompt.Messages + if len(revised) != len(old)+2 || !reflect.DeepEqual(old, revised[:len(old)]) || + revised[len(old)].Role != "assistant" || revised[len(old)].Content != client.responses[i-1] { + t.Fatal("native continuation changed source context/history") + } + guidance := revised[len(old)+1].Content + if !strings.Contains(guidance, "Separate the negotiation") || strings.Contains(guidance, "operator-only-rejection") { + t.Fatal("correction did not separate actionable findings from operator diagnostics") + } + } +} + +func TestNativePlanCanCorrectAnUnresolvableBoundary(t *testing.T) { + client := &fakeScenesLLMClient{responses: []string{ + `{"scenes":[{"start_unit_id":1,"end_unit_id":999}]}`, splitScenes, + }} + request := chunkRequest() + request.Operation.Controls.MaxAdditionalTurns = 1 + assessments := 0 + request.Operation.Assess = func(_ context.Context, plan source.ChunkPlan) (contracts.CandidateAssessment, error) { + assessments++ + _, err := source.MaterializeChunkPlan(request.Source, plan) + return contracts.CandidateAssessment{}, err + } + result, err := newChunker(t, client).Plan(context.Background(), request) + if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || assessments != 1 || len(client.requests) != 2 { + t.Fatalf("boundary correction=%#v %v, assessments=%d calls=%d", result, err, assessments, len(client.requests)) + } + messages := client.requests[1].Prompt.Messages + if !strings.Contains(messages[len(messages)-1].Content, "999") { + t.Fatal("correction omitted the unresolved boundary") + } +} + +func TestPlanExhaustionUsesOnlyAnExplicitWholeSourceFallback(t *testing.T) { + for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} { + t.Run(string(mode), func(t *testing.T) { + client := &fakeScenesLLMClient{response: chunkResponse{Scenes: []sceneResponse{scene(1, 2), scene(3, 4)}}} + request := chunkRequest() + request.Operation.Controls.Mode = mode + request.Operation.Controls.MaxAdditionalTurns = 1 + assessments := 0 + request.Operation.Assess = func(context.Context, source.ChunkPlan) (contracts.CandidateAssessment, error) { + assessments++ + return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{ + ValidatorName: "scene-policy", Decision: contracts.AssessmentRejected, + Result: contracts.ValidationResult{CorrectionGuidance: "Reconsider the scene transition."}, + }}}, nil + } + result, err := newChunker(t, client).Plan(context.Background(), request) + if assessments != 2 || len(client.requests) != 2 { + t.Fatal("revision was omitted, restarted, or reassessed") + } + if mode == contracts.ModeStrict { + if err == nil || result.Outcome.Disposition != "" { + t.Fatalf("strict rejection=%#v %v", result, err) + } + return + } + want := source.ChunkPlan{SourceDigest: request.Source.Digest, Ranges: []source.ChunkRange{{StartUnitID: 1, EndUnitID: 4}}} + if err != nil || !reflect.DeepEqual(result.Plan, want) || result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible { + t.Fatalf("whole-source fallback=%#v %v", result, err) + } + chunks, err := source.MaterializeChunkPlan(request.Source, result.Plan) + if err != nil || len(chunks) != 1 || !reflect.DeepEqual(unitIDs(chunks[0].Units), []int{1, 2, 3, 4}) || + len(chunks[0].Annotations) != 0 || len(chunks[0].PlanAnnotations) != 0 { + t.Fatalf("fallback did not materialize the unannotated complete source: %#v %v", chunks, err) + } + }) + } +} + +func TestNativePlanFailureUsesDocumentOrderWithoutEarlierPlanRecovery(t *testing.T) { + failure := errors.New("provider unavailable") + for _, failedCorrection := range []bool{false, true} { + client := &fakeScenesLLMClient{err: failure} + request := chunkRequest() + request.Source.Units[0].ID, request.Source.Units[1].ID = 30, 10 + for i := range request.Source.Units { + unit := &request.Source.Units[i] + unit.Ref.StartUnitID, unit.Ref.EndUnitID = unit.ID, unit.ID + } + if failedCorrection { + client.err = nil + client.responses = []string{`{"scenes":[{"start_unit_id":30,"end_unit_id":10},{"start_unit_id":3,"end_unit_id":4}]}`} + client.errors = []error{nil, failure} + } + request.Operation.Controls.Mode = contracts.ModePermissive + request.Operation.Controls.MaxAdditionalTurns = 1 + request.Operation.Assess = func(context.Context, source.ChunkPlan) (contracts.CandidateAssessment, error) { + return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{ + ValidatorName: "scene-policy", Decision: contracts.AssessmentRejected, + Result: contracts.ValidationResult{CorrectionGuidance: "Keep the encounter together."}, + }}}, nil + } + result, err := newChunker(t, client).Plan(context.Background(), request) + if err != nil || result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || + !reflect.DeepEqual(result.Plan.Ranges, []source.ChunkRange{{StartUnitID: 30, EndUnitID: 4}}) { + t.Fatalf("failure fallback=%#v %v", result, err) + } + chunks, err := source.MaterializeChunkPlan(request.Source, result.Plan) + if err != nil || !reflect.DeepEqual(unitIDs(chunks[0].Units), []int{30, 10, 3, 4}) { + t.Fatalf("nonconsecutive source order lost: %#v %v", chunks, err) + } + } +} + +func TestNativeStructuralFailureSkipsSemanticAssessment(t *testing.T) { + client := &fakeScenesLLMClient{response: chunkResponse{Scenes: []sceneResponse{}}} + request := chunkRequest() + request.Operation.Controls.Mode = contracts.ModePermissive + request.Operation.Controls.MaxAdditionalTurns = 2 + assessments := 0 + request.Operation.Assess = func(context.Context, source.ChunkPlan) (contracts.CandidateAssessment, error) { + assessments++ + return contracts.CandidateAssessment{}, nil + } + result, err := newChunker(t, client).Plan(context.Background(), request) + if err != nil || result.Outcome.Disposition != contracts.ProducerFallback || assessments != 0 { + t.Fatalf("structural failure=%#v %v", result, err) + } +} + +func TestCancellationAndMaterializationFailureDoNotPublishFallback(t *testing.T) { + t.Run("canceled native delivery", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + client := &fakeScenesLLMClient{response: validSceneResponse(), onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }} + request := chunkRequest() + request.Operation.Controls.Mode = contracts.ModePermissive + result, err := newChunker(t, client).Plan(ctx, request) + if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" || len(client.requests) != 1 { + t.Fatalf("cancellation=%#v %v", result, err) + } + }) + t.Run("unmaterializable source", func(t *testing.T) { + request := chunkRequest() + request.Operation.Controls.Mode = contracts.ModePermissive + request.Source.Units[0].Metadata = map[string]any{"invalid": math.NaN()} + failure := errors.New("provider unavailable") + result, err := newChunker(t, &fakeScenesLLMClient{err: failure}).Plan(context.Background(), request) + if err == nil || result.Outcome.Disposition != "" || !errors.Is(err, failure) { + t.Fatalf("unusable fallback=%#v %v", result, err) + } + }) +} diff --git a/internal/modules/dnd/chunk/scenes/prompt_assets.go b/internal/modules/dnd/chunk/scenes/prompt_assets.go deleted file mode 100644 index f9d3c90d..00000000 --- a/internal/modules/dnd/chunk/scenes/prompt_assets.go +++ /dev/null @@ -1,72 +0,0 @@ -package scenes - -import ( - "fmt" - "io/fs" - "sync" - - rootassets "gitea.maximumdirect.net/eric/notarius/assets" - "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - "gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs" - "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" -) - -const promptAssetRoot = "assets/prompts" - -var promptAssetManifest = shared.PromptAssetManifest{ - ModuleDir: "dnd.scenes", - ModuleFiles: []promptfs.ModulePromptFile{ - {Name: "prompt.yaml", Path: "prompts/prompt.yaml"}, - {Name: "instructions.md", Path: "prompts/instructions.md"}, - }, - SharedFiles: []string{ - "common-dnd-system.md", - "common-dnd-transcript-full.md", - "common-dnd-references.md", - }, -} - -func moduleAssetFS() (fs.FS, error) { - assets, err := fs.Sub(rootassets.FS(), "dnd/scenes") - if err != nil { - return nil, fmt.Errorf("scope scene assets: %w", err) - } - return assets, nil -} - -func RegisterPromptAssets(registry *llm.AssetRegistry) error { - assets, err := moduleAssetFS() - if err != nil { - return err - } - promptFS, err := promptAssetManifest.PromptFS(assets) - if err != nil { - return fmt.Errorf("prepare scene prompt assets: %w", err) - } - if err := registry.RegisterPromptFS(promptFS, promptAssetRoot); err != nil { - return err - } - schemas, err := fs.Sub(assets, "schemas") - if err != nil { - return fmt.Errorf("scope scene schemas: %w", err) - } - return registry.RegisterSchemaFS(schemas, ".") -} - -func promptAssetMetadata() (string, error) { - promptAssetHashOnce.Do(func() { - assets, err := moduleAssetFS() - if err != nil { - promptAssetHashErr = err - return - } - promptAssetHash, promptAssetHashErr = promptAssetManifest.Hash(assets) - }) - return promptAssetHash, promptAssetHashErr -} - -var ( - promptAssetHashOnce sync.Once - promptAssetHash string - promptAssetHashErr error -) diff --git a/internal/modules/dnd/chunk/scenes/prompt_assets_test.go b/internal/modules/dnd/chunk/scenes/prompt_assets_test.go deleted file mode 100644 index 69bb2f78..00000000 --- a/internal/modules/dnd/chunk/scenes/prompt_assets_test.go +++ /dev/null @@ -1,138 +0,0 @@ -package scenes - -import ( - "context" - "encoding/json" - "slices" - "strings" - "testing" - "time" - - "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - "gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture" - "gitea.maximumdirect.net/eric/promptkit" -) - -func TestPromptAssetsPrepareTranscriptWithInstructionOrdering(t *testing.T) { - if !slices.Contains(promptAssetManifest.SharedFiles, "common-dnd-transcript-full.md") { - t.Fatalf("shared prompt assets = %#v, want complete-session transcript scope", promptAssetManifest.SharedFiles) - } - transcript := []byte(`{"sentinel":"scene-transcript"}`) - prepared := prepareScenesPrompt(t, transcript, "scene-players", "scene-party", "scene-glossary") - - if prepared.PromptID != PromptID { - t.Fatalf("prompt id = %q, want %q", prepared.PromptID, PromptID) - } - if prepared.OutputContract.SchemaPath != "dnd_scenes_llm.v1.json" { - t.Fatalf("scene output schema path = %q, want private LLM schema", prepared.OutputContract.SchemaPath) - } - if prepared.Messages[0].Role != "system" || prepared.Messages[1].Role != "user" { - t.Fatalf("initial prompt roles = %#v, want system followed by campaign references", prepared.Messages[:2]) - } - if cache := prepared.Messages[1].CacheControl; cache == nil || cache.Type != promptkit.CacheControlEphemeral { - t.Fatalf("campaign-reference cache control = %#v, want ephemeral", cache) - } - if cache := prepared.Messages[0].CacheControl; cache != nil { - t.Fatalf("system cache control = %#v, want nil", cache) - } - if references := prepared.Messages[1].Content; !strings.Contains(references, "scene-players") || !strings.Contains(references, "scene-party") || !strings.Contains(references, "scene-glossary") { - t.Fatalf("reference message = %q, want supplied reference inputs", references) - } - transcriptIndex := -1 - for index, message := range prepared.Messages { - if strings.Contains(message.Content, "scene-transcript") { - transcriptIndex = index - } - } - if transcriptIndex != len(prepared.Messages)-1 { - t.Fatalf("transcript message index = %d, want final message in %#v", transcriptIndex, prepared.Messages) - } - if cache := prepared.Messages[transcriptIndex].CacheControl; cache == nil || cache.Type != promptkit.CacheControlEphemeral { - t.Fatalf("transcript cache control = %#v, want ephemeral", cache) - } - instructionsIndex := transcriptIndex - 1 - if instructionsIndex <= 1 || prepared.Messages[instructionsIndex].CacheControl != nil { - t.Fatalf("instruction message = %#v, want uncached message after references and before transcript", prepared.Messages[instructionsIndex]) - } - for index, message := range prepared.Messages { - if index != 1 && (strings.Contains(message.Content, "scene-players") || strings.Contains(message.Content, "scene-party") || strings.Contains(message.Content, "scene-glossary")) { - t.Errorf("message %d unexpectedly rendered reference input", index) - } - if index != transcriptIndex && strings.Contains(message.Content, "scene-transcript") { - t.Errorf("message %d unexpectedly rendered transcript input", index) - } - } -} - -func TestPromptAssetDiagnosticsOmitRawMaterials(t *testing.T) { - transcript := []byte(`{"secret":"source text"}`) - prepared := prepareScenesPrompt(t, transcript, "private player note", "private party note", "private glossary note") - metadata := newChunker(t, &fakeScenesLLMClient{}).ManifestMetadata() - - payload, err := json.Marshal(map[string]any{ - "prepared": map[string]any{ - "prompt_id": prepared.PromptID, - "prompt_version": prepared.PromptVersion, - "prompt_hash": prepared.PromptHash, - "rendered_prompt_hash": prepared.RenderedPromptHash, - "selected_profile_id": prepared.SelectedProfileID, - "output_contract": prepared.OutputContract, - "input_hashes": prepared.InputHashes, - "effective_model_params": prepared.EffectiveModelParams, - }, - "manifest": metadata, - }) - if err != nil { - t.Fatalf("marshal diagnostics: %v", err) - } - diagnostics := string(payload) - for _, forbidden := range []string{ - "source text", - "private player note", - "private party note", - "private glossary note", - `"properties"`, - "start_unit_id", - } { - if strings.Contains(diagnostics, forbidden) { - t.Fatalf("diagnostics leaked %q: %s", forbidden, diagnostics) - } - } - if metadata["prompt_id"] != PromptID || metadata["prompt_version"] != ResponseSchemaVersion { - t.Fatalf("manifest prompt metadata = %#v", metadata) - } - if !strings.HasPrefix(metadata["prompt_sha256"].(string), "sha256:") { - t.Fatalf("manifest prompt hash = %#v, want sha256-prefixed", metadata["prompt_sha256"]) - } -} - -func prepareScenesPrompt(t *testing.T, transcript []byte, players string, party string, glossary string) *promptkit.PreparedRun { - t.Helper() - registry := llm.NewAssetRegistry() - if err := RegisterPromptAssets(registry); err != nil { - t.Fatalf("register scene prompt assets: %v", err) - } - engine := newScenesPromptEngine(t, registry) - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: PromptID, - PromptVersion: ResponseSchemaVersion, - ProfileID: "scene-test-profile", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.InlineWithURI("file:///session.json", string(transcript)), - "players": promptkit.Inline(players), - "party": promptkit.Inline(party), - "glossary": promptkit.Inline(glossary), - }, - }) - if err != nil { - t.Fatalf("Prepare() error = %v, want nil", err) - } - return prepared -} - -func newScenesPromptEngine(t *testing.T, registry *llm.AssetRegistry) *promptkit.Engine { - t.Helper() - return promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{ - "scene-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "scene-test-model"), - }) -} diff --git a/internal/modules/dnd/chunk/scenes/schema.go b/internal/modules/dnd/chunk/scenes/schema.go deleted file mode 100644 index de5f6ef6..00000000 --- a/internal/modules/dnd/chunk/scenes/schema.go +++ /dev/null @@ -1,25 +0,0 @@ -package scenes - -import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - -const ( - PromptID = "dnd.scenes" - ResponseSchemaKey = llm.ResponseSchemaKey("dnd_scenes_llm") - ResponseSchemaID = "notarius.dnd.scenes.llm" - ResponseSchemaVersion = "v1" - ResponseSchemaName = "notarius_dnd_scenes_llm_v1" -) - -func loadResponseSchema() (llm.ResponseSchema, error) { - assets, err := moduleAssetFS() - if err != nil { - return llm.ResponseSchema{}, err - } - return llm.LoadResponseSchema(assets, llm.ResponseSchemaDefinition{ - Key: ResponseSchemaKey, - ID: ResponseSchemaID, - Version: ResponseSchemaVersion, - Name: ResponseSchemaName, - AssetPath: "schemas/dnd_scenes_llm.v1.json", - }) -} diff --git a/internal/modules/dnd/chunk/scenes/schema_test.go b/internal/modules/dnd/chunk/scenes/schema_test.go deleted file mode 100644 index 8b12b9c2..00000000 --- a/internal/modules/dnd/chunk/scenes/schema_test.go +++ /dev/null @@ -1,179 +0,0 @@ -package scenes - -import ( - "bytes" - "encoding/json" - "fmt" - "strings" - "testing" - - "github.com/santhosh-tekuri/jsonschema/v6" -) - -func TestLoadResponseSchemaForScenes(t *testing.T) { - schema, err := loadResponseSchema() - if err != nil { - t.Fatalf("loadResponseSchema() error = %v, want nil", err) - } - if got, want := string(schema.Key), "dnd_scenes_llm"; got != want { - t.Fatalf("schema.Key = %q, want %q", got, want) - } - if got, want := schema.ID, "notarius.dnd.scenes.llm"; got != want { - t.Fatalf("schema.ID = %q, want %q", got, want) - } - if got, want := schema.Version, "v1"; got != want { - t.Fatalf("schema.Version = %q, want %q", got, want) - } - if got, want := schema.Name, "notarius_dnd_scenes_llm_v1"; got != want { - t.Fatalf("schema.Name = %q, want %q", got, want) - } - if !strings.HasPrefix(schema.SHA256, "sha256:") { - t.Fatalf("schema.SHA256 = %q, want sha256 prefix", schema.SHA256) - } - if !json.Valid(schema.JSONSchema) { - t.Fatalf("schema.JSONSchema is invalid JSON: %s", schema.JSONSchema) - } -} - -func TestResponseSchemaValidatesMinimalSceneResponses(t *testing.T) { - schema, err := loadResponseSchema() - if err != nil { - t.Fatalf("loadResponseSchema() error = %v, want nil", err) - } - - valid := validSceneSchemaResponse() - validJSON, err := json.Marshal(valid) - if err != nil { - t.Fatalf("Marshal(validSceneSchemaResponse()) error = %v, want nil", err) - } - if err := validateJSONSchema(validJSON, schema.JSONSchema); err != nil { - t.Fatalf("valid scene response rejected: %v", err) - } - - tests := []struct { - name string - mutate func(map[string]any) - }{ - { - name: "missing scenes", - mutate: func(response map[string]any) { - delete(response, "scenes") - }, - }, - { - name: "empty scenes", - mutate: func(response map[string]any) { - response["scenes"] = []any{} - }, - }, - { - name: "non-positive start unit", - mutate: func(response map[string]any) { - response["scenes"].([]any)[0].(map[string]any)["start_unit_id"] = 0 - }, - }, - { - name: "non-positive end unit", - mutate: func(response map[string]any) { - response["scenes"].([]any)[0].(map[string]any)["end_unit_id"] = 0 - }, - }, - { - name: "non-integer endpoint", - mutate: func(response map[string]any) { - response["scenes"].([]any)[0].(map[string]any)["start_unit_id"] = 1.5 - }, - }, - { - name: "unknown top-level field", - mutate: func(response map[string]any) { - response["boundary_caveats"] = []any{} - }, - }, - { - name: "unknown scene field", - mutate: func(response map[string]any) { - response["scenes"].([]any)[0].(map[string]any)["short_title"] = "Old contract" - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - response := validSceneSchemaResponse() - tt.mutate(response) - content, err := json.Marshal(response) - if err != nil { - t.Fatalf("Marshal() error = %v, want nil", err) - } - if err := validateJSONSchema(content, schema.JSONSchema); err == nil { - t.Fatal("validateJSONSchema() error = nil, want rejected response") - } - }) - } -} - -func TestResponseStructAcceptsIntegerBoundaries(t *testing.T) { - raw := []byte(`{"scenes":[{"start_unit_id":1,"end_unit_id":3}]}`) - - var response chunkResponse - if err := json.Unmarshal(raw, &response); err != nil { - t.Fatalf("Unmarshal() error = %v, want nil", err) - } - if got := response.Scenes[0].StartUnitID.String(); got != "1" { - t.Fatalf("StartUnitID = %q, want 1", got) - } - if got := response.Scenes[0].EndUnitID.String(); got != "3" { - t.Fatalf("EndUnitID = %q, want 3", got) - } -} - -func TestResponseSchemaJSONIsMutationSafe(t *testing.T) { - first, err := loadResponseSchema() - if err != nil { - t.Fatalf("loadResponseSchema() error = %v, want nil", err) - } - first.JSONSchema[0] = '[' - - second, err := loadResponseSchema() - if err != nil { - t.Fatalf("loadResponseSchema() error = %v, want nil", err) - } - if !json.Valid(second.JSONSchema) { - t.Fatalf("schema JSON was mutated: %s", second.JSONSchema) - } - if len(second.JSONSchema) > 0 && second.JSONSchema[0] == '[' { - t.Fatal("schema JSON did not use defensive copy") - } -} - -func validSceneSchemaResponse() map[string]any { - return map[string]any{ - "scenes": []any{ - map[string]any{ - "start_unit_id": 1, - "end_unit_id": 3, - }, - }, - } -} - -func validateJSONSchema(instanceContent, schemaContent []byte) error { - instance, err := jsonschema.UnmarshalJSON(bytes.NewReader(instanceContent)) - if err != nil { - return fmt.Errorf("parse instance: %w", err) - } - schemaDocument, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaContent)) - if err != nil { - return fmt.Errorf("parse schema: %w", err) - } - compiler := jsonschema.NewCompiler() - if err := compiler.AddResource("schema.json", schemaDocument); err != nil { - return fmt.Errorf("load schema: %w", err) - } - schema, err := compiler.Compile("schema.json") - if err != nil { - return fmt.Errorf("compile schema: %w", err) - } - return schema.Validate(instance) -}