diff --git a/internal/modules/dnd/validate/scenedescriptions/combat_semantics/schema_test.go b/assets/combat_semantics_test.go similarity index 56% rename from internal/modules/dnd/validate/scenedescriptions/combat_semantics/schema_test.go rename to assets/combat_semantics_test.go index 522bd3ec..a546e4ac 100644 --- a/internal/modules/dnd/validate/scenedescriptions/combat_semantics/schema_test.go +++ b/assets/combat_semantics_test.go @@ -1,24 +1,58 @@ -package combat_semantics +package assets_test import ( "bytes" + "context" "encoding/json" + "gitea.maximumdirect.net/eric/notarius/assets" + "io/fs" "strings" "testing" + "go.maximumdirect.net/promptkit" + "github.com/santhosh-tekuri/jsonschema/v6" ) -func TestLoadResponseSchemaUsesStrictCombatSemanticsContract(t *testing.T) { - schema, err := loadResponseSchema() +func TestCombatSemanticsPromptKeepsItsSceneLocalPolicyAndInput(t *testing.T) { + _, engine := loadNativeAssets(t) + const transcript = `{"sentinel":"combat-semantics-transcript"}` + handle, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{ + PromptID: "notarius/dnd.scene_descriptions.validate_combat", APIKey: "test-only", + Inputs: map[string]promptkit.ArtifactRef{"transcript": promptkit.Inline(transcript)}, + }) + if err != nil { + t.Fatal(err) + } + defer handle.Discard() + details := handle.Details() + if details.SelectedProfileID != "notarius/dnd-extraction" || details.StructuredOutput == nil { + t.Fatal("native validator defaults/output contract lost") + } + var all string + for _, message := range details.Messages { + all += message.Content + if strings.Contains(message.Content, transcript) && (message.CacheControl == nil || message.CacheControl.Type != promptkit.CacheControlEphemeral) { + t.Fatal("chunk transcript native cache hint lost") + } + } + for _, content := range []string{"substantive active combat materially organizes the", "Do not judge the scene title", transcript} { + if strings.Count(all, content) != 1 { + t.Fatalf("scene-local context not rendered once: %q", content) + } + } + if _, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "notarius/dnd.scene_descriptions.validate_combat", APIKey: "test-only"}); err == nil { + t.Fatal("native validator accepted missing transcript") + } +} + +func TestCombatSemanticsResponseSchemaRetainsClassificationAndExplanationRules(t *testing.T) { + schema, err := fs.ReadFile(assets.FS(), "dnd/scene-descriptions/validate/combat-semantics/schemas/dnd_scene_combat_semantics_llm.v1.json") if err != nil { t.Fatalf("loadResponseSchema() error = %v, want nil", err) } - if schema.Key != ResponseSchemaKey || schema.ID != ResponseSchemaID || schema.Version != SchemaVersion || schema.Name != ResponseSchemaName || !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) { - t.Fatalf("schema = %#v, want private combat-semantics schema identity", schema) - } for _, classification := range []string{"combat", "non_combat"} { - if err := validateCombatSemanticsSchema(map[string]any{"classification": classification, "explanation": "The chunk contains sustained attack exchanges."}, schema.JSONSchema); err != nil { + if err := validateCombatSemanticsSchema(map[string]any{"classification": classification, "explanation": "The chunk contains sustained attack exchanges."}, schema); err != nil { t.Fatalf("valid %q classification rejected: %v", classification, err) } } @@ -36,29 +70,11 @@ func TestLoadResponseSchemaUsesStrictCombatSemanticsContract(t *testing.T) { {name: "oversized explanation", response: map[string]any{"classification": "combat", "explanation": strings.Repeat("a", 513)}}, } { t.Run(test.name, func(t *testing.T) { - if err := validateCombatSemanticsSchema(test.response, schema.JSONSchema); err == nil { + if err := validateCombatSemanticsSchema(test.response, schema); err == nil { t.Fatal("invalid private response accepted") } }) } - if bytes.Contains(schema.JSONSchema, []byte("uniqueItems")) { - t.Fatalf("schema included unsupported uniqueItems constraint: %s", schema.JSONSchema) - } -} - -func TestResponseSchemaIsMutationSafeAndDiagnosticsRedactContent(t *testing.T) { - first, err := loadResponseSchema() - if err != nil { - t.Fatal(err) - } - first.JSONSchema[0] = '[' - second, err := loadResponseSchema() - if err != nil || !strings.HasPrefix(second.SHA256, "sha256:") || !json.Valid(second.JSONSchema) || bytes.Equal(first.JSONSchema, second.JSONSchema) { - t.Fatalf("second schema = %s, %v; want defensive copy", second.JSONSchema, err) - } - if diagnostics := second.DiagnosticsMap(); diagnostics["json_schema"] != nil { - t.Fatalf("schema diagnostics included raw content: %#v", diagnostics) - } } func validateCombatSemanticsSchema(instance map[string]any, schemaContent []byte) error { diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 2870d2cd..def8339b 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -764,6 +764,29 @@ units, full-source fallback, final-turn acceptance, and cancellation. ## Stage 17: Migrate The Combat-Semantics LLM Validator +**Status:** Complete. The validator now makes one native request on the shared +engine, preserving the validator profile/default, session, execution overrides, +transient-retry override, observer, and preparation callback. Static prompt +dependencies replace asset/schema registration and manifest/fingerprint metadata. +Native errors and failed validation status are checked before decoding; execution +and interpretation failures produce an unavailable assessment with inspectable +causes, not semantic rejection or invented correction guidance. Cancellation +does not deliver a judgment. Successful classification retains the original +combat policy, scene-local transcript boundary, explanation checks, and actionable +producer guidance. Interpretation and its existing cases are independently +testable; evaluation fixtures and schema/policy assets are unchanged. +Added native provider tests for judgments, override/input isolation, nil-error +failed validation, unavailable provider/capacity, interpretation failure, +cancellation, and nested same-backend admission. A runnable framework contract +proves complete ordered native assessment on a backend limited to one active call, +without holding producer admission or repeating the final chain. +Asset tests/vet/race, independent interpretation/evaluation tests/vet/race, +focused pipeline tests/vet/race, and `git diff --check` pass. The concrete +validator package remains blocked at setup by shared legacy asset helpers +importing old ConfigKit's removed YAML `plugin/limit`; its native runtime tests +are not claimed passing. Scheduled dependency cleanup and final verification +must execute them. No validator-owned semantic continuation loop was added. + **Goal:** validators use native execution without another semantic retry loop. **Inspect/change:** `internal/modules/dnd/validate/scenedescriptions/combat_semantics`, diff --git a/internal/framework/pipeline/native_assessment_test.go b/internal/framework/pipeline/native_assessment_test.go new file mode 100644 index 00000000..2049a0db --- /dev/null +++ b/internal/framework/pipeline/native_assessment_test.go @@ -0,0 +1,58 @@ +package pipeline + +import ( + "context" + "fmt" + "reflect" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "go.maximumdirect.net/configkit" + "go.maximumdirect.net/promptkit" +) + +func TestNativeAssessmentChainSharesOneLimitedBackendWithoutHoldingProducerAdmission(t *testing.T) { + calls := 0 + engine := nativeProducerEngine(t, producerClientFunc(func(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + if request.Target.BackendID != "limited" { + t.Fatal("fixture did not select the shared backend") + } + calls++ + return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil + }), promptkit.SettingsInput(configkit.Bytes("shared assessment backend", []byte(`backends: + - id: limited + endpoint: https://example.invalid/v1 + concurrency_limit: 1 + queue_capacity: 0 +profiles: + - id: base + backend: limited + model: assessment-model + api_key_required: false +`)))) + chain := preparedValidatorChain{} + for _, name := range []string{"first-native", "second-native"} { + chain.validators = append(chain.validators, preparedValidator{resolved: ResolvedValidator{Binding: Binding(name)}}) + } + var order []string + operation := contracts.ProducerOperation[int]{ + Controls: contracts.ProducerControls{Mode: contracts.ModeStrict}, ReuseEligible: true, + Assess: func(ctx context.Context, _ int) (contracts.CandidateAssessment, error) { + return executeValidationChain(ctx, chain, func(ctx context.Context, validator preparedValidator) (contracts.ValidationResult, error) { + order = append(order, validator.resolved.Binding.Module) + result, err := engine.Run(ctx, nativeProducerInitial()) + if err != nil { + return contracts.ValidationResult{}, &contracts.ValidationUnavailableError{Cause: err} + } + if !result.Validation.IsValid { + return contracts.ValidationResult{}, &contracts.ValidationUnavailableError{Cause: fmt.Errorf("native contract failed")} + } + return contracts.ValidationResult{Approved: true}, nil + }) + }, + } + value, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, nil) + if err != nil || value != 1 || outcome.Disposition != contracts.ProducerAccepted || calls != 3 || !reflect.DeepEqual(order, []string{"first-native", "second-native"}) { + t.Fatalf("nested assessment=%d %#v %v, native calls=%d order=%v", value, outcome, err, calls, order) + } +} diff --git a/internal/framework/pipeline/producer_test.go b/internal/framework/pipeline/producer_test.go index 8a7b71ea..59c8e8d2 100644 --- a/internal/framework/pipeline/producer_test.go +++ b/internal/framework/pipeline/producer_test.go @@ -22,7 +22,7 @@ func (f producerClientFunc) Generate(ctx context.Context, request promptkit.Gene return f(ctx, request) } -func nativeProducerEngine(t *testing.T, client promptkit.LLMClient) *promptkit.Engine { +func nativeProducerEngine(t *testing.T, client promptkit.LLMClient, extraInputs ...configkit.Input) *promptkit.Engine { t.Helper() settings := `profiles: - id: base @@ -52,8 +52,12 @@ prompts: - {role: user, content: 'Correct: {{input "guidance"}}'} output: {format: json, validation_mode: json, repair_attempts: 0} ` + underlays := []configkit.Underlay{{Inputs: []configkit.Input{promptkit.SettingsInput(configkit.Bytes("producer fixture", []byte(settings)))}}} + if len(extraInputs) > 0 { + underlays = append(underlays, configkit.Underlay{Inputs: extraInputs}) + } engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ - Underlays: []configkit.Underlay{{Inputs: []configkit.Input{promptkit.SettingsInput(configkit.Bytes("producer fixture", []byte(settings)))}}}, + Underlays: underlays, }}, promptkit.WithBundledOpenRouter(false), promptkit.WithLLMClient(client)) if err != nil { t.Fatal(err) diff --git a/internal/modules/dnd/validate/scenedescriptions/combat_semantics/evaluation_cases_test.go b/internal/modules/dnd/validate/scenedescriptions/combat_semantics/evaluation_cases_test.go new file mode 100644 index 00000000..a71cc934 --- /dev/null +++ b/internal/modules/dnd/validate/scenedescriptions/combat_semantics/evaluation_cases_test.go @@ -0,0 +1,61 @@ +package combat_semantics + +import ( + "encoding/json" + "errors" + "io" + "os" + "strings" + "testing" +) + +func TestEvaluationCasesAreStrictAndCoverClassifications(t *testing.T) { + content, err := os.ReadFile("testdata/evaluation_cases.json") + if err != nil { + t.Fatal(err) + } + var cases []struct { + Name string `json:"name"` + TranscriptUnits []struct { + ID int `json:"id"` + Text string `json:"text"` + } `json:"transcript_units"` + ProposedKind string `json:"proposed_kind"` + ExpectedClassification string `json:"expected_classification"` + ReviewerRationale string `json:"reviewer_rationale"` + } + decoder := json.NewDecoder(strings.NewReader(string(content))) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&cases); err != nil { + t.Fatalf("decode evaluation cases: %v", err) + } + if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + t.Fatalf("decode trailing evaluation data: %v, want EOF", err) + } + seenNames, seenClassifications := map[string]bool{}, map[string]bool{} + for _, value := range cases { + if strings.TrimSpace(value.Name) == "" || seenNames[value.Name] || len(value.TranscriptUnits) == 0 || strings.TrimSpace(value.ProposedKind) == "" || strings.TrimSpace(value.ReviewerRationale) == "" { + t.Fatalf("invalid evaluation case: %#v", value) + } + seenNames[value.Name] = true + switch value.ProposedKind { + case "combat", "narrative", "recap", "meta": + default: + t.Fatalf("unsupported proposed kind %q", value.ProposedKind) + } + switch value.ExpectedClassification { + case "combat", "non_combat": + seenClassifications[value.ExpectedClassification] = true + default: + t.Fatalf("unsupported classification %q", value.ExpectedClassification) + } + for _, unit := range value.TranscriptUnits { + if unit.ID <= 0 || strings.TrimSpace(unit.Text) == "" { + t.Fatalf("invalid transcript unit: %#v", unit) + } + } + } + if len(seenClassifications) != 2 { + t.Fatalf("evaluation classifications = %#v, want both classes", seenClassifications) + } +} diff --git a/internal/modules/dnd/validate/scenedescriptions/combat_semantics/interpretation.go b/internal/modules/dnd/validate/scenedescriptions/combat_semantics/interpretation.go new file mode 100644 index 00000000..745f68c2 --- /dev/null +++ b/internal/modules/dnd/validate/scenedescriptions/combat_semantics/interpretation.go @@ -0,0 +1,70 @@ +package combat_semantics + +import ( + "fmt" + "strings" + "unicode/utf8" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" +) + +const ( + ReasonCodeActiveCombatNotClassified = "scene_active_combat_not_classified" + ReasonCodeCombatClassificationUnsupported = "scene_combat_classification_unsupported" + maximumExplanationRunes = 512 +) + +type classificationResponse struct { + Classification string `json:"classification"` + Explanation string `json:"explanation"` +} + +func interpretResponse(response classificationResponse, proposedKind dnd.SceneKind) (contracts.ValidationResult, error) { + explanation, err := validateExplanation(response.Explanation) + if err != nil { + return contracts.ValidationResult{}, err + } + switch response.Classification { + case "combat": + if proposedKind == dnd.SceneKindCombat { + return contracts.ValidationResult{Approved: true}, nil + } + return contracts.ValidationResult{ + Approved: false, + ReasonCode: ReasonCodeActiveCombatNotClassified, + Message: "The current chunk contains substantive active combat that is not classified as combat.", + CorrectionGuidance: "Return kind: combat for this scene. " + explanation, + }, nil + case "non_combat": + if proposedKind != dnd.SceneKindCombat { + return contracts.ValidationResult{Approved: true}, nil + } + return contracts.ValidationResult{ + Approved: false, + ReasonCode: ReasonCodeCombatClassificationUnsupported, + Message: "The current chunk does not support a combat classification.", + CorrectionGuidance: "Choose the appropriate narrative, recap, or meta kind for this scene. " + explanation, + }, nil + default: + return contracts.ValidationResult{}, validatorErrorf("unsupported combat-semantics classification %q", response.Classification) + } +} + +func validateExplanation(value string) (string, error) { + if !utf8.ValidString(value) { + return "", validatorErrorf("validator explanation must be valid UTF-8") + } + trimmed := strings.TrimSpace(value) + if trimmed == "" { + return "", validatorErrorf("validator explanation must not be blank") + } + if utf8.RuneCountInString(trimmed) > maximumExplanationRunes { + return "", validatorErrorf("validator explanation exceeds %d Unicode code points", maximumExplanationRunes) + } + return trimmed, nil +} + +func validatorErrorf(format string, args ...any) error { + return fmt.Errorf("scene combat-semantics validator: "+format, args...) +} diff --git a/internal/modules/dnd/validate/scenedescriptions/combat_semantics/interpretation_test.go b/internal/modules/dnd/validate/scenedescriptions/combat_semantics/interpretation_test.go new file mode 100644 index 00000000..50266de3 --- /dev/null +++ b/internal/modules/dnd/validate/scenedescriptions/combat_semantics/interpretation_test.go @@ -0,0 +1,53 @@ +package combat_semantics + +import ( + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" +) + +func TestValidatorInterpretsCombatSemanticsResponses(t *testing.T) { + for _, test := range []struct { + name string + kind dnd.SceneKind + response classificationResponse + approved bool + reasonCode string + guidance string + correction string + wantError string + }{ + {name: "approve non-combat", kind: dnd.SceneKindNarrative, response: classificationResponse{Classification: "non_combat", Explanation: "No active encounter occurs."}, approved: true}, + {name: "approve combat", kind: dnd.SceneKindCombat, response: classificationResponse{Classification: "combat", Explanation: "Initiative and attacks organize the chunk."}, approved: true}, + {name: "add combat", kind: dnd.SceneKindNarrative, response: classificationResponse{Classification: "combat", Explanation: "The combatants exchange attacks and damage."}, reasonCode: ReasonCodeActiveCombatNotClassified, guidance: "Return kind: combat"}, + {name: "remove combat", kind: dnd.SceneKindCombat, response: classificationResponse{Classification: "non_combat", Explanation: "The group only plans for a possible fight."}, reasonCode: ReasonCodeCombatClassificationUnsupported, guidance: "narrative, recap, or meta"}, + {name: "unknown classification", kind: dnd.SceneKindNarrative, response: classificationResponse{Classification: "uncertain", Explanation: "The group only plans."}, wantError: "unsupported"}, + {name: "blank explanation", kind: dnd.SceneKindNarrative, response: classificationResponse{Classification: "non_combat", Explanation: " "}, wantError: "blank"}, + {name: "trim explanation", kind: dnd.SceneKindNarrative, response: classificationResponse{Classification: "combat", Explanation: " The combatants exchange attacks.\n"}, reasonCode: ReasonCodeActiveCombatNotClassified, correction: "Return kind: combat for this scene. The combatants exchange attacks."}, + {name: "oversized explanation", kind: dnd.SceneKindNarrative, response: classificationResponse{Classification: "non_combat", Explanation: strings.Repeat("界", 513)}, wantError: "exceeds"}, + } { + t.Run(test.name, func(t *testing.T) { + result, err := interpretResponse(test.response, test.kind) + if test.wantError != "" { + if err == nil || !strings.Contains(err.Error(), test.wantError) { + t.Fatalf("Validate() error = %v, want %q", err, test.wantError) + } + return + } + if err != nil || result.Approved != test.approved || result.ReasonCode != test.reasonCode || !strings.Contains(result.CorrectionGuidance, test.guidance) { + t.Fatalf("Validate() = %#v, %v", result, err) + } + if test.correction != "" && result.CorrectionGuidance != test.correction { + t.Fatalf("CorrectionGuidance = %q, want %q", result.CorrectionGuidance, test.correction) + } + if !result.Approved && strings.Contains(result.CorrectionGuidance, result.ReasonCode) { + t.Fatalf("correction guidance leaked opaque application detail: %q", result.CorrectionGuidance) + } + if err := contracts.ValidateValidationResult(result); err != nil { + t.Fatalf("ValidateValidationResult(%#v) = %v", result, err) + } + }) + } +} diff --git a/internal/modules/dnd/validate/scenedescriptions/combat_semantics/native_execution_test.go b/internal/modules/dnd/validate/scenedescriptions/combat_semantics/native_execution_test.go new file mode 100644 index 00000000..421544f9 --- /dev/null +++ b/internal/modules/dnd/validate/scenedescriptions/combat_semantics/native_execution_test.go @@ -0,0 +1,144 @@ +package combat_semantics + +import ( + "context" + "errors" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + "go.maximumdirect.net/promptkit" +) + +func TestNativeValidatorReturnsOnlyCompletedSemanticJudgments(t *testing.T) { + for _, classification := range []string{"combat", "non_combat"} { + for _, proposed := range []dnd.SceneKind{dnd.SceneKindCombat, dnd.SceneKindNarrative} { + request := validRequest(proposed) + client := &fakeCombatSemanticsClient{response: classificationResponse{Classification: classification, Explanation: "The supplied transcript supports this judgment."}} + result, err := newValidator(t, client).Validate(context.Background(), request) + wantApproval := (classification == "combat") == (proposed == dnd.SceneKindCombat) + if err != nil || result.Approved != wantApproval || len(client.requests) != 1 { + t.Fatalf("judgment=%#v %v", result, err) + } + if !wantApproval && (strings.TrimSpace(result.CorrectionGuidance) == "" || strings.Contains(result.CorrectionGuidance, result.ReasonCode)) { + t.Fatal("rejection lost actionable guidance or copied diagnostic labels") + } + } + } +} + +func TestNativeFailuresAreUnavailableRatherThanRejections(t *testing.T) { + failure := errors.New("provider unavailable") + for _, test := range []struct { + name, raw string + failure error + }{ + {"provider failure", "", failure}, + {"invalid JSON", "{", nil}, + {"invalid classification", `{"classification":"uncertain","explanation":"Uncertain."}`, nil}, + {"uninterpretable explanation", `{"classification":"combat","explanation":" "}`, nil}, + } { + t.Run(test.name, func(t *testing.T) { + client := &fakeCombatSemanticsClient{raw: test.raw, err: test.failure} + request := validRequest(dnd.SceneKindCombat) + prepared := 0 + request.Execution.OnPrepared = func(context.Context, promptkit.PreparedRun) { prepared++ } + result, err := newValidator(t, client).Validate(context.Background(), request) + var unavailable *contracts.ValidationUnavailableError + if !errors.As(err, &unavailable) || result.Approved || result.ReasonCode != "" || result.CorrectionGuidance != "" || prepared != 1 { + t.Fatalf("unavailable assessment=%#v %v, native executions=%d", result, err, prepared) + } + if test.failure != nil && (!errors.Is(err, failure) || len(client.requests) != 1) { + t.Fatalf("provider failure was flattened or retried: %v", err) + } + }) + } +} + +func TestNilErrorFailedNativeValidationCannotBecomeApproval(t *testing.T) { + client := &fakeCombatSemanticsClient{raw: `{"classification":"combat"}`} + validator := newValidator(t, client) + request := validRequest(dnd.SceneKindCombat) + native, err := validator.engine.Run(context.Background(), promptkit.RunRequest{ + PromptID: PromptID, ProfileID: request.LLMProfile, + Inputs: map[string]promptkit.ArtifactRef{"transcript": promptkit.Inline(string(request.Chunk.Content))}, + }) + if err != nil || native == nil || native.Validation.Status != promptkit.ValidationFailed || native.Validation.IsValid { + t.Fatalf("fixture did not exercise nil-error structural failure: %#v %v", native, err) + } + result, err := validator.Validate(context.Background(), request) + var unavailable *contracts.ValidationUnavailableError + if !errors.As(err, &unavailable) || !errors.Is(err, promptkit.ErrValidation) || result.Approved || result.CorrectionGuidance != "" { + t.Fatalf("failed native contract accepted/interpreted: %#v %v", result, err) + } +} + +func TestNestedValidatorUsesTheReleasedNativeBackendAdmission(t *testing.T) { + client := &fakeCombatSemanticsClient{response: classificationResponse{Classification: "non_combat", Explanation: "No active encounter occurs."}} + validator := newValidator(t, client) + request := validRequest(dnd.SceneKindNarrative) + assessments := 0 + operation := contracts.ProducerOperation[dnd.SceneDescriptionList]{ + Controls: contracts.ProducerControls{Mode: contracts.ModeStrict}, ReuseEligible: true, + Assess: func(ctx context.Context, candidate dnd.SceneDescriptionList) (contracts.CandidateAssessment, error) { + assessments++ + nested := request + nested.Value = candidate + judgment, err := validator.Validate(ctx, nested) + if err != nil { + return contracts.CandidateAssessment{}, err + } + decision := contracts.AssessmentRejected + if judgment.Approved { + decision = contracts.AssessmentApproved + } + return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{ValidatorName: Key, Decision: decision, Result: judgment}}}, nil + }, + } + _, outcome, err := pipeline.RunProducer(context.Background(), validator.engine, operation, promptkit.RunRequest{ + PromptID: PromptID, ProfileID: request.LLMProfile, + Inputs: map[string]promptkit.ArtifactRef{"transcript": promptkit.Inline(string(request.Chunk.Content))}, + }, func(context.Context, *promptkit.RunResult) (*dnd.SceneDescriptionList, []string, error) { + return &request.Value, nil, nil + }, nil, nil) + if err != nil || outcome.Disposition != contracts.ProducerAccepted || assessments != 1 || len(client.requests) != 2 { + t.Fatalf("nested native admission=%#v %v, assessments=%d calls=%d", outcome, err, assessments, len(client.requests)) + } + for _, call := range client.requests { + if call.Target.BackendID != "validator-backend" { + t.Fatal("fixture did not exercise the same limited backend") + } + } +} + +func TestCapacityFailureAndCancellationNeverBecomeSemanticGuidance(t *testing.T) { + t.Run("capacity unavailable", func(t *testing.T) { + var validator *Validator + var nestedErr error + request := validRequest(dnd.SceneKindNarrative) + client := &fakeCombatSemanticsClient{response: classificationResponse{Classification: "non_combat", Explanation: "No encounter."}} + client.onGenerate = func(ctx context.Context, _ promptkit.GenerateRequest) { + // The outer fake provider still holds the single active native admission. + // The configured zero queue makes this a capacity error, not a waiting test. + _, nestedErr = validator.Validate(ctx, request) + } + validator = newValidator(t, client) + result, err := validator.Validate(context.Background(), request) + var unavailable *contracts.ValidationUnavailableError + if err != nil || !result.Approved || !errors.As(nestedErr, &unavailable) || !errors.Is(nestedErr, promptkit.ErrCapacityExceeded) || len(client.requests) != 1 { + t.Fatalf("capacity assessment=%#v %v, nested=%v", result, err, nestedErr) + } + }) + t.Run("canceled delivery", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + client := &fakeCombatSemanticsClient{response: classificationResponse{Classification: "non_combat", Explanation: "No encounter."}, + onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }} + result, err := newValidator(t, client).Validate(ctx, validRequest(dnd.SceneKindNarrative)) + if !errors.Is(err, context.Canceled) || result.Approved || result.CorrectionGuidance != "" || len(client.requests) != 1 { + t.Fatalf("cancellation=%#v %v", result, err) + } + }) +} diff --git a/internal/modules/dnd/validate/scenedescriptions/combat_semantics/prompt_assets.go b/internal/modules/dnd/validate/scenedescriptions/combat_semantics/prompt_assets.go deleted file mode 100644 index e1a239e0..00000000 --- a/internal/modules/dnd/validate/scenedescriptions/combat_semantics/prompt_assets.go +++ /dev/null @@ -1,72 +0,0 @@ -package combat_semantics - -import ( - "fmt" - "io/fs" - "sync" - - rootassets "gitea.maximumdirect.net/eric/notarius/assets" - "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - "gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs" - "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" -) - -const promptAssetRoot = "assets/prompts" - -var promptAssetManifest = shared.PromptAssetManifest{ - ModuleDir: PromptID, - ModuleFiles: []promptfs.ModulePromptFile{ - {Name: "prompt.yaml", Path: "prompts/prompt.yaml"}, - {Name: "instructions.md", Path: "prompts/instructions.md"}, - }, - SharedFiles: []string{ - "common-dnd-system.md", - "common-dnd-scene-combat-policy.md", - "common-dnd-transcript-chunk.md", - }, -} - -func moduleAssetFS() (fs.FS, error) { - assets, err := fs.Sub(rootassets.FS(), "dnd/scene-descriptions/validate/combat-semantics") - if err != nil { - return nil, fmt.Errorf("scope scene combat-semantics validator assets: %w", err) - } - return assets, nil -} - -func RegisterPromptAssets(registry *llm.AssetRegistry) error { - assets, err := moduleAssetFS() - if err != nil { - return err - } - promptFS, err := promptAssetManifest.PromptFS(assets) - if err != nil { - return fmt.Errorf("prepare scene combat-semantics validator prompt assets: %w", err) - } - if err := registry.RegisterPromptFS(promptFS, promptAssetRoot); err != nil { - return err - } - schemas, err := fs.Sub(assets, "schemas") - if err != nil { - return fmt.Errorf("scope scene combat-semantics validator schemas: %w", err) - } - return registry.RegisterSchemaFS(schemas, ".") -} - -func promptAssetMetadata() (string, error) { - promptAssetHashOnce.Do(func() { - assets, err := moduleAssetFS() - if err != nil { - promptAssetHashErr = err - return - } - promptAssetHash, promptAssetHashErr = promptAssetManifest.Hash(assets) - }) - return promptAssetHash, promptAssetHashErr -} - -var ( - promptAssetHashOnce sync.Once - promptAssetHash string - promptAssetHashErr error -) diff --git a/internal/modules/dnd/validate/scenedescriptions/combat_semantics/prompt_assets_test.go b/internal/modules/dnd/validate/scenedescriptions/combat_semantics/prompt_assets_test.go deleted file mode 100644 index 7be02052..00000000 --- a/internal/modules/dnd/validate/scenedescriptions/combat_semantics/prompt_assets_test.go +++ /dev/null @@ -1,90 +0,0 @@ -package combat_semantics - -import ( - "context" - "strings" - "testing" - "time" - - "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - "gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture" - "gitea.maximumdirect.net/eric/promptkit" -) - -func TestRegisterPromptAssetsPreparesCombatSemanticsPrompt(t *testing.T) { - registry := llm.NewAssetRegistry() - if err := RegisterPromptAssets(registry); err != nil { - t.Fatalf("RegisterPromptAssets() error = %v, want nil", err) - } - engine := newCombatSemanticsPromptEngine(t, registry) - inputs := map[string]promptkit.ArtifactRef{ - "transcript": promptkit.InlineWithURI("file:///session.json", `{"sentinel":"combat-semantics-transcript"}`), - } - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "combat-semantics-test-profile", Inputs: inputs, - }) - if err != nil { - t.Fatalf("Prepare() error = %v, want nil", err) - } - if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_scene_combat_semantics_llm.v1.json" { - t.Fatalf("prepared prompt = %#v, want combat-semantics prompt identity and schema wiring", prepared) - } - if len(prepared.Messages) != 4 { - t.Fatalf("prepared message count = %d, want 4", len(prepared.Messages)) - } - for index, message := range prepared.Messages { - wantEphemeral := index == 3 - gotEphemeral := message.CacheControl != nil && message.CacheControl.Type == promptkit.CacheControlEphemeral - if gotEphemeral != wantEphemeral { - t.Fatalf("message %d cache control = %#v, want ephemeral=%t", index, message.CacheControl, wantEphemeral) - } - } - assertRenderedExactlyOnce(t, prepared.Messages, "substantive active combat materially organizes the", 1) - assertRenderedExactlyOnce(t, prepared.Messages, "combat-semantics-transcript", 3) - for name := range inputs { - missing := make(map[string]promptkit.ArtifactRef, len(inputs)-1) - for inputName, input := range inputs { - if inputName != name { - missing[inputName] = input - } - } - if _, err := engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "combat-semantics-test-profile", Inputs: missing, - }); err == nil { - t.Fatalf("Prepare() without required %q input = nil error", name) - } - } -} - -func TestPromptAssetMetadataIsContentSafe(t *testing.T) { - hash, err := promptAssetMetadata() - if err != nil || !strings.HasPrefix(hash, "sha256:") { - t.Fatalf("promptAssetMetadata() = %q, %v; want hash", hash, err) - } - if strings.Contains(hash, "non_combat") { - t.Fatalf("prompt asset metadata leaked raw prompt content: %q", hash) - } -} - -func newCombatSemanticsPromptEngine(t *testing.T, registry *llm.AssetRegistry) *promptkit.Engine { - t.Helper() - return promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{ - "combat-semantics-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "combat-semantics-test-model"), - }) -} - -func assertRenderedExactlyOnce(t *testing.T, messages []promptkit.RenderedMessage, sentinel string, wantIndex int) { - t.Helper() - index := -1 - occurrences := 0 - for messageIndex, message := range messages { - count := strings.Count(message.Content, sentinel) - if count > 0 { - index = messageIndex - occurrences += count - } - } - if occurrences != 1 || index != wantIndex { - t.Fatalf("sentinel %q rendered %d times at message %d, want once at message %d", sentinel, occurrences, index, wantIndex) - } -} diff --git a/internal/modules/dnd/validate/scenedescriptions/combat_semantics/schema.go b/internal/modules/dnd/validate/scenedescriptions/combat_semantics/schema.go deleted file mode 100644 index 265cda61..00000000 --- a/internal/modules/dnd/validate/scenedescriptions/combat_semantics/schema.go +++ /dev/null @@ -1,25 +0,0 @@ -package combat_semantics - -import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - -const ( - PromptID = "dnd.scene_descriptions.validate_combat" - ResponseSchemaKey = llm.ResponseSchemaKey("dnd_scene_combat_semantics_llm") - ResponseSchemaID = "notarius.dnd.scene_descriptions.combat_semantics.llm" - ResponseSchemaName = "notarius_dnd_scene_combat_semantics_llm_v1" - SchemaVersion = "v1" -) - -func loadResponseSchema() (llm.ResponseSchema, error) { - assets, err := moduleAssetFS() - if err != nil { - return llm.ResponseSchema{}, err - } - return llm.LoadResponseSchema(assets, llm.ResponseSchemaDefinition{ - Key: ResponseSchemaKey, - ID: ResponseSchemaID, - Version: SchemaVersion, - Name: ResponseSchemaName, - AssetPath: "schemas/dnd_scene_combat_semantics_llm.v1.json", - }) -} diff --git a/internal/modules/dnd/validate/scenedescriptions/combat_semantics/validator.go b/internal/modules/dnd/validate/scenedescriptions/combat_semantics/validator.go index 832b8d09..e8c18688 100644 --- a/internal/modules/dnd/validate/scenedescriptions/combat_semantics/validator.go +++ b/internal/modules/dnd/validate/scenedescriptions/combat_semantics/validator.go @@ -2,9 +2,10 @@ package combat_semantics import ( "context" - "fmt" + "encoding/json" "strings" - "unicode/utf8" + + "go.maximumdirect.net/promptkit" "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" @@ -14,44 +15,20 @@ import ( "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/shape" ) -const ( - Key = "extract/dnd/scene-descriptions/combat_semantics" - ReasonCodeActiveCombatNotClassified = "scene_active_combat_not_classified" - ReasonCodeCombatClassificationUnsupported = "scene_combat_classification_unsupported" - combatPolicy = "dnd.scene_descriptions.combat_semantics.v1" - maximumExplanationRunes = 512 -) +const Key = "extract/dnd/scene-descriptions/combat_semantics" +const PromptID = "notarius/dnd.scene_descriptions.validate_combat" type Options struct{} -type completionResponse struct { - Classification string `json:"classification"` - Explanation string `json:"explanation"` -} - -type Validator struct { - llm contracts.StructuredLLMClient - promptSHA string - responseSchemaSHA string -} +type Validator struct{ engine *promptkit.Engine } var _ contracts.TypedValidator[dnd.SceneDescriptionList] = (*Validator)(nil) -var _ contracts.ManifestMetadataProvider = (*Validator)(nil) -var _ pipeline.CheckpointFingerprintProvider = (*Validator)(nil) -func New(llmClient contracts.StructuredLLMClient, _ Options) (*Validator, error) { - if llmClient == nil { - return nil, validatorErrorf("LLM client must not be nil") +func New(engine *promptkit.Engine, _ Options) (*Validator, error) { + if engine == nil { + return nil, validatorErrorf("native engine must not be nil") } - promptSHA, err := promptAssetMetadata() - if err != nil { - return nil, validatorErrorf("load prompt metadata: %w", err) - } - responseSchema, err := loadResponseSchema() - if err != nil { - return nil, validatorErrorf("load response schema: %w", err) - } - return &Validator{llm: llmClient, promptSHA: promptSHA, responseSchemaSHA: responseSchema.SHA256}, nil + return &Validator{engine: engine}, nil } func (v *Validator) Name() string { return Key } @@ -60,62 +37,46 @@ func (v *Validator) ExecutionClass() contracts.ExecutionClass { return contracts.ExecutionClassLLMBacked } -func (v *Validator) ManifestMetadata() map[string]any { - if v == nil { - return nil - } - return map[string]any{ - "prompt_id": PromptID, - "prompt_version": SchemaVersion, - "prompt_sha256": v.promptSHA, - "response_schema_key": string(ResponseSchemaKey), - "response_schema_id": ResponseSchemaID, - "response_schema_name": ResponseSchemaName, - "response_schema_version": SchemaVersion, - "response_schema_sha256": v.responseSchemaSHA, - "combat_policy": combatPolicy, - } -} - -func (v *Validator) CheckpointFingerprints() []pipeline.CheckpointFingerprint { - if v == nil { - return nil - } - return []pipeline.CheckpointFingerprint{ - {Name: "prompt", Value: v.promptSHA}, - {Name: "response_schema", Value: v.responseSchemaSHA}, - {Name: "combat_policy", Value: combatPolicy}, - } -} - func (v *Validator) Validate(ctx context.Context, req contracts.TypedValidationRequest[dnd.SceneDescriptionList]) (contracts.ValidationResult, error) { if v == nil { return contracts.ValidationResult{}, validatorErrorf("validator must not be nil") } - if v.llm == nil { - return contracts.ValidationResult{}, validatorErrorf("LLM client must not be nil") + if v.engine == nil { + return contracts.ValidationResult{}, validatorErrorf("native engine must not be nil") } sourceInput, scene, err := validateRequest(ctx, req) if err != nil { return contracts.ValidationResult{}, err } - var response completionResponse - _, err = v.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{ - StageName: Key, - PromptID: PromptID, - PromptVersion: SchemaVersion, - ProfileID: req.LLMProfile, - SessionID: req.SessionID, - StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts, - Inputs: contracts.LLMInputSet{ - "transcript": shared.TranscriptPromptMaterial(sourceInput), - }, - }, &response) - if err != nil { - return contracts.ValidationResult{}, validatorErrorf("complete structured output: %w", err) + material := shared.TranscriptPromptMaterial(sourceInput) + result, err := v.engine.Run(ctx, promptkit.RunRequest{ + PromptID: PromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID, + Inputs: map[string]promptkit.ArtifactRef{"transcript": promptkit.InlineWithURI(material.OriginURI, string(material.Content))}, + Execution: req.Execution.Execution, TransientRetries: req.Execution.TransientRetries, + Observer: req.Execution.Observer, OnPrepared: req.Execution.OnPrepared, + }) + if canceled := ctx.Err(); canceled != nil { + return contracts.ValidationResult{}, canceled } - return interpretResponse(response, scene.Kind) + if err != nil { + return contracts.ValidationResult{}, unavailable(err) + } + if result == nil || result.Validation.Status == promptkit.ValidationFailed || !result.Validation.IsValid { + return contracts.ValidationResult{}, unavailable(validatorErrorf("native output contract failed: %w", promptkit.ErrValidation)) + } + var response classificationResponse + if err := json.Unmarshal([]byte(result.RawOutput), &response); err != nil { + return contracts.ValidationResult{}, unavailable(validatorErrorf("decode native output: %w", err)) + } + assessment, err := interpretResponse(response, scene.Kind) + if canceled := ctx.Err(); canceled != nil { + return contracts.ValidationResult{}, canceled + } + if err != nil { + return contracts.ValidationResult{}, unavailable(err) + } + return assessment, nil } func validateRequest(ctx context.Context, req contracts.TypedValidationRequest[dnd.SceneDescriptionList]) (contracts.LLMInputMaterial, dnd.SceneDescription, error) { @@ -128,7 +89,7 @@ func validateRequest(ctx context.Context, req contracts.TypedValidationRequest[d if req.Stage != string(pipeline.StageExtract) { return contracts.LLMInputMaterial{}, dnd.SceneDescription{}, validatorErrorf("validation stage must be extract, got %q", req.Stage) } - sourceInput, err := shared.PrepareChunkExtraction(ctx, contracts.TypedExtractionRequest{Source: req.Source, Chunk: req.Chunk, SourceInput: req.SourceInput}) + sourceInput, err := shared.PrepareChunkExtraction(ctx, contracts.TypedExtractionRequest[dnd.SceneDescriptionList]{Source: req.Source, Chunk: req.Chunk, SourceInput: req.SourceInput}) if err != nil { return contracts.LLMInputMaterial{}, dnd.SceneDescription{}, validatorErrorf("prepare chunk source input: %w", err) } @@ -154,53 +115,12 @@ func validateRequest(ctx context.Context, req contracts.TypedValidationRequest[d return sourceInput, scene, nil } -func interpretResponse(response completionResponse, proposedKind dnd.SceneKind) (contracts.ValidationResult, error) { - explanation, err := validateExplanation(response.Explanation) - if err != nil { - return contracts.ValidationResult{}, err - } - switch response.Classification { - case "combat": - if proposedKind == dnd.SceneKindCombat { - return contracts.ValidationResult{Approved: true}, nil - } - return contracts.ValidationResult{ - Approved: false, - ReasonCode: ReasonCodeActiveCombatNotClassified, - Message: "The current chunk contains substantive active combat that is not classified as combat.", - CorrectionGuidance: "Return kind: combat for this scene. " + explanation, - }, nil - case "non_combat": - if proposedKind != dnd.SceneKindCombat { - return contracts.ValidationResult{Approved: true}, nil - } - return contracts.ValidationResult{ - Approved: false, - ReasonCode: ReasonCodeCombatClassificationUnsupported, - Message: "The current chunk does not support a combat classification.", - CorrectionGuidance: "Choose the appropriate narrative, recap, or meta kind for this scene. " + explanation, - }, nil - default: - return contracts.ValidationResult{}, validatorErrorf("unsupported combat-semantics classification %q", response.Classification) - } -} - -func validateExplanation(value string) (string, error) { - if !utf8.ValidString(value) { - return "", validatorErrorf("validator explanation must be valid UTF-8") - } - trimmed := strings.TrimSpace(value) - if trimmed == "" { - return "", validatorErrorf("validator explanation must not be blank") - } - if utf8.RuneCountInString(trimmed) > maximumExplanationRunes { - return "", validatorErrorf("validator explanation exceeds %d Unicode code points", maximumExplanationRunes) - } - return trimmed, nil +func unavailable(cause error) error { + return &contracts.ValidationUnavailableError{Cause: validatorErrorf("native assessment unavailable: %w", cause)} } func Spec() pipeline.ValidatorSpec { - return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassLLMBacked} + return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{PromptID}} } func Register(registry *pipeline.ValidatorRegistry) error { @@ -209,7 +129,7 @@ func Register(registry *pipeline.ValidatorRegistry) error { if err != nil { return nil, err } - return New(request.Dependencies.LLM, options) + return New(request.Dependencies.Engine, options) }) } @@ -221,7 +141,3 @@ func DecodeOptions(options map[string]any) (Options, error) { } func validateOptions(options map[string]any) error { _, err := DecodeOptions(options); return err } - -func validatorErrorf(format string, args ...any) error { - return fmt.Errorf("scene combat-semantics validator: "+format, args...) -} diff --git a/internal/modules/dnd/validate/scenedescriptions/combat_semantics/validator_test.go b/internal/modules/dnd/validate/scenedescriptions/combat_semantics/validator_test.go index 5956dacb..f449bf61 100644 --- a/internal/modules/dnd/validate/scenedescriptions/combat_semantics/validator_test.go +++ b/internal/modules/dnd/validate/scenedescriptions/combat_semantics/validator_test.go @@ -4,75 +4,31 @@ import ( "context" "encoding/json" "errors" - "io" - "os" "reflect" "strings" "testing" + "gitea.maximumdirect.net/eric/notarius/assets" "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + "go.maximumdirect.net/configkit" + "go.maximumdirect.net/promptkit" ) -func TestValidatorInterpretsCombatSemanticsResponses(t *testing.T) { - request := validRequest(dnd.SceneKindNarrative) - for _, test := range []struct { - name string - kind dnd.SceneKind - response completionResponse - approved bool - reasonCode string - guidance string - correction string - wantError string - }{ - {name: "approve non-combat", kind: dnd.SceneKindNarrative, response: completionResponse{Classification: "non_combat", Explanation: "No active encounter occurs."}, approved: true}, - {name: "approve combat", kind: dnd.SceneKindCombat, response: completionResponse{Classification: "combat", Explanation: "Initiative and attacks organize the chunk."}, approved: true}, - {name: "add combat", kind: dnd.SceneKindNarrative, response: completionResponse{Classification: "combat", Explanation: "The combatants exchange attacks and damage."}, reasonCode: ReasonCodeActiveCombatNotClassified, guidance: "Return kind: combat"}, - {name: "remove combat", kind: dnd.SceneKindCombat, response: completionResponse{Classification: "non_combat", Explanation: "The group only plans for a possible fight."}, reasonCode: ReasonCodeCombatClassificationUnsupported, guidance: "narrative, recap, or meta"}, - {name: "unknown classification", kind: dnd.SceneKindNarrative, response: completionResponse{Classification: "uncertain", Explanation: "The group only plans."}, wantError: "unsupported"}, - {name: "blank explanation", kind: dnd.SceneKindNarrative, response: completionResponse{Classification: "non_combat", Explanation: " "}, wantError: "blank"}, - {name: "trim explanation", kind: dnd.SceneKindNarrative, response: completionResponse{Classification: "combat", Explanation: " The combatants exchange attacks.\n"}, reasonCode: ReasonCodeActiveCombatNotClassified, correction: "Return kind: combat for this scene. The combatants exchange attacks."}, - {name: "oversized explanation", kind: dnd.SceneKindNarrative, response: completionResponse{Classification: "non_combat", Explanation: strings.Repeat("界", 513)}, wantError: "exceeds"}, - } { - t.Run(test.name, func(t *testing.T) { - request.Value.Scenes[0].Kind = test.kind - client := &fakeCombatSemanticsClient{response: test.response} - validator := newValidator(t, client) - result, err := validator.Validate(context.Background(), request) - if test.wantError != "" { - if err == nil || !strings.Contains(err.Error(), test.wantError) { - t.Fatalf("Validate() error = %v, want %q", err, test.wantError) - } - return - } - if err != nil || result.Approved != test.approved || result.ReasonCode != test.reasonCode || !strings.Contains(result.CorrectionGuidance, test.guidance) { - t.Fatalf("Validate() = %#v, %v", result, err) - } - if test.correction != "" && result.CorrectionGuidance != test.correction { - t.Fatalf("CorrectionGuidance = %q, want %q", result.CorrectionGuidance, test.correction) - } - if !result.Approved && (strings.Contains(result.CorrectionGuidance, Key) || strings.Contains(result.CorrectionGuidance, result.ReasonCode) || strings.Contains(result.CorrectionGuidance, request.Chunk.ID)) { - t.Fatalf("correction guidance leaked opaque application detail: %q", result.CorrectionGuidance) - } - if err := contracts.ValidateValidationResult(result); err != nil { - t.Fatalf("ValidateValidationResult(%#v) = %v", result, err) - } - }) - } -} - func TestValidatorUsesOnlyChunkTranscript(t *testing.T) { request := validRequest(dnd.SceneKindNarrative) - repairAttempts := 2 request.LLMProfile = "validator-profile" request.SessionID = "validator-session" - request.StructuredOutputRepairAttempts = &repairAttempts request.References = contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{"players": {Items: []contracts.ReferenceItem{{Content: []byte("hidden reference")}}}}} - client := &fakeCombatSemanticsClient{response: completionResponse{Classification: "non_combat", Explanation: "No active encounter occurs."}} + client := &fakeCombatSemanticsClient{response: classificationResponse{Classification: "non_combat", Explanation: "No active encounter occurs."}} validator := newValidator(t, client) + model := "validator-override" + request.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: model} + var prepared promptkit.PreparedRun + preparedCalls := 0 + request.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = value; preparedCalls++ } if result, err := validator.Validate(context.Background(), request); err != nil || !result.Approved { t.Fatalf("Validate() = %#v, %v", result, err) } @@ -80,19 +36,22 @@ func TestValidatorUsesOnlyChunkTranscript(t *testing.T) { t.Fatalf("completion count = %d, want 1", len(client.requests)) } completed := client.requests[0] - if completed.StageName != Key || completed.PromptID != PromptID || completed.PromptVersion != SchemaVersion || completed.ProfileID != request.LLMProfile || completed.SessionID != request.SessionID || completed.StructuredOutputRepairAttempts == request.StructuredOutputRepairAttempts || *completed.StructuredOutputRepairAttempts != repairAttempts { - t.Fatalf("completion request = %#v", completed) + if preparedCalls != 1 || prepared.PromptID != PromptID || prepared.SelectedProfileID != request.LLMProfile || completed.Prompt.SessionID != request.SessionID || completed.Target.Model != model { + t.Fatal("native execution selection/override missing") } - if completed.Correction != nil || !reflect.DeepEqual(sortedInputNames(completed.Inputs), []string{"transcript"}) || !reflect.DeepEqual(completed.Inputs["transcript"].Content, request.Chunk.Content) { - t.Fatalf("completion inputs = %#v", completed.Inputs) + var messages string + for _, message := range completed.Prompt.Messages { + messages += message.Content + } + if strings.Count(messages, string(request.Chunk.Content)) != 1 { + t.Fatal("chunk transcript not rendered exactly once") } for _, forbidden := range []string{request.Value.Scenes[0].ID, request.Value.Scenes[0].Title, request.Value.Scenes[0].Summary, "hidden reference"} { - for name, input := range completed.Inputs { - if strings.Contains(string(input.Content), forbidden) { - t.Fatalf("input %q leaked forbidden model-visible value %q", name, forbidden) - } + if strings.Contains(messages, forbidden) { + t.Fatalf("model-visible context leaked %q", forbidden) } } + } func TestValidatorRejectsInvalidRequestsAndCompletionFailures(t *testing.T) { @@ -130,7 +89,7 @@ func TestValidatorRejectsInvalidRequestsAndCompletionFailures(t *testing.T) { if test.mutate != nil { test.mutate(&request) } - client := &fakeCombatSemanticsClient{response: completionResponse{Classification: "non_combat", Explanation: "No active encounter occurs."}} + client := &fakeCombatSemanticsClient{response: classificationResponse{Classification: "non_combat", Explanation: "No active encounter occurs."}} _, err := newValidator(t, client).Validate(test.ctx, request) if err == nil || !strings.Contains(err.Error(), test.wantErr) || len(client.requests) != 0 { t.Fatalf("Validate() error = %v, calls = %d; want %q and no calls", err, len(client.requests), test.wantErr) @@ -138,29 +97,26 @@ func TestValidatorRejectsInvalidRequestsAndCompletionFailures(t *testing.T) { }) } client := &fakeCombatSemanticsClient{err: errors.New("provider unavailable")} - if _, err := newValidator(t, client).Validate(context.Background(), base); err == nil || !strings.Contains(err.Error(), "complete structured output") { + if _, err := newValidator(t, client).Validate(context.Background(), base); err == nil || !strings.Contains(err.Error(), "assessment unavailable") { t.Fatalf("completion failure error = %v", err) } } -func TestValidatorConstructionOptionsAndMetadata(t *testing.T) { - if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "LLM client") { +func TestValidatorConstructionOptionsAndDependencies(t *testing.T) { + if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "native engine") { t.Fatalf("New(nil) error = %v", err) } if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil { t.Fatal("DecodeOptions() accepted unknown options") } validator := newValidator(t, &fakeCombatSemanticsClient{}) - if validator.Name() != Key || validator.ExecutionClass() != contracts.ExecutionClassLLMBacked || Spec().CorrectionProtocol != "" { + if validator.Name() != Key || validator.ExecutionClass() != contracts.ExecutionClassLLMBacked || !reflect.DeepEqual(Spec().Prompts, []string{PromptID}) { t.Fatalf("validator contract = %#v / %#v", validator, Spec()) } - metadata, err := json.Marshal(validator.ManifestMetadata()) - if err != nil || !strings.Contains(string(metadata), ResponseSchemaID) || strings.Contains(string(metadata), "non_combat") { - t.Fatalf("manifest metadata = %s, %v", metadata, err) - } - fingerprints := validator.CheckpointFingerprints() - if len(fingerprints) != 3 || fingerprints[0].Name != "prompt" || fingerprints[1].Name != "response_schema" || fingerprints[2] != (pipeline.CheckpointFingerprint{Name: "combat_policy", Value: combatPolicy}) { - t.Fatalf("fingerprints = %#v", fingerprints) + selected := Spec() + selected.Prompts[0] = "mutated" + if !reflect.DeepEqual(Spec().Prompts, []string{PromptID}) { + t.Fatal("prompt dependencies alias caller memory") } registry := pipeline.NewValidatorRegistry() if err := Register(registry); err != nil { @@ -171,60 +127,36 @@ func TestValidatorConstructionOptionsAndMetadata(t *testing.T) { } } -func TestEvaluationCasesAreStrictAndCoverClassifications(t *testing.T) { - content, err := os.ReadFile("testdata/evaluation_cases.json") +func newValidator(t *testing.T, client promptkit.LLMClient) *Validator { + t.Helper() + type settings struct { + PromptKit promptkit.Settings `yaml:"promptkit"` + } + definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit }) + filesystem := configkit.NewFilesystem("Notarius assets", assets.FS()) + result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("validator fixture", []byte(`promptkit: + backends: + - id: validator-backend + endpoint: https://example.invalid/v1 + concurrency_limit: 1 + queue_capacity: 0 + profiles: + - id: validator-profile + backend: validator-backend + model: validator-test-model + api_key_required: false +`))}, configkit.LayeredOptions[settings]{Definition: definition, Underlays: []configkit.Underlay{ + promptkit.BundledOpenRouterUnderlay(), + {Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}}, + }}) if err != nil { t.Fatal(err) } - var cases []struct { - Name string `json:"name"` - TranscriptUnits []struct { - ID int `json:"id"` - Text string `json:"text"` - } `json:"transcript_units"` - ProposedKind string `json:"proposed_kind"` - ExpectedClassification string `json:"expected_classification"` - ReviewerRationale string `json:"reviewer_rationale"` + engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client)) + if err != nil { + t.Fatal(err) } - decoder := json.NewDecoder(strings.NewReader(string(content))) - decoder.DisallowUnknownFields() - if err := decoder.Decode(&cases); err != nil { - t.Fatalf("decode evaluation cases: %v", err) - } - if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { - t.Fatalf("decode trailing evaluation data: %v, want EOF", err) - } - seenNames, seenClassifications := map[string]bool{}, map[string]bool{} - for _, value := range cases { - if strings.TrimSpace(value.Name) == "" || seenNames[value.Name] || len(value.TranscriptUnits) == 0 || strings.TrimSpace(value.ProposedKind) == "" || strings.TrimSpace(value.ReviewerRationale) == "" { - t.Fatalf("invalid evaluation case: %#v", value) - } - seenNames[value.Name] = true - switch value.ProposedKind { - case "combat", "narrative", "recap", "meta": - default: - t.Fatalf("unsupported proposed kind %q", value.ProposedKind) - } - switch value.ExpectedClassification { - case "combat", "non_combat": - seenClassifications[value.ExpectedClassification] = true - default: - t.Fatalf("unsupported classification %q", value.ExpectedClassification) - } - for _, unit := range value.TranscriptUnits { - if unit.ID <= 0 || strings.TrimSpace(unit.Text) == "" { - t.Fatalf("invalid transcript unit: %#v", unit) - } - } - } - if len(seenClassifications) != 2 { - t.Fatalf("evaluation classifications = %#v, want both classes", seenClassifications) - } -} - -func newValidator(t *testing.T, client contracts.StructuredLLMClient) *Validator { - t.Helper() - validator, err := New(client, Options{}) + validator, err := New(engine, Options{}) if err != nil { t.Fatalf("New() error = %v", err) } @@ -234,7 +166,7 @@ func newValidator(t *testing.T, client contracts.StructuredLLMClient) *Validator func validRequest(kind dnd.SceneKind) contracts.TypedValidationRequest[dnd.SceneDescriptionList] { document := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{{ID: 1, Text: "The foes attack."}, {ID: 2, Text: "The party responds."}}} chunk := &source.Chunk{ID: "session:chunk:0", SourceID: document.ID, Ref: source.SourceRef{SourceID: document.ID, StartUnitID: 1, EndUnitID: 2}, Content: []byte(`{"units":[1,2]}`), MediaType: "application/json", Units: append([]source.SourceUnit(nil), document.Units...)} - return contracts.TypedValidationRequest[dnd.SceneDescriptionList]{Stage: string(pipeline.StageExtract), Source: document, Chunk: chunk, SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session.json"), Value: dnd.SceneDescriptionList{Scenes: []dnd.SceneDescription{{ID: chunk.ID, SourceRef: chunk.Ref, Kind: kind, Title: "A Scene", Summary: "A valid scene."}}}} + return contracts.TypedValidationRequest[dnd.SceneDescriptionList]{Stage: string(pipeline.StageExtract), LLMProfile: "validator-profile", Source: document, Chunk: chunk, SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session.json"), Value: dnd.SceneDescriptionList{Scenes: []dnd.SceneDescription{{ID: chunk.ID, SourceRef: chunk.Ref, Kind: kind, Title: "A Scene", Summary: "A valid scene."}}}} } func cloneValidationRequest(request contracts.TypedValidationRequest[dnd.SceneDescriptionList]) contracts.TypedValidationRequest[dnd.SceneDescriptionList] { @@ -247,40 +179,28 @@ func cloneValidationRequest(request contracts.TypedValidationRequest[dnd.SceneDe return request } -func sortedInputNames(inputs contracts.LLMInputSet) []string { - names := make([]string, 0, len(inputs)) - for name := range inputs { - names = append(names, name) - } - if len(names) == 2 && names[0] == "transcript" { - names[0], names[1] = names[1], names[0] - } - return names -} - type fakeCombatSemanticsClient struct { - response completionResponse - err error - requests []contracts.StructuredCompletionRequest + response classificationResponse + raw string + err error + requests []promptkit.GenerateRequest + onGenerate func(context.Context, promptkit.GenerateRequest) } -func (client *fakeCombatSemanticsClient) CompleteStructured(_ context.Context, request contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { - cloned, err := contracts.CloneStructuredCompletionRequest(request) - if err != nil { - return contracts.StructuredCompletionResponse{}, err +func (client *fakeCombatSemanticsClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + client.requests = append(client.requests, request) + if client.onGenerate != nil { + client.onGenerate(ctx, request) } - client.requests = append(client.requests, cloned) if client.err != nil { - return contracts.StructuredCompletionResponse{}, client.err + return nil, client.err } - response, ok := out.(*completionResponse) - if !ok { - return contracts.StructuredCompletionResponse{}, errors.New("unexpected completion target") + if client.raw != "" { + return &promptkit.GenerateResponse{Content: client.raw}, nil } - *response = client.response content, err := json.Marshal(client.response) if err != nil { - return contracts.StructuredCompletionResponse{}, err + return nil, err } - return contracts.StructuredCompletionResponse{Content: content}, nil + return &promptkit.GenerateResponse{Content: string(content)}, nil }