Migrate combat semantics validation to native execution and unavailable assessments
This commit is contained in:
@@ -1,24 +1,58 @@
|
|||||||
package combat_semantics
|
package assets_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/assets"
|
||||||
|
"io/fs"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
|
||||||
"github.com/santhosh-tekuri/jsonschema/v6"
|
"github.com/santhosh-tekuri/jsonschema/v6"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestLoadResponseSchemaUsesStrictCombatSemanticsContract(t *testing.T) {
|
func TestCombatSemanticsPromptKeepsItsSceneLocalPolicyAndInput(t *testing.T) {
|
||||||
schema, err := loadResponseSchema()
|
_, engine := loadNativeAssets(t)
|
||||||
|
const transcript = `{"sentinel":"combat-semantics-transcript"}`
|
||||||
|
handle, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
|
||||||
|
PromptID: "notarius/dnd.scene_descriptions.validate_combat", APIKey: "test-only",
|
||||||
|
Inputs: map[string]promptkit.ArtifactRef{"transcript": promptkit.Inline(transcript)},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer handle.Discard()
|
||||||
|
details := handle.Details()
|
||||||
|
if details.SelectedProfileID != "notarius/dnd-extraction" || details.StructuredOutput == nil {
|
||||||
|
t.Fatal("native validator defaults/output contract lost")
|
||||||
|
}
|
||||||
|
var all string
|
||||||
|
for _, message := range details.Messages {
|
||||||
|
all += message.Content
|
||||||
|
if strings.Contains(message.Content, transcript) && (message.CacheControl == nil || message.CacheControl.Type != promptkit.CacheControlEphemeral) {
|
||||||
|
t.Fatal("chunk transcript native cache hint lost")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, content := range []string{"substantive active combat materially organizes the", "Do not judge the scene title", transcript} {
|
||||||
|
if strings.Count(all, content) != 1 {
|
||||||
|
t.Fatalf("scene-local context not rendered once: %q", content)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "notarius/dnd.scene_descriptions.validate_combat", APIKey: "test-only"}); err == nil {
|
||||||
|
t.Fatal("native validator accepted missing transcript")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCombatSemanticsResponseSchemaRetainsClassificationAndExplanationRules(t *testing.T) {
|
||||||
|
schema, err := fs.ReadFile(assets.FS(), "dnd/scene-descriptions/validate/combat-semantics/schemas/dnd_scene_combat_semantics_llm.v1.json")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
|
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
if schema.Key != ResponseSchemaKey || schema.ID != ResponseSchemaID || schema.Version != SchemaVersion || schema.Name != ResponseSchemaName || !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) {
|
|
||||||
t.Fatalf("schema = %#v, want private combat-semantics schema identity", schema)
|
|
||||||
}
|
|
||||||
for _, classification := range []string{"combat", "non_combat"} {
|
for _, classification := range []string{"combat", "non_combat"} {
|
||||||
if err := validateCombatSemanticsSchema(map[string]any{"classification": classification, "explanation": "The chunk contains sustained attack exchanges."}, schema.JSONSchema); err != nil {
|
if err := validateCombatSemanticsSchema(map[string]any{"classification": classification, "explanation": "The chunk contains sustained attack exchanges."}, schema); err != nil {
|
||||||
t.Fatalf("valid %q classification rejected: %v", classification, err)
|
t.Fatalf("valid %q classification rejected: %v", classification, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -36,29 +70,11 @@ func TestLoadResponseSchemaUsesStrictCombatSemanticsContract(t *testing.T) {
|
|||||||
{name: "oversized explanation", response: map[string]any{"classification": "combat", "explanation": strings.Repeat("a", 513)}},
|
{name: "oversized explanation", response: map[string]any{"classification": "combat", "explanation": strings.Repeat("a", 513)}},
|
||||||
} {
|
} {
|
||||||
t.Run(test.name, func(t *testing.T) {
|
t.Run(test.name, func(t *testing.T) {
|
||||||
if err := validateCombatSemanticsSchema(test.response, schema.JSONSchema); err == nil {
|
if err := validateCombatSemanticsSchema(test.response, schema); err == nil {
|
||||||
t.Fatal("invalid private response accepted")
|
t.Fatal("invalid private response accepted")
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
if bytes.Contains(schema.JSONSchema, []byte("uniqueItems")) {
|
|
||||||
t.Fatalf("schema included unsupported uniqueItems constraint: %s", schema.JSONSchema)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestResponseSchemaIsMutationSafeAndDiagnosticsRedactContent(t *testing.T) {
|
|
||||||
first, err := loadResponseSchema()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
first.JSONSchema[0] = '['
|
|
||||||
second, err := loadResponseSchema()
|
|
||||||
if err != nil || !strings.HasPrefix(second.SHA256, "sha256:") || !json.Valid(second.JSONSchema) || bytes.Equal(first.JSONSchema, second.JSONSchema) {
|
|
||||||
t.Fatalf("second schema = %s, %v; want defensive copy", second.JSONSchema, err)
|
|
||||||
}
|
|
||||||
if diagnostics := second.DiagnosticsMap(); diagnostics["json_schema"] != nil {
|
|
||||||
t.Fatalf("schema diagnostics included raw content: %#v", diagnostics)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateCombatSemanticsSchema(instance map[string]any, schemaContent []byte) error {
|
func validateCombatSemanticsSchema(instance map[string]any, schemaContent []byte) error {
|
||||||
@@ -764,6 +764,29 @@ units, full-source fallback, final-turn acceptance, and cancellation.
|
|||||||
|
|
||||||
## Stage 17: Migrate The Combat-Semantics LLM Validator
|
## Stage 17: Migrate The Combat-Semantics LLM Validator
|
||||||
|
|
||||||
|
**Status:** Complete. The validator now makes one native request on the shared
|
||||||
|
engine, preserving the validator profile/default, session, execution overrides,
|
||||||
|
transient-retry override, observer, and preparation callback. Static prompt
|
||||||
|
dependencies replace asset/schema registration and manifest/fingerprint metadata.
|
||||||
|
Native errors and failed validation status are checked before decoding; execution
|
||||||
|
and interpretation failures produce an unavailable assessment with inspectable
|
||||||
|
causes, not semantic rejection or invented correction guidance. Cancellation
|
||||||
|
does not deliver a judgment. Successful classification retains the original
|
||||||
|
combat policy, scene-local transcript boundary, explanation checks, and actionable
|
||||||
|
producer guidance. Interpretation and its existing cases are independently
|
||||||
|
testable; evaluation fixtures and schema/policy assets are unchanged.
|
||||||
|
Added native provider tests for judgments, override/input isolation, nil-error
|
||||||
|
failed validation, unavailable provider/capacity, interpretation failure,
|
||||||
|
cancellation, and nested same-backend admission. A runnable framework contract
|
||||||
|
proves complete ordered native assessment on a backend limited to one active call,
|
||||||
|
without holding producer admission or repeating the final chain.
|
||||||
|
Asset tests/vet/race, independent interpretation/evaluation tests/vet/race,
|
||||||
|
focused pipeline tests/vet/race, and `git diff --check` pass. The concrete
|
||||||
|
validator package remains blocked at setup by shared legacy asset helpers
|
||||||
|
importing old ConfigKit's removed YAML `plugin/limit`; its native runtime tests
|
||||||
|
are not claimed passing. Scheduled dependency cleanup and final verification
|
||||||
|
must execute them. No validator-owned semantic continuation loop was added.
|
||||||
|
|
||||||
**Goal:** validators use native execution without another semantic retry loop.
|
**Goal:** validators use native execution without another semantic retry loop.
|
||||||
|
|
||||||
**Inspect/change:** `internal/modules/dnd/validate/scenedescriptions/combat_semantics`,
|
**Inspect/change:** `internal/modules/dnd/validate/scenedescriptions/combat_semantics`,
|
||||||
|
|||||||
58
internal/framework/pipeline/native_assessment_test.go
Normal file
58
internal/framework/pipeline/native_assessment_test.go
Normal file
@@ -0,0 +1,58 @@
|
|||||||
|
package pipeline
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
"go.maximumdirect.net/configkit"
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNativeAssessmentChainSharesOneLimitedBackendWithoutHoldingProducerAdmission(t *testing.T) {
|
||||||
|
calls := 0
|
||||||
|
engine := nativeProducerEngine(t, producerClientFunc(func(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||||
|
if request.Target.BackendID != "limited" {
|
||||||
|
t.Fatal("fixture did not select the shared backend")
|
||||||
|
}
|
||||||
|
calls++
|
||||||
|
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
|
||||||
|
}), promptkit.SettingsInput(configkit.Bytes("shared assessment backend", []byte(`backends:
|
||||||
|
- id: limited
|
||||||
|
endpoint: https://example.invalid/v1
|
||||||
|
concurrency_limit: 1
|
||||||
|
queue_capacity: 0
|
||||||
|
profiles:
|
||||||
|
- id: base
|
||||||
|
backend: limited
|
||||||
|
model: assessment-model
|
||||||
|
api_key_required: false
|
||||||
|
`))))
|
||||||
|
chain := preparedValidatorChain{}
|
||||||
|
for _, name := range []string{"first-native", "second-native"} {
|
||||||
|
chain.validators = append(chain.validators, preparedValidator{resolved: ResolvedValidator{Binding: Binding(name)}})
|
||||||
|
}
|
||||||
|
var order []string
|
||||||
|
operation := contracts.ProducerOperation[int]{
|
||||||
|
Controls: contracts.ProducerControls{Mode: contracts.ModeStrict}, ReuseEligible: true,
|
||||||
|
Assess: func(ctx context.Context, _ int) (contracts.CandidateAssessment, error) {
|
||||||
|
return executeValidationChain(ctx, chain, func(ctx context.Context, validator preparedValidator) (contracts.ValidationResult, error) {
|
||||||
|
order = append(order, validator.resolved.Binding.Module)
|
||||||
|
result, err := engine.Run(ctx, nativeProducerInitial())
|
||||||
|
if err != nil {
|
||||||
|
return contracts.ValidationResult{}, &contracts.ValidationUnavailableError{Cause: err}
|
||||||
|
}
|
||||||
|
if !result.Validation.IsValid {
|
||||||
|
return contracts.ValidationResult{}, &contracts.ValidationUnavailableError{Cause: fmt.Errorf("native contract failed")}
|
||||||
|
}
|
||||||
|
return contracts.ValidationResult{Approved: true}, nil
|
||||||
|
})
|
||||||
|
},
|
||||||
|
}
|
||||||
|
value, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, nil)
|
||||||
|
if err != nil || value != 1 || outcome.Disposition != contracts.ProducerAccepted || calls != 3 || !reflect.DeepEqual(order, []string{"first-native", "second-native"}) {
|
||||||
|
t.Fatalf("nested assessment=%d %#v %v, native calls=%d order=%v", value, outcome, err, calls, order)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -22,7 +22,7 @@ func (f producerClientFunc) Generate(ctx context.Context, request promptkit.Gene
|
|||||||
return f(ctx, request)
|
return f(ctx, request)
|
||||||
}
|
}
|
||||||
|
|
||||||
func nativeProducerEngine(t *testing.T, client promptkit.LLMClient) *promptkit.Engine {
|
func nativeProducerEngine(t *testing.T, client promptkit.LLMClient, extraInputs ...configkit.Input) *promptkit.Engine {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
settings := `profiles:
|
settings := `profiles:
|
||||||
- id: base
|
- id: base
|
||||||
@@ -52,8 +52,12 @@ prompts:
|
|||||||
- {role: user, content: 'Correct: {{input "guidance"}}'}
|
- {role: user, content: 'Correct: {{input "guidance"}}'}
|
||||||
output: {format: json, validation_mode: json, repair_attempts: 0}
|
output: {format: json, validation_mode: json, repair_attempts: 0}
|
||||||
`
|
`
|
||||||
|
underlays := []configkit.Underlay{{Inputs: []configkit.Input{promptkit.SettingsInput(configkit.Bytes("producer fixture", []byte(settings)))}}}
|
||||||
|
if len(extraInputs) > 0 {
|
||||||
|
underlays = append(underlays, configkit.Underlay{Inputs: extraInputs})
|
||||||
|
}
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{promptkit.SettingsInput(configkit.Bytes("producer fixture", []byte(settings)))}}},
|
Underlays: underlays,
|
||||||
}}, promptkit.WithBundledOpenRouter(false), promptkit.WithLLMClient(client))
|
}}, promptkit.WithBundledOpenRouter(false), promptkit.WithLLMClient(client))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
|
|||||||
@@ -0,0 +1,61 @@
|
|||||||
|
package combat_semantics
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestEvaluationCasesAreStrictAndCoverClassifications(t *testing.T) {
|
||||||
|
content, err := os.ReadFile("testdata/evaluation_cases.json")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var cases []struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
TranscriptUnits []struct {
|
||||||
|
ID int `json:"id"`
|
||||||
|
Text string `json:"text"`
|
||||||
|
} `json:"transcript_units"`
|
||||||
|
ProposedKind string `json:"proposed_kind"`
|
||||||
|
ExpectedClassification string `json:"expected_classification"`
|
||||||
|
ReviewerRationale string `json:"reviewer_rationale"`
|
||||||
|
}
|
||||||
|
decoder := json.NewDecoder(strings.NewReader(string(content)))
|
||||||
|
decoder.DisallowUnknownFields()
|
||||||
|
if err := decoder.Decode(&cases); err != nil {
|
||||||
|
t.Fatalf("decode evaluation cases: %v", err)
|
||||||
|
}
|
||||||
|
if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) {
|
||||||
|
t.Fatalf("decode trailing evaluation data: %v, want EOF", err)
|
||||||
|
}
|
||||||
|
seenNames, seenClassifications := map[string]bool{}, map[string]bool{}
|
||||||
|
for _, value := range cases {
|
||||||
|
if strings.TrimSpace(value.Name) == "" || seenNames[value.Name] || len(value.TranscriptUnits) == 0 || strings.TrimSpace(value.ProposedKind) == "" || strings.TrimSpace(value.ReviewerRationale) == "" {
|
||||||
|
t.Fatalf("invalid evaluation case: %#v", value)
|
||||||
|
}
|
||||||
|
seenNames[value.Name] = true
|
||||||
|
switch value.ProposedKind {
|
||||||
|
case "combat", "narrative", "recap", "meta":
|
||||||
|
default:
|
||||||
|
t.Fatalf("unsupported proposed kind %q", value.ProposedKind)
|
||||||
|
}
|
||||||
|
switch value.ExpectedClassification {
|
||||||
|
case "combat", "non_combat":
|
||||||
|
seenClassifications[value.ExpectedClassification] = true
|
||||||
|
default:
|
||||||
|
t.Fatalf("unsupported classification %q", value.ExpectedClassification)
|
||||||
|
}
|
||||||
|
for _, unit := range value.TranscriptUnits {
|
||||||
|
if unit.ID <= 0 || strings.TrimSpace(unit.Text) == "" {
|
||||||
|
t.Fatalf("invalid transcript unit: %#v", unit)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(seenClassifications) != 2 {
|
||||||
|
t.Fatalf("evaluation classifications = %#v, want both classes", seenClassifications)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,70 @@
|
|||||||
|
package combat_semantics
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"unicode/utf8"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
ReasonCodeActiveCombatNotClassified = "scene_active_combat_not_classified"
|
||||||
|
ReasonCodeCombatClassificationUnsupported = "scene_combat_classification_unsupported"
|
||||||
|
maximumExplanationRunes = 512
|
||||||
|
)
|
||||||
|
|
||||||
|
type classificationResponse struct {
|
||||||
|
Classification string `json:"classification"`
|
||||||
|
Explanation string `json:"explanation"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func interpretResponse(response classificationResponse, proposedKind dnd.SceneKind) (contracts.ValidationResult, error) {
|
||||||
|
explanation, err := validateExplanation(response.Explanation)
|
||||||
|
if err != nil {
|
||||||
|
return contracts.ValidationResult{}, err
|
||||||
|
}
|
||||||
|
switch response.Classification {
|
||||||
|
case "combat":
|
||||||
|
if proposedKind == dnd.SceneKindCombat {
|
||||||
|
return contracts.ValidationResult{Approved: true}, nil
|
||||||
|
}
|
||||||
|
return contracts.ValidationResult{
|
||||||
|
Approved: false,
|
||||||
|
ReasonCode: ReasonCodeActiveCombatNotClassified,
|
||||||
|
Message: "The current chunk contains substantive active combat that is not classified as combat.",
|
||||||
|
CorrectionGuidance: "Return kind: combat for this scene. " + explanation,
|
||||||
|
}, nil
|
||||||
|
case "non_combat":
|
||||||
|
if proposedKind != dnd.SceneKindCombat {
|
||||||
|
return contracts.ValidationResult{Approved: true}, nil
|
||||||
|
}
|
||||||
|
return contracts.ValidationResult{
|
||||||
|
Approved: false,
|
||||||
|
ReasonCode: ReasonCodeCombatClassificationUnsupported,
|
||||||
|
Message: "The current chunk does not support a combat classification.",
|
||||||
|
CorrectionGuidance: "Choose the appropriate narrative, recap, or meta kind for this scene. " + explanation,
|
||||||
|
}, nil
|
||||||
|
default:
|
||||||
|
return contracts.ValidationResult{}, validatorErrorf("unsupported combat-semantics classification %q", response.Classification)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateExplanation(value string) (string, error) {
|
||||||
|
if !utf8.ValidString(value) {
|
||||||
|
return "", validatorErrorf("validator explanation must be valid UTF-8")
|
||||||
|
}
|
||||||
|
trimmed := strings.TrimSpace(value)
|
||||||
|
if trimmed == "" {
|
||||||
|
return "", validatorErrorf("validator explanation must not be blank")
|
||||||
|
}
|
||||||
|
if utf8.RuneCountInString(trimmed) > maximumExplanationRunes {
|
||||||
|
return "", validatorErrorf("validator explanation exceeds %d Unicode code points", maximumExplanationRunes)
|
||||||
|
}
|
||||||
|
return trimmed, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validatorErrorf(format string, args ...any) error {
|
||||||
|
return fmt.Errorf("scene combat-semantics validator: "+format, args...)
|
||||||
|
}
|
||||||
@@ -0,0 +1,53 @@
|
|||||||
|
package combat_semantics
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestValidatorInterpretsCombatSemanticsResponses(t *testing.T) {
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
kind dnd.SceneKind
|
||||||
|
response classificationResponse
|
||||||
|
approved bool
|
||||||
|
reasonCode string
|
||||||
|
guidance string
|
||||||
|
correction string
|
||||||
|
wantError string
|
||||||
|
}{
|
||||||
|
{name: "approve non-combat", kind: dnd.SceneKindNarrative, response: classificationResponse{Classification: "non_combat", Explanation: "No active encounter occurs."}, approved: true},
|
||||||
|
{name: "approve combat", kind: dnd.SceneKindCombat, response: classificationResponse{Classification: "combat", Explanation: "Initiative and attacks organize the chunk."}, approved: true},
|
||||||
|
{name: "add combat", kind: dnd.SceneKindNarrative, response: classificationResponse{Classification: "combat", Explanation: "The combatants exchange attacks and damage."}, reasonCode: ReasonCodeActiveCombatNotClassified, guidance: "Return kind: combat"},
|
||||||
|
{name: "remove combat", kind: dnd.SceneKindCombat, response: classificationResponse{Classification: "non_combat", Explanation: "The group only plans for a possible fight."}, reasonCode: ReasonCodeCombatClassificationUnsupported, guidance: "narrative, recap, or meta"},
|
||||||
|
{name: "unknown classification", kind: dnd.SceneKindNarrative, response: classificationResponse{Classification: "uncertain", Explanation: "The group only plans."}, wantError: "unsupported"},
|
||||||
|
{name: "blank explanation", kind: dnd.SceneKindNarrative, response: classificationResponse{Classification: "non_combat", Explanation: " "}, wantError: "blank"},
|
||||||
|
{name: "trim explanation", kind: dnd.SceneKindNarrative, response: classificationResponse{Classification: "combat", Explanation: " The combatants exchange attacks.\n"}, reasonCode: ReasonCodeActiveCombatNotClassified, correction: "Return kind: combat for this scene. The combatants exchange attacks."},
|
||||||
|
{name: "oversized explanation", kind: dnd.SceneKindNarrative, response: classificationResponse{Classification: "non_combat", Explanation: strings.Repeat("界", 513)}, wantError: "exceeds"},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
result, err := interpretResponse(test.response, test.kind)
|
||||||
|
if test.wantError != "" {
|
||||||
|
if err == nil || !strings.Contains(err.Error(), test.wantError) {
|
||||||
|
t.Fatalf("Validate() error = %v, want %q", err, test.wantError)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil || result.Approved != test.approved || result.ReasonCode != test.reasonCode || !strings.Contains(result.CorrectionGuidance, test.guidance) {
|
||||||
|
t.Fatalf("Validate() = %#v, %v", result, err)
|
||||||
|
}
|
||||||
|
if test.correction != "" && result.CorrectionGuidance != test.correction {
|
||||||
|
t.Fatalf("CorrectionGuidance = %q, want %q", result.CorrectionGuidance, test.correction)
|
||||||
|
}
|
||||||
|
if !result.Approved && strings.Contains(result.CorrectionGuidance, result.ReasonCode) {
|
||||||
|
t.Fatalf("correction guidance leaked opaque application detail: %q", result.CorrectionGuidance)
|
||||||
|
}
|
||||||
|
if err := contracts.ValidateValidationResult(result); err != nil {
|
||||||
|
t.Fatalf("ValidateValidationResult(%#v) = %v", result, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,144 @@
|
|||||||
|
package combat_semantics
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNativeValidatorReturnsOnlyCompletedSemanticJudgments(t *testing.T) {
|
||||||
|
for _, classification := range []string{"combat", "non_combat"} {
|
||||||
|
for _, proposed := range []dnd.SceneKind{dnd.SceneKindCombat, dnd.SceneKindNarrative} {
|
||||||
|
request := validRequest(proposed)
|
||||||
|
client := &fakeCombatSemanticsClient{response: classificationResponse{Classification: classification, Explanation: "The supplied transcript supports this judgment."}}
|
||||||
|
result, err := newValidator(t, client).Validate(context.Background(), request)
|
||||||
|
wantApproval := (classification == "combat") == (proposed == dnd.SceneKindCombat)
|
||||||
|
if err != nil || result.Approved != wantApproval || len(client.requests) != 1 {
|
||||||
|
t.Fatalf("judgment=%#v %v", result, err)
|
||||||
|
}
|
||||||
|
if !wantApproval && (strings.TrimSpace(result.CorrectionGuidance) == "" || strings.Contains(result.CorrectionGuidance, result.ReasonCode)) {
|
||||||
|
t.Fatal("rejection lost actionable guidance or copied diagnostic labels")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNativeFailuresAreUnavailableRatherThanRejections(t *testing.T) {
|
||||||
|
failure := errors.New("provider unavailable")
|
||||||
|
for _, test := range []struct {
|
||||||
|
name, raw string
|
||||||
|
failure error
|
||||||
|
}{
|
||||||
|
{"provider failure", "", failure},
|
||||||
|
{"invalid JSON", "{", nil},
|
||||||
|
{"invalid classification", `{"classification":"uncertain","explanation":"Uncertain."}`, nil},
|
||||||
|
{"uninterpretable explanation", `{"classification":"combat","explanation":" "}`, nil},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
client := &fakeCombatSemanticsClient{raw: test.raw, err: test.failure}
|
||||||
|
request := validRequest(dnd.SceneKindCombat)
|
||||||
|
prepared := 0
|
||||||
|
request.Execution.OnPrepared = func(context.Context, promptkit.PreparedRun) { prepared++ }
|
||||||
|
result, err := newValidator(t, client).Validate(context.Background(), request)
|
||||||
|
var unavailable *contracts.ValidationUnavailableError
|
||||||
|
if !errors.As(err, &unavailable) || result.Approved || result.ReasonCode != "" || result.CorrectionGuidance != "" || prepared != 1 {
|
||||||
|
t.Fatalf("unavailable assessment=%#v %v, native executions=%d", result, err, prepared)
|
||||||
|
}
|
||||||
|
if test.failure != nil && (!errors.Is(err, failure) || len(client.requests) != 1) {
|
||||||
|
t.Fatalf("provider failure was flattened or retried: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNilErrorFailedNativeValidationCannotBecomeApproval(t *testing.T) {
|
||||||
|
client := &fakeCombatSemanticsClient{raw: `{"classification":"combat"}`}
|
||||||
|
validator := newValidator(t, client)
|
||||||
|
request := validRequest(dnd.SceneKindCombat)
|
||||||
|
native, err := validator.engine.Run(context.Background(), promptkit.RunRequest{
|
||||||
|
PromptID: PromptID, ProfileID: request.LLMProfile,
|
||||||
|
Inputs: map[string]promptkit.ArtifactRef{"transcript": promptkit.Inline(string(request.Chunk.Content))},
|
||||||
|
})
|
||||||
|
if err != nil || native == nil || native.Validation.Status != promptkit.ValidationFailed || native.Validation.IsValid {
|
||||||
|
t.Fatalf("fixture did not exercise nil-error structural failure: %#v %v", native, err)
|
||||||
|
}
|
||||||
|
result, err := validator.Validate(context.Background(), request)
|
||||||
|
var unavailable *contracts.ValidationUnavailableError
|
||||||
|
if !errors.As(err, &unavailable) || !errors.Is(err, promptkit.ErrValidation) || result.Approved || result.CorrectionGuidance != "" {
|
||||||
|
t.Fatalf("failed native contract accepted/interpreted: %#v %v", result, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNestedValidatorUsesTheReleasedNativeBackendAdmission(t *testing.T) {
|
||||||
|
client := &fakeCombatSemanticsClient{response: classificationResponse{Classification: "non_combat", Explanation: "No active encounter occurs."}}
|
||||||
|
validator := newValidator(t, client)
|
||||||
|
request := validRequest(dnd.SceneKindNarrative)
|
||||||
|
assessments := 0
|
||||||
|
operation := contracts.ProducerOperation[dnd.SceneDescriptionList]{
|
||||||
|
Controls: contracts.ProducerControls{Mode: contracts.ModeStrict}, ReuseEligible: true,
|
||||||
|
Assess: func(ctx context.Context, candidate dnd.SceneDescriptionList) (contracts.CandidateAssessment, error) {
|
||||||
|
assessments++
|
||||||
|
nested := request
|
||||||
|
nested.Value = candidate
|
||||||
|
judgment, err := validator.Validate(ctx, nested)
|
||||||
|
if err != nil {
|
||||||
|
return contracts.CandidateAssessment{}, err
|
||||||
|
}
|
||||||
|
decision := contracts.AssessmentRejected
|
||||||
|
if judgment.Approved {
|
||||||
|
decision = contracts.AssessmentApproved
|
||||||
|
}
|
||||||
|
return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{ValidatorName: Key, Decision: decision, Result: judgment}}}, nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
_, outcome, err := pipeline.RunProducer(context.Background(), validator.engine, operation, promptkit.RunRequest{
|
||||||
|
PromptID: PromptID, ProfileID: request.LLMProfile,
|
||||||
|
Inputs: map[string]promptkit.ArtifactRef{"transcript": promptkit.Inline(string(request.Chunk.Content))},
|
||||||
|
}, func(context.Context, *promptkit.RunResult) (*dnd.SceneDescriptionList, []string, error) {
|
||||||
|
return &request.Value, nil, nil
|
||||||
|
}, nil, nil)
|
||||||
|
if err != nil || outcome.Disposition != contracts.ProducerAccepted || assessments != 1 || len(client.requests) != 2 {
|
||||||
|
t.Fatalf("nested native admission=%#v %v, assessments=%d calls=%d", outcome, err, assessments, len(client.requests))
|
||||||
|
}
|
||||||
|
for _, call := range client.requests {
|
||||||
|
if call.Target.BackendID != "validator-backend" {
|
||||||
|
t.Fatal("fixture did not exercise the same limited backend")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCapacityFailureAndCancellationNeverBecomeSemanticGuidance(t *testing.T) {
|
||||||
|
t.Run("capacity unavailable", func(t *testing.T) {
|
||||||
|
var validator *Validator
|
||||||
|
var nestedErr error
|
||||||
|
request := validRequest(dnd.SceneKindNarrative)
|
||||||
|
client := &fakeCombatSemanticsClient{response: classificationResponse{Classification: "non_combat", Explanation: "No encounter."}}
|
||||||
|
client.onGenerate = func(ctx context.Context, _ promptkit.GenerateRequest) {
|
||||||
|
// The outer fake provider still holds the single active native admission.
|
||||||
|
// The configured zero queue makes this a capacity error, not a waiting test.
|
||||||
|
_, nestedErr = validator.Validate(ctx, request)
|
||||||
|
}
|
||||||
|
validator = newValidator(t, client)
|
||||||
|
result, err := validator.Validate(context.Background(), request)
|
||||||
|
var unavailable *contracts.ValidationUnavailableError
|
||||||
|
if err != nil || !result.Approved || !errors.As(nestedErr, &unavailable) || !errors.Is(nestedErr, promptkit.ErrCapacityExceeded) || len(client.requests) != 1 {
|
||||||
|
t.Fatalf("capacity assessment=%#v %v, nested=%v", result, err, nestedErr)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
t.Run("canceled delivery", func(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
client := &fakeCombatSemanticsClient{response: classificationResponse{Classification: "non_combat", Explanation: "No encounter."},
|
||||||
|
onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }}
|
||||||
|
result, err := newValidator(t, client).Validate(ctx, validRequest(dnd.SceneKindNarrative))
|
||||||
|
if !errors.Is(err, context.Canceled) || result.Approved || result.CorrectionGuidance != "" || len(client.requests) != 1 {
|
||||||
|
t.Fatalf("cancellation=%#v %v", result, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -1,72 +0,0 @@
|
|||||||
package combat_semantics
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"io/fs"
|
|
||||||
"sync"
|
|
||||||
|
|
||||||
rootassets "gitea.maximumdirect.net/eric/notarius/assets"
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs"
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
|
||||||
)
|
|
||||||
|
|
||||||
const promptAssetRoot = "assets/prompts"
|
|
||||||
|
|
||||||
var promptAssetManifest = shared.PromptAssetManifest{
|
|
||||||
ModuleDir: PromptID,
|
|
||||||
ModuleFiles: []promptfs.ModulePromptFile{
|
|
||||||
{Name: "prompt.yaml", Path: "prompts/prompt.yaml"},
|
|
||||||
{Name: "instructions.md", Path: "prompts/instructions.md"},
|
|
||||||
},
|
|
||||||
SharedFiles: []string{
|
|
||||||
"common-dnd-system.md",
|
|
||||||
"common-dnd-scene-combat-policy.md",
|
|
||||||
"common-dnd-transcript-chunk.md",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
func moduleAssetFS() (fs.FS, error) {
|
|
||||||
assets, err := fs.Sub(rootassets.FS(), "dnd/scene-descriptions/validate/combat-semantics")
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("scope scene combat-semantics validator assets: %w", err)
|
|
||||||
}
|
|
||||||
return assets, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func RegisterPromptAssets(registry *llm.AssetRegistry) error {
|
|
||||||
assets, err := moduleAssetFS()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
promptFS, err := promptAssetManifest.PromptFS(assets)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("prepare scene combat-semantics validator prompt assets: %w", err)
|
|
||||||
}
|
|
||||||
if err := registry.RegisterPromptFS(promptFS, promptAssetRoot); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
schemas, err := fs.Sub(assets, "schemas")
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("scope scene combat-semantics validator schemas: %w", err)
|
|
||||||
}
|
|
||||||
return registry.RegisterSchemaFS(schemas, ".")
|
|
||||||
}
|
|
||||||
|
|
||||||
func promptAssetMetadata() (string, error) {
|
|
||||||
promptAssetHashOnce.Do(func() {
|
|
||||||
assets, err := moduleAssetFS()
|
|
||||||
if err != nil {
|
|
||||||
promptAssetHashErr = err
|
|
||||||
return
|
|
||||||
}
|
|
||||||
promptAssetHash, promptAssetHashErr = promptAssetManifest.Hash(assets)
|
|
||||||
})
|
|
||||||
return promptAssetHash, promptAssetHashErr
|
|
||||||
}
|
|
||||||
|
|
||||||
var (
|
|
||||||
promptAssetHashOnce sync.Once
|
|
||||||
promptAssetHash string
|
|
||||||
promptAssetHashErr error
|
|
||||||
)
|
|
||||||
@@ -1,90 +0,0 @@
|
|||||||
package combat_semantics
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestRegisterPromptAssetsPreparesCombatSemanticsPrompt(t *testing.T) {
|
|
||||||
registry := llm.NewAssetRegistry()
|
|
||||||
if err := RegisterPromptAssets(registry); err != nil {
|
|
||||||
t.Fatalf("RegisterPromptAssets() error = %v, want nil", err)
|
|
||||||
}
|
|
||||||
engine := newCombatSemanticsPromptEngine(t, registry)
|
|
||||||
inputs := map[string]promptkit.ArtifactRef{
|
|
||||||
"transcript": promptkit.InlineWithURI("file:///session.json", `{"sentinel":"combat-semantics-transcript"}`),
|
|
||||||
}
|
|
||||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
|
||||||
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "combat-semantics-test-profile", Inputs: inputs,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Prepare() error = %v, want nil", err)
|
|
||||||
}
|
|
||||||
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_scene_combat_semantics_llm.v1.json" {
|
|
||||||
t.Fatalf("prepared prompt = %#v, want combat-semantics prompt identity and schema wiring", prepared)
|
|
||||||
}
|
|
||||||
if len(prepared.Messages) != 4 {
|
|
||||||
t.Fatalf("prepared message count = %d, want 4", len(prepared.Messages))
|
|
||||||
}
|
|
||||||
for index, message := range prepared.Messages {
|
|
||||||
wantEphemeral := index == 3
|
|
||||||
gotEphemeral := message.CacheControl != nil && message.CacheControl.Type == promptkit.CacheControlEphemeral
|
|
||||||
if gotEphemeral != wantEphemeral {
|
|
||||||
t.Fatalf("message %d cache control = %#v, want ephemeral=%t", index, message.CacheControl, wantEphemeral)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
assertRenderedExactlyOnce(t, prepared.Messages, "substantive active combat materially organizes the", 1)
|
|
||||||
assertRenderedExactlyOnce(t, prepared.Messages, "combat-semantics-transcript", 3)
|
|
||||||
for name := range inputs {
|
|
||||||
missing := make(map[string]promptkit.ArtifactRef, len(inputs)-1)
|
|
||||||
for inputName, input := range inputs {
|
|
||||||
if inputName != name {
|
|
||||||
missing[inputName] = input
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if _, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
|
||||||
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "combat-semantics-test-profile", Inputs: missing,
|
|
||||||
}); err == nil {
|
|
||||||
t.Fatalf("Prepare() without required %q input = nil error", name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPromptAssetMetadataIsContentSafe(t *testing.T) {
|
|
||||||
hash, err := promptAssetMetadata()
|
|
||||||
if err != nil || !strings.HasPrefix(hash, "sha256:") {
|
|
||||||
t.Fatalf("promptAssetMetadata() = %q, %v; want hash", hash, err)
|
|
||||||
}
|
|
||||||
if strings.Contains(hash, "non_combat") {
|
|
||||||
t.Fatalf("prompt asset metadata leaked raw prompt content: %q", hash)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func newCombatSemanticsPromptEngine(t *testing.T, registry *llm.AssetRegistry) *promptkit.Engine {
|
|
||||||
t.Helper()
|
|
||||||
return promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
|
|
||||||
"combat-semantics-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "combat-semantics-test-model"),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func assertRenderedExactlyOnce(t *testing.T, messages []promptkit.RenderedMessage, sentinel string, wantIndex int) {
|
|
||||||
t.Helper()
|
|
||||||
index := -1
|
|
||||||
occurrences := 0
|
|
||||||
for messageIndex, message := range messages {
|
|
||||||
count := strings.Count(message.Content, sentinel)
|
|
||||||
if count > 0 {
|
|
||||||
index = messageIndex
|
|
||||||
occurrences += count
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if occurrences != 1 || index != wantIndex {
|
|
||||||
t.Fatalf("sentinel %q rendered %d times at message %d, want once at message %d", sentinel, occurrences, index, wantIndex)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,25 +0,0 @@
|
|||||||
package combat_semantics
|
|
||||||
|
|
||||||
import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
|
||||||
|
|
||||||
const (
|
|
||||||
PromptID = "dnd.scene_descriptions.validate_combat"
|
|
||||||
ResponseSchemaKey = llm.ResponseSchemaKey("dnd_scene_combat_semantics_llm")
|
|
||||||
ResponseSchemaID = "notarius.dnd.scene_descriptions.combat_semantics.llm"
|
|
||||||
ResponseSchemaName = "notarius_dnd_scene_combat_semantics_llm_v1"
|
|
||||||
SchemaVersion = "v1"
|
|
||||||
)
|
|
||||||
|
|
||||||
func loadResponseSchema() (llm.ResponseSchema, error) {
|
|
||||||
assets, err := moduleAssetFS()
|
|
||||||
if err != nil {
|
|
||||||
return llm.ResponseSchema{}, err
|
|
||||||
}
|
|
||||||
return llm.LoadResponseSchema(assets, llm.ResponseSchemaDefinition{
|
|
||||||
Key: ResponseSchemaKey,
|
|
||||||
ID: ResponseSchemaID,
|
|
||||||
Version: SchemaVersion,
|
|
||||||
Name: ResponseSchemaName,
|
|
||||||
AssetPath: "schemas/dnd_scene_combat_semantics_llm.v1.json",
|
|
||||||
})
|
|
||||||
}
|
|
||||||
@@ -2,9 +2,10 @@ package combat_semantics
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"encoding/json"
|
||||||
"strings"
|
"strings"
|
||||||
"unicode/utf8"
|
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
@@ -14,44 +15,20 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/shape"
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/shape"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const Key = "extract/dnd/scene-descriptions/combat_semantics"
|
||||||
Key = "extract/dnd/scene-descriptions/combat_semantics"
|
const PromptID = "notarius/dnd.scene_descriptions.validate_combat"
|
||||||
ReasonCodeActiveCombatNotClassified = "scene_active_combat_not_classified"
|
|
||||||
ReasonCodeCombatClassificationUnsupported = "scene_combat_classification_unsupported"
|
|
||||||
combatPolicy = "dnd.scene_descriptions.combat_semantics.v1"
|
|
||||||
maximumExplanationRunes = 512
|
|
||||||
)
|
|
||||||
|
|
||||||
type Options struct{}
|
type Options struct{}
|
||||||
|
|
||||||
type completionResponse struct {
|
type Validator struct{ engine *promptkit.Engine }
|
||||||
Classification string `json:"classification"`
|
|
||||||
Explanation string `json:"explanation"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type Validator struct {
|
|
||||||
llm contracts.StructuredLLMClient
|
|
||||||
promptSHA string
|
|
||||||
responseSchemaSHA string
|
|
||||||
}
|
|
||||||
|
|
||||||
var _ contracts.TypedValidator[dnd.SceneDescriptionList] = (*Validator)(nil)
|
var _ contracts.TypedValidator[dnd.SceneDescriptionList] = (*Validator)(nil)
|
||||||
var _ contracts.ManifestMetadataProvider = (*Validator)(nil)
|
|
||||||
var _ pipeline.CheckpointFingerprintProvider = (*Validator)(nil)
|
|
||||||
|
|
||||||
func New(llmClient contracts.StructuredLLMClient, _ Options) (*Validator, error) {
|
func New(engine *promptkit.Engine, _ Options) (*Validator, error) {
|
||||||
if llmClient == nil {
|
if engine == nil {
|
||||||
return nil, validatorErrorf("LLM client must not be nil")
|
return nil, validatorErrorf("native engine must not be nil")
|
||||||
}
|
}
|
||||||
promptSHA, err := promptAssetMetadata()
|
return &Validator{engine: engine}, nil
|
||||||
if err != nil {
|
|
||||||
return nil, validatorErrorf("load prompt metadata: %w", err)
|
|
||||||
}
|
|
||||||
responseSchema, err := loadResponseSchema()
|
|
||||||
if err != nil {
|
|
||||||
return nil, validatorErrorf("load response schema: %w", err)
|
|
||||||
}
|
|
||||||
return &Validator{llm: llmClient, promptSHA: promptSHA, responseSchemaSHA: responseSchema.SHA256}, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (v *Validator) Name() string { return Key }
|
func (v *Validator) Name() string { return Key }
|
||||||
@@ -60,62 +37,46 @@ func (v *Validator) ExecutionClass() contracts.ExecutionClass {
|
|||||||
return contracts.ExecutionClassLLMBacked
|
return contracts.ExecutionClassLLMBacked
|
||||||
}
|
}
|
||||||
|
|
||||||
func (v *Validator) ManifestMetadata() map[string]any {
|
|
||||||
if v == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return map[string]any{
|
|
||||||
"prompt_id": PromptID,
|
|
||||||
"prompt_version": SchemaVersion,
|
|
||||||
"prompt_sha256": v.promptSHA,
|
|
||||||
"response_schema_key": string(ResponseSchemaKey),
|
|
||||||
"response_schema_id": ResponseSchemaID,
|
|
||||||
"response_schema_name": ResponseSchemaName,
|
|
||||||
"response_schema_version": SchemaVersion,
|
|
||||||
"response_schema_sha256": v.responseSchemaSHA,
|
|
||||||
"combat_policy": combatPolicy,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (v *Validator) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
|
|
||||||
if v == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return []pipeline.CheckpointFingerprint{
|
|
||||||
{Name: "prompt", Value: v.promptSHA},
|
|
||||||
{Name: "response_schema", Value: v.responseSchemaSHA},
|
|
||||||
{Name: "combat_policy", Value: combatPolicy},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (v *Validator) Validate(ctx context.Context, req contracts.TypedValidationRequest[dnd.SceneDescriptionList]) (contracts.ValidationResult, error) {
|
func (v *Validator) Validate(ctx context.Context, req contracts.TypedValidationRequest[dnd.SceneDescriptionList]) (contracts.ValidationResult, error) {
|
||||||
if v == nil {
|
if v == nil {
|
||||||
return contracts.ValidationResult{}, validatorErrorf("validator must not be nil")
|
return contracts.ValidationResult{}, validatorErrorf("validator must not be nil")
|
||||||
}
|
}
|
||||||
if v.llm == nil {
|
if v.engine == nil {
|
||||||
return contracts.ValidationResult{}, validatorErrorf("LLM client must not be nil")
|
return contracts.ValidationResult{}, validatorErrorf("native engine must not be nil")
|
||||||
}
|
}
|
||||||
sourceInput, scene, err := validateRequest(ctx, req)
|
sourceInput, scene, err := validateRequest(ctx, req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return contracts.ValidationResult{}, err
|
return contracts.ValidationResult{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
var response completionResponse
|
material := shared.TranscriptPromptMaterial(sourceInput)
|
||||||
_, err = v.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
result, err := v.engine.Run(ctx, promptkit.RunRequest{
|
||||||
StageName: Key,
|
PromptID: PromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID,
|
||||||
PromptID: PromptID,
|
Inputs: map[string]promptkit.ArtifactRef{"transcript": promptkit.InlineWithURI(material.OriginURI, string(material.Content))},
|
||||||
PromptVersion: SchemaVersion,
|
Execution: req.Execution.Execution, TransientRetries: req.Execution.TransientRetries,
|
||||||
ProfileID: req.LLMProfile,
|
Observer: req.Execution.Observer, OnPrepared: req.Execution.OnPrepared,
|
||||||
SessionID: req.SessionID,
|
})
|
||||||
StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts,
|
if canceled := ctx.Err(); canceled != nil {
|
||||||
Inputs: contracts.LLMInputSet{
|
return contracts.ValidationResult{}, canceled
|
||||||
"transcript": shared.TranscriptPromptMaterial(sourceInput),
|
|
||||||
},
|
|
||||||
}, &response)
|
|
||||||
if err != nil {
|
|
||||||
return contracts.ValidationResult{}, validatorErrorf("complete structured output: %w", err)
|
|
||||||
}
|
}
|
||||||
return interpretResponse(response, scene.Kind)
|
if err != nil {
|
||||||
|
return contracts.ValidationResult{}, unavailable(err)
|
||||||
|
}
|
||||||
|
if result == nil || result.Validation.Status == promptkit.ValidationFailed || !result.Validation.IsValid {
|
||||||
|
return contracts.ValidationResult{}, unavailable(validatorErrorf("native output contract failed: %w", promptkit.ErrValidation))
|
||||||
|
}
|
||||||
|
var response classificationResponse
|
||||||
|
if err := json.Unmarshal([]byte(result.RawOutput), &response); err != nil {
|
||||||
|
return contracts.ValidationResult{}, unavailable(validatorErrorf("decode native output: %w", err))
|
||||||
|
}
|
||||||
|
assessment, err := interpretResponse(response, scene.Kind)
|
||||||
|
if canceled := ctx.Err(); canceled != nil {
|
||||||
|
return contracts.ValidationResult{}, canceled
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return contracts.ValidationResult{}, unavailable(err)
|
||||||
|
}
|
||||||
|
return assessment, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateRequest(ctx context.Context, req contracts.TypedValidationRequest[dnd.SceneDescriptionList]) (contracts.LLMInputMaterial, dnd.SceneDescription, error) {
|
func validateRequest(ctx context.Context, req contracts.TypedValidationRequest[dnd.SceneDescriptionList]) (contracts.LLMInputMaterial, dnd.SceneDescription, error) {
|
||||||
@@ -128,7 +89,7 @@ func validateRequest(ctx context.Context, req contracts.TypedValidationRequest[d
|
|||||||
if req.Stage != string(pipeline.StageExtract) {
|
if req.Stage != string(pipeline.StageExtract) {
|
||||||
return contracts.LLMInputMaterial{}, dnd.SceneDescription{}, validatorErrorf("validation stage must be extract, got %q", req.Stage)
|
return contracts.LLMInputMaterial{}, dnd.SceneDescription{}, validatorErrorf("validation stage must be extract, got %q", req.Stage)
|
||||||
}
|
}
|
||||||
sourceInput, err := shared.PrepareChunkExtraction(ctx, contracts.TypedExtractionRequest{Source: req.Source, Chunk: req.Chunk, SourceInput: req.SourceInput})
|
sourceInput, err := shared.PrepareChunkExtraction(ctx, contracts.TypedExtractionRequest[dnd.SceneDescriptionList]{Source: req.Source, Chunk: req.Chunk, SourceInput: req.SourceInput})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return contracts.LLMInputMaterial{}, dnd.SceneDescription{}, validatorErrorf("prepare chunk source input: %w", err)
|
return contracts.LLMInputMaterial{}, dnd.SceneDescription{}, validatorErrorf("prepare chunk source input: %w", err)
|
||||||
}
|
}
|
||||||
@@ -154,53 +115,12 @@ func validateRequest(ctx context.Context, req contracts.TypedValidationRequest[d
|
|||||||
return sourceInput, scene, nil
|
return sourceInput, scene, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func interpretResponse(response completionResponse, proposedKind dnd.SceneKind) (contracts.ValidationResult, error) {
|
func unavailable(cause error) error {
|
||||||
explanation, err := validateExplanation(response.Explanation)
|
return &contracts.ValidationUnavailableError{Cause: validatorErrorf("native assessment unavailable: %w", cause)}
|
||||||
if err != nil {
|
|
||||||
return contracts.ValidationResult{}, err
|
|
||||||
}
|
|
||||||
switch response.Classification {
|
|
||||||
case "combat":
|
|
||||||
if proposedKind == dnd.SceneKindCombat {
|
|
||||||
return contracts.ValidationResult{Approved: true}, nil
|
|
||||||
}
|
|
||||||
return contracts.ValidationResult{
|
|
||||||
Approved: false,
|
|
||||||
ReasonCode: ReasonCodeActiveCombatNotClassified,
|
|
||||||
Message: "The current chunk contains substantive active combat that is not classified as combat.",
|
|
||||||
CorrectionGuidance: "Return kind: combat for this scene. " + explanation,
|
|
||||||
}, nil
|
|
||||||
case "non_combat":
|
|
||||||
if proposedKind != dnd.SceneKindCombat {
|
|
||||||
return contracts.ValidationResult{Approved: true}, nil
|
|
||||||
}
|
|
||||||
return contracts.ValidationResult{
|
|
||||||
Approved: false,
|
|
||||||
ReasonCode: ReasonCodeCombatClassificationUnsupported,
|
|
||||||
Message: "The current chunk does not support a combat classification.",
|
|
||||||
CorrectionGuidance: "Choose the appropriate narrative, recap, or meta kind for this scene. " + explanation,
|
|
||||||
}, nil
|
|
||||||
default:
|
|
||||||
return contracts.ValidationResult{}, validatorErrorf("unsupported combat-semantics classification %q", response.Classification)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateExplanation(value string) (string, error) {
|
|
||||||
if !utf8.ValidString(value) {
|
|
||||||
return "", validatorErrorf("validator explanation must be valid UTF-8")
|
|
||||||
}
|
|
||||||
trimmed := strings.TrimSpace(value)
|
|
||||||
if trimmed == "" {
|
|
||||||
return "", validatorErrorf("validator explanation must not be blank")
|
|
||||||
}
|
|
||||||
if utf8.RuneCountInString(trimmed) > maximumExplanationRunes {
|
|
||||||
return "", validatorErrorf("validator explanation exceeds %d Unicode code points", maximumExplanationRunes)
|
|
||||||
}
|
|
||||||
return trimmed, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func Spec() pipeline.ValidatorSpec {
|
func Spec() pipeline.ValidatorSpec {
|
||||||
return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassLLMBacked}
|
return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{PromptID}}
|
||||||
}
|
}
|
||||||
|
|
||||||
func Register(registry *pipeline.ValidatorRegistry) error {
|
func Register(registry *pipeline.ValidatorRegistry) error {
|
||||||
@@ -209,7 +129,7 @@ func Register(registry *pipeline.ValidatorRegistry) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return New(request.Dependencies.LLM, options)
|
return New(request.Dependencies.Engine, options)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -221,7 +141,3 @@ func DecodeOptions(options map[string]any) (Options, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func validateOptions(options map[string]any) error { _, err := DecodeOptions(options); return err }
|
func validateOptions(options map[string]any) error { _, err := DecodeOptions(options); return err }
|
||||||
|
|
||||||
func validatorErrorf(format string, args ...any) error {
|
|
||||||
return fmt.Errorf("scene combat-semantics validator: "+format, args...)
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -4,75 +4,31 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"io"
|
|
||||||
"os"
|
|
||||||
"reflect"
|
"reflect"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/assets"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||||
|
"go.maximumdirect.net/configkit"
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestValidatorInterpretsCombatSemanticsResponses(t *testing.T) {
|
|
||||||
request := validRequest(dnd.SceneKindNarrative)
|
|
||||||
for _, test := range []struct {
|
|
||||||
name string
|
|
||||||
kind dnd.SceneKind
|
|
||||||
response completionResponse
|
|
||||||
approved bool
|
|
||||||
reasonCode string
|
|
||||||
guidance string
|
|
||||||
correction string
|
|
||||||
wantError string
|
|
||||||
}{
|
|
||||||
{name: "approve non-combat", kind: dnd.SceneKindNarrative, response: completionResponse{Classification: "non_combat", Explanation: "No active encounter occurs."}, approved: true},
|
|
||||||
{name: "approve combat", kind: dnd.SceneKindCombat, response: completionResponse{Classification: "combat", Explanation: "Initiative and attacks organize the chunk."}, approved: true},
|
|
||||||
{name: "add combat", kind: dnd.SceneKindNarrative, response: completionResponse{Classification: "combat", Explanation: "The combatants exchange attacks and damage."}, reasonCode: ReasonCodeActiveCombatNotClassified, guidance: "Return kind: combat"},
|
|
||||||
{name: "remove combat", kind: dnd.SceneKindCombat, response: completionResponse{Classification: "non_combat", Explanation: "The group only plans for a possible fight."}, reasonCode: ReasonCodeCombatClassificationUnsupported, guidance: "narrative, recap, or meta"},
|
|
||||||
{name: "unknown classification", kind: dnd.SceneKindNarrative, response: completionResponse{Classification: "uncertain", Explanation: "The group only plans."}, wantError: "unsupported"},
|
|
||||||
{name: "blank explanation", kind: dnd.SceneKindNarrative, response: completionResponse{Classification: "non_combat", Explanation: " "}, wantError: "blank"},
|
|
||||||
{name: "trim explanation", kind: dnd.SceneKindNarrative, response: completionResponse{Classification: "combat", Explanation: " The combatants exchange attacks.\n"}, reasonCode: ReasonCodeActiveCombatNotClassified, correction: "Return kind: combat for this scene. The combatants exchange attacks."},
|
|
||||||
{name: "oversized explanation", kind: dnd.SceneKindNarrative, response: completionResponse{Classification: "non_combat", Explanation: strings.Repeat("界", 513)}, wantError: "exceeds"},
|
|
||||||
} {
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
request.Value.Scenes[0].Kind = test.kind
|
|
||||||
client := &fakeCombatSemanticsClient{response: test.response}
|
|
||||||
validator := newValidator(t, client)
|
|
||||||
result, err := validator.Validate(context.Background(), request)
|
|
||||||
if test.wantError != "" {
|
|
||||||
if err == nil || !strings.Contains(err.Error(), test.wantError) {
|
|
||||||
t.Fatalf("Validate() error = %v, want %q", err, test.wantError)
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err != nil || result.Approved != test.approved || result.ReasonCode != test.reasonCode || !strings.Contains(result.CorrectionGuidance, test.guidance) {
|
|
||||||
t.Fatalf("Validate() = %#v, %v", result, err)
|
|
||||||
}
|
|
||||||
if test.correction != "" && result.CorrectionGuidance != test.correction {
|
|
||||||
t.Fatalf("CorrectionGuidance = %q, want %q", result.CorrectionGuidance, test.correction)
|
|
||||||
}
|
|
||||||
if !result.Approved && (strings.Contains(result.CorrectionGuidance, Key) || strings.Contains(result.CorrectionGuidance, result.ReasonCode) || strings.Contains(result.CorrectionGuidance, request.Chunk.ID)) {
|
|
||||||
t.Fatalf("correction guidance leaked opaque application detail: %q", result.CorrectionGuidance)
|
|
||||||
}
|
|
||||||
if err := contracts.ValidateValidationResult(result); err != nil {
|
|
||||||
t.Fatalf("ValidateValidationResult(%#v) = %v", result, err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestValidatorUsesOnlyChunkTranscript(t *testing.T) {
|
func TestValidatorUsesOnlyChunkTranscript(t *testing.T) {
|
||||||
request := validRequest(dnd.SceneKindNarrative)
|
request := validRequest(dnd.SceneKindNarrative)
|
||||||
repairAttempts := 2
|
|
||||||
request.LLMProfile = "validator-profile"
|
request.LLMProfile = "validator-profile"
|
||||||
request.SessionID = "validator-session"
|
request.SessionID = "validator-session"
|
||||||
request.StructuredOutputRepairAttempts = &repairAttempts
|
|
||||||
request.References = contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{"players": {Items: []contracts.ReferenceItem{{Content: []byte("hidden reference")}}}}}
|
request.References = contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{"players": {Items: []contracts.ReferenceItem{{Content: []byte("hidden reference")}}}}}
|
||||||
client := &fakeCombatSemanticsClient{response: completionResponse{Classification: "non_combat", Explanation: "No active encounter occurs."}}
|
client := &fakeCombatSemanticsClient{response: classificationResponse{Classification: "non_combat", Explanation: "No active encounter occurs."}}
|
||||||
validator := newValidator(t, client)
|
validator := newValidator(t, client)
|
||||||
|
model := "validator-override"
|
||||||
|
request.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: model}
|
||||||
|
var prepared promptkit.PreparedRun
|
||||||
|
preparedCalls := 0
|
||||||
|
request.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = value; preparedCalls++ }
|
||||||
if result, err := validator.Validate(context.Background(), request); err != nil || !result.Approved {
|
if result, err := validator.Validate(context.Background(), request); err != nil || !result.Approved {
|
||||||
t.Fatalf("Validate() = %#v, %v", result, err)
|
t.Fatalf("Validate() = %#v, %v", result, err)
|
||||||
}
|
}
|
||||||
@@ -80,19 +36,22 @@ func TestValidatorUsesOnlyChunkTranscript(t *testing.T) {
|
|||||||
t.Fatalf("completion count = %d, want 1", len(client.requests))
|
t.Fatalf("completion count = %d, want 1", len(client.requests))
|
||||||
}
|
}
|
||||||
completed := client.requests[0]
|
completed := client.requests[0]
|
||||||
if completed.StageName != Key || completed.PromptID != PromptID || completed.PromptVersion != SchemaVersion || completed.ProfileID != request.LLMProfile || completed.SessionID != request.SessionID || completed.StructuredOutputRepairAttempts == request.StructuredOutputRepairAttempts || *completed.StructuredOutputRepairAttempts != repairAttempts {
|
if preparedCalls != 1 || prepared.PromptID != PromptID || prepared.SelectedProfileID != request.LLMProfile || completed.Prompt.SessionID != request.SessionID || completed.Target.Model != model {
|
||||||
t.Fatalf("completion request = %#v", completed)
|
t.Fatal("native execution selection/override missing")
|
||||||
}
|
}
|
||||||
if completed.Correction != nil || !reflect.DeepEqual(sortedInputNames(completed.Inputs), []string{"transcript"}) || !reflect.DeepEqual(completed.Inputs["transcript"].Content, request.Chunk.Content) {
|
var messages string
|
||||||
t.Fatalf("completion inputs = %#v", completed.Inputs)
|
for _, message := range completed.Prompt.Messages {
|
||||||
|
messages += message.Content
|
||||||
|
}
|
||||||
|
if strings.Count(messages, string(request.Chunk.Content)) != 1 {
|
||||||
|
t.Fatal("chunk transcript not rendered exactly once")
|
||||||
}
|
}
|
||||||
for _, forbidden := range []string{request.Value.Scenes[0].ID, request.Value.Scenes[0].Title, request.Value.Scenes[0].Summary, "hidden reference"} {
|
for _, forbidden := range []string{request.Value.Scenes[0].ID, request.Value.Scenes[0].Title, request.Value.Scenes[0].Summary, "hidden reference"} {
|
||||||
for name, input := range completed.Inputs {
|
if strings.Contains(messages, forbidden) {
|
||||||
if strings.Contains(string(input.Content), forbidden) {
|
t.Fatalf("model-visible context leaked %q", forbidden)
|
||||||
t.Fatalf("input %q leaked forbidden model-visible value %q", name, forbidden)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestValidatorRejectsInvalidRequestsAndCompletionFailures(t *testing.T) {
|
func TestValidatorRejectsInvalidRequestsAndCompletionFailures(t *testing.T) {
|
||||||
@@ -130,7 +89,7 @@ func TestValidatorRejectsInvalidRequestsAndCompletionFailures(t *testing.T) {
|
|||||||
if test.mutate != nil {
|
if test.mutate != nil {
|
||||||
test.mutate(&request)
|
test.mutate(&request)
|
||||||
}
|
}
|
||||||
client := &fakeCombatSemanticsClient{response: completionResponse{Classification: "non_combat", Explanation: "No active encounter occurs."}}
|
client := &fakeCombatSemanticsClient{response: classificationResponse{Classification: "non_combat", Explanation: "No active encounter occurs."}}
|
||||||
_, err := newValidator(t, client).Validate(test.ctx, request)
|
_, err := newValidator(t, client).Validate(test.ctx, request)
|
||||||
if err == nil || !strings.Contains(err.Error(), test.wantErr) || len(client.requests) != 0 {
|
if err == nil || !strings.Contains(err.Error(), test.wantErr) || len(client.requests) != 0 {
|
||||||
t.Fatalf("Validate() error = %v, calls = %d; want %q and no calls", err, len(client.requests), test.wantErr)
|
t.Fatalf("Validate() error = %v, calls = %d; want %q and no calls", err, len(client.requests), test.wantErr)
|
||||||
@@ -138,29 +97,26 @@ func TestValidatorRejectsInvalidRequestsAndCompletionFailures(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
client := &fakeCombatSemanticsClient{err: errors.New("provider unavailable")}
|
client := &fakeCombatSemanticsClient{err: errors.New("provider unavailable")}
|
||||||
if _, err := newValidator(t, client).Validate(context.Background(), base); err == nil || !strings.Contains(err.Error(), "complete structured output") {
|
if _, err := newValidator(t, client).Validate(context.Background(), base); err == nil || !strings.Contains(err.Error(), "assessment unavailable") {
|
||||||
t.Fatalf("completion failure error = %v", err)
|
t.Fatalf("completion failure error = %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestValidatorConstructionOptionsAndMetadata(t *testing.T) {
|
func TestValidatorConstructionOptionsAndDependencies(t *testing.T) {
|
||||||
if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "LLM client") {
|
if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "native engine") {
|
||||||
t.Fatalf("New(nil) error = %v", err)
|
t.Fatalf("New(nil) error = %v", err)
|
||||||
}
|
}
|
||||||
if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil {
|
if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil {
|
||||||
t.Fatal("DecodeOptions() accepted unknown options")
|
t.Fatal("DecodeOptions() accepted unknown options")
|
||||||
}
|
}
|
||||||
validator := newValidator(t, &fakeCombatSemanticsClient{})
|
validator := newValidator(t, &fakeCombatSemanticsClient{})
|
||||||
if validator.Name() != Key || validator.ExecutionClass() != contracts.ExecutionClassLLMBacked || Spec().CorrectionProtocol != "" {
|
if validator.Name() != Key || validator.ExecutionClass() != contracts.ExecutionClassLLMBacked || !reflect.DeepEqual(Spec().Prompts, []string{PromptID}) {
|
||||||
t.Fatalf("validator contract = %#v / %#v", validator, Spec())
|
t.Fatalf("validator contract = %#v / %#v", validator, Spec())
|
||||||
}
|
}
|
||||||
metadata, err := json.Marshal(validator.ManifestMetadata())
|
selected := Spec()
|
||||||
if err != nil || !strings.Contains(string(metadata), ResponseSchemaID) || strings.Contains(string(metadata), "non_combat") {
|
selected.Prompts[0] = "mutated"
|
||||||
t.Fatalf("manifest metadata = %s, %v", metadata, err)
|
if !reflect.DeepEqual(Spec().Prompts, []string{PromptID}) {
|
||||||
}
|
t.Fatal("prompt dependencies alias caller memory")
|
||||||
fingerprints := validator.CheckpointFingerprints()
|
|
||||||
if len(fingerprints) != 3 || fingerprints[0].Name != "prompt" || fingerprints[1].Name != "response_schema" || fingerprints[2] != (pipeline.CheckpointFingerprint{Name: "combat_policy", Value: combatPolicy}) {
|
|
||||||
t.Fatalf("fingerprints = %#v", fingerprints)
|
|
||||||
}
|
}
|
||||||
registry := pipeline.NewValidatorRegistry()
|
registry := pipeline.NewValidatorRegistry()
|
||||||
if err := Register(registry); err != nil {
|
if err := Register(registry); err != nil {
|
||||||
@@ -171,60 +127,36 @@ func TestValidatorConstructionOptionsAndMetadata(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestEvaluationCasesAreStrictAndCoverClassifications(t *testing.T) {
|
func newValidator(t *testing.T, client promptkit.LLMClient) *Validator {
|
||||||
content, err := os.ReadFile("testdata/evaluation_cases.json")
|
t.Helper()
|
||||||
|
type settings struct {
|
||||||
|
PromptKit promptkit.Settings `yaml:"promptkit"`
|
||||||
|
}
|
||||||
|
definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit })
|
||||||
|
filesystem := configkit.NewFilesystem("Notarius assets", assets.FS())
|
||||||
|
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("validator fixture", []byte(`promptkit:
|
||||||
|
backends:
|
||||||
|
- id: validator-backend
|
||||||
|
endpoint: https://example.invalid/v1
|
||||||
|
concurrency_limit: 1
|
||||||
|
queue_capacity: 0
|
||||||
|
profiles:
|
||||||
|
- id: validator-profile
|
||||||
|
backend: validator-backend
|
||||||
|
model: validator-test-model
|
||||||
|
api_key_required: false
|
||||||
|
`))}, configkit.LayeredOptions[settings]{Definition: definition, Underlays: []configkit.Underlay{
|
||||||
|
promptkit.BundledOpenRouterUnderlay(),
|
||||||
|
{Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}},
|
||||||
|
}})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
var cases []struct {
|
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client))
|
||||||
Name string `json:"name"`
|
if err != nil {
|
||||||
TranscriptUnits []struct {
|
t.Fatal(err)
|
||||||
ID int `json:"id"`
|
|
||||||
Text string `json:"text"`
|
|
||||||
} `json:"transcript_units"`
|
|
||||||
ProposedKind string `json:"proposed_kind"`
|
|
||||||
ExpectedClassification string `json:"expected_classification"`
|
|
||||||
ReviewerRationale string `json:"reviewer_rationale"`
|
|
||||||
}
|
}
|
||||||
decoder := json.NewDecoder(strings.NewReader(string(content)))
|
validator, err := New(engine, Options{})
|
||||||
decoder.DisallowUnknownFields()
|
|
||||||
if err := decoder.Decode(&cases); err != nil {
|
|
||||||
t.Fatalf("decode evaluation cases: %v", err)
|
|
||||||
}
|
|
||||||
if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) {
|
|
||||||
t.Fatalf("decode trailing evaluation data: %v, want EOF", err)
|
|
||||||
}
|
|
||||||
seenNames, seenClassifications := map[string]bool{}, map[string]bool{}
|
|
||||||
for _, value := range cases {
|
|
||||||
if strings.TrimSpace(value.Name) == "" || seenNames[value.Name] || len(value.TranscriptUnits) == 0 || strings.TrimSpace(value.ProposedKind) == "" || strings.TrimSpace(value.ReviewerRationale) == "" {
|
|
||||||
t.Fatalf("invalid evaluation case: %#v", value)
|
|
||||||
}
|
|
||||||
seenNames[value.Name] = true
|
|
||||||
switch value.ProposedKind {
|
|
||||||
case "combat", "narrative", "recap", "meta":
|
|
||||||
default:
|
|
||||||
t.Fatalf("unsupported proposed kind %q", value.ProposedKind)
|
|
||||||
}
|
|
||||||
switch value.ExpectedClassification {
|
|
||||||
case "combat", "non_combat":
|
|
||||||
seenClassifications[value.ExpectedClassification] = true
|
|
||||||
default:
|
|
||||||
t.Fatalf("unsupported classification %q", value.ExpectedClassification)
|
|
||||||
}
|
|
||||||
for _, unit := range value.TranscriptUnits {
|
|
||||||
if unit.ID <= 0 || strings.TrimSpace(unit.Text) == "" {
|
|
||||||
t.Fatalf("invalid transcript unit: %#v", unit)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(seenClassifications) != 2 {
|
|
||||||
t.Fatalf("evaluation classifications = %#v, want both classes", seenClassifications)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func newValidator(t *testing.T, client contracts.StructuredLLMClient) *Validator {
|
|
||||||
t.Helper()
|
|
||||||
validator, err := New(client, Options{})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("New() error = %v", err)
|
t.Fatalf("New() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -234,7 +166,7 @@ func newValidator(t *testing.T, client contracts.StructuredLLMClient) *Validator
|
|||||||
func validRequest(kind dnd.SceneKind) contracts.TypedValidationRequest[dnd.SceneDescriptionList] {
|
func validRequest(kind dnd.SceneKind) contracts.TypedValidationRequest[dnd.SceneDescriptionList] {
|
||||||
document := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{{ID: 1, Text: "The foes attack."}, {ID: 2, Text: "The party responds."}}}
|
document := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{{ID: 1, Text: "The foes attack."}, {ID: 2, Text: "The party responds."}}}
|
||||||
chunk := &source.Chunk{ID: "session:chunk:0", SourceID: document.ID, Ref: source.SourceRef{SourceID: document.ID, StartUnitID: 1, EndUnitID: 2}, Content: []byte(`{"units":[1,2]}`), MediaType: "application/json", Units: append([]source.SourceUnit(nil), document.Units...)}
|
chunk := &source.Chunk{ID: "session:chunk:0", SourceID: document.ID, Ref: source.SourceRef{SourceID: document.ID, StartUnitID: 1, EndUnitID: 2}, Content: []byte(`{"units":[1,2]}`), MediaType: "application/json", Units: append([]source.SourceUnit(nil), document.Units...)}
|
||||||
return contracts.TypedValidationRequest[dnd.SceneDescriptionList]{Stage: string(pipeline.StageExtract), Source: document, Chunk: chunk, SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session.json"), Value: dnd.SceneDescriptionList{Scenes: []dnd.SceneDescription{{ID: chunk.ID, SourceRef: chunk.Ref, Kind: kind, Title: "A Scene", Summary: "A valid scene."}}}}
|
return contracts.TypedValidationRequest[dnd.SceneDescriptionList]{Stage: string(pipeline.StageExtract), LLMProfile: "validator-profile", Source: document, Chunk: chunk, SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session.json"), Value: dnd.SceneDescriptionList{Scenes: []dnd.SceneDescription{{ID: chunk.ID, SourceRef: chunk.Ref, Kind: kind, Title: "A Scene", Summary: "A valid scene."}}}}
|
||||||
}
|
}
|
||||||
|
|
||||||
func cloneValidationRequest(request contracts.TypedValidationRequest[dnd.SceneDescriptionList]) contracts.TypedValidationRequest[dnd.SceneDescriptionList] {
|
func cloneValidationRequest(request contracts.TypedValidationRequest[dnd.SceneDescriptionList]) contracts.TypedValidationRequest[dnd.SceneDescriptionList] {
|
||||||
@@ -247,40 +179,28 @@ func cloneValidationRequest(request contracts.TypedValidationRequest[dnd.SceneDe
|
|||||||
return request
|
return request
|
||||||
}
|
}
|
||||||
|
|
||||||
func sortedInputNames(inputs contracts.LLMInputSet) []string {
|
|
||||||
names := make([]string, 0, len(inputs))
|
|
||||||
for name := range inputs {
|
|
||||||
names = append(names, name)
|
|
||||||
}
|
|
||||||
if len(names) == 2 && names[0] == "transcript" {
|
|
||||||
names[0], names[1] = names[1], names[0]
|
|
||||||
}
|
|
||||||
return names
|
|
||||||
}
|
|
||||||
|
|
||||||
type fakeCombatSemanticsClient struct {
|
type fakeCombatSemanticsClient struct {
|
||||||
response completionResponse
|
response classificationResponse
|
||||||
err error
|
raw string
|
||||||
requests []contracts.StructuredCompletionRequest
|
err error
|
||||||
|
requests []promptkit.GenerateRequest
|
||||||
|
onGenerate func(context.Context, promptkit.GenerateRequest)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (client *fakeCombatSemanticsClient) CompleteStructured(_ context.Context, request contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
func (client *fakeCombatSemanticsClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||||
cloned, err := contracts.CloneStructuredCompletionRequest(request)
|
client.requests = append(client.requests, request)
|
||||||
if err != nil {
|
if client.onGenerate != nil {
|
||||||
return contracts.StructuredCompletionResponse{}, err
|
client.onGenerate(ctx, request)
|
||||||
}
|
}
|
||||||
client.requests = append(client.requests, cloned)
|
|
||||||
if client.err != nil {
|
if client.err != nil {
|
||||||
return contracts.StructuredCompletionResponse{}, client.err
|
return nil, client.err
|
||||||
}
|
}
|
||||||
response, ok := out.(*completionResponse)
|
if client.raw != "" {
|
||||||
if !ok {
|
return &promptkit.GenerateResponse{Content: client.raw}, nil
|
||||||
return contracts.StructuredCompletionResponse{}, errors.New("unexpected completion target")
|
|
||||||
}
|
}
|
||||||
*response = client.response
|
|
||||||
content, err := json.Marshal(client.response)
|
content, err := json.Marshal(client.response)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return contracts.StructuredCompletionResponse{}, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return contracts.StructuredCompletionResponse{Content: content}, nil
|
return &promptkit.GenerateResponse{Content: string(content)}, nil
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user