Migrate combat semantics validation to native execution and unavailable assessments
This commit is contained in:
@@ -0,0 +1,61 @@
|
||||
package combat_semantics
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestEvaluationCasesAreStrictAndCoverClassifications(t *testing.T) {
|
||||
content, err := os.ReadFile("testdata/evaluation_cases.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var cases []struct {
|
||||
Name string `json:"name"`
|
||||
TranscriptUnits []struct {
|
||||
ID int `json:"id"`
|
||||
Text string `json:"text"`
|
||||
} `json:"transcript_units"`
|
||||
ProposedKind string `json:"proposed_kind"`
|
||||
ExpectedClassification string `json:"expected_classification"`
|
||||
ReviewerRationale string `json:"reviewer_rationale"`
|
||||
}
|
||||
decoder := json.NewDecoder(strings.NewReader(string(content)))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(&cases); err != nil {
|
||||
t.Fatalf("decode evaluation cases: %v", err)
|
||||
}
|
||||
if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) {
|
||||
t.Fatalf("decode trailing evaluation data: %v, want EOF", err)
|
||||
}
|
||||
seenNames, seenClassifications := map[string]bool{}, map[string]bool{}
|
||||
for _, value := range cases {
|
||||
if strings.TrimSpace(value.Name) == "" || seenNames[value.Name] || len(value.TranscriptUnits) == 0 || strings.TrimSpace(value.ProposedKind) == "" || strings.TrimSpace(value.ReviewerRationale) == "" {
|
||||
t.Fatalf("invalid evaluation case: %#v", value)
|
||||
}
|
||||
seenNames[value.Name] = true
|
||||
switch value.ProposedKind {
|
||||
case "combat", "narrative", "recap", "meta":
|
||||
default:
|
||||
t.Fatalf("unsupported proposed kind %q", value.ProposedKind)
|
||||
}
|
||||
switch value.ExpectedClassification {
|
||||
case "combat", "non_combat":
|
||||
seenClassifications[value.ExpectedClassification] = true
|
||||
default:
|
||||
t.Fatalf("unsupported classification %q", value.ExpectedClassification)
|
||||
}
|
||||
for _, unit := range value.TranscriptUnits {
|
||||
if unit.ID <= 0 || strings.TrimSpace(unit.Text) == "" {
|
||||
t.Fatalf("invalid transcript unit: %#v", unit)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(seenClassifications) != 2 {
|
||||
t.Fatalf("evaluation classifications = %#v, want both classes", seenClassifications)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package combat_semantics
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
)
|
||||
|
||||
const (
|
||||
ReasonCodeActiveCombatNotClassified = "scene_active_combat_not_classified"
|
||||
ReasonCodeCombatClassificationUnsupported = "scene_combat_classification_unsupported"
|
||||
maximumExplanationRunes = 512
|
||||
)
|
||||
|
||||
type classificationResponse struct {
|
||||
Classification string `json:"classification"`
|
||||
Explanation string `json:"explanation"`
|
||||
}
|
||||
|
||||
func interpretResponse(response classificationResponse, proposedKind dnd.SceneKind) (contracts.ValidationResult, error) {
|
||||
explanation, err := validateExplanation(response.Explanation)
|
||||
if err != nil {
|
||||
return contracts.ValidationResult{}, err
|
||||
}
|
||||
switch response.Classification {
|
||||
case "combat":
|
||||
if proposedKind == dnd.SceneKindCombat {
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
}
|
||||
return contracts.ValidationResult{
|
||||
Approved: false,
|
||||
ReasonCode: ReasonCodeActiveCombatNotClassified,
|
||||
Message: "The current chunk contains substantive active combat that is not classified as combat.",
|
||||
CorrectionGuidance: "Return kind: combat for this scene. " + explanation,
|
||||
}, nil
|
||||
case "non_combat":
|
||||
if proposedKind != dnd.SceneKindCombat {
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
}
|
||||
return contracts.ValidationResult{
|
||||
Approved: false,
|
||||
ReasonCode: ReasonCodeCombatClassificationUnsupported,
|
||||
Message: "The current chunk does not support a combat classification.",
|
||||
CorrectionGuidance: "Choose the appropriate narrative, recap, or meta kind for this scene. " + explanation,
|
||||
}, nil
|
||||
default:
|
||||
return contracts.ValidationResult{}, validatorErrorf("unsupported combat-semantics classification %q", response.Classification)
|
||||
}
|
||||
}
|
||||
|
||||
func validateExplanation(value string) (string, error) {
|
||||
if !utf8.ValidString(value) {
|
||||
return "", validatorErrorf("validator explanation must be valid UTF-8")
|
||||
}
|
||||
trimmed := strings.TrimSpace(value)
|
||||
if trimmed == "" {
|
||||
return "", validatorErrorf("validator explanation must not be blank")
|
||||
}
|
||||
if utf8.RuneCountInString(trimmed) > maximumExplanationRunes {
|
||||
return "", validatorErrorf("validator explanation exceeds %d Unicode code points", maximumExplanationRunes)
|
||||
}
|
||||
return trimmed, nil
|
||||
}
|
||||
|
||||
func validatorErrorf(format string, args ...any) error {
|
||||
return fmt.Errorf("scene combat-semantics validator: "+format, args...)
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package combat_semantics
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
)
|
||||
|
||||
func TestValidatorInterpretsCombatSemanticsResponses(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
kind dnd.SceneKind
|
||||
response classificationResponse
|
||||
approved bool
|
||||
reasonCode string
|
||||
guidance string
|
||||
correction string
|
||||
wantError string
|
||||
}{
|
||||
{name: "approve non-combat", kind: dnd.SceneKindNarrative, response: classificationResponse{Classification: "non_combat", Explanation: "No active encounter occurs."}, approved: true},
|
||||
{name: "approve combat", kind: dnd.SceneKindCombat, response: classificationResponse{Classification: "combat", Explanation: "Initiative and attacks organize the chunk."}, approved: true},
|
||||
{name: "add combat", kind: dnd.SceneKindNarrative, response: classificationResponse{Classification: "combat", Explanation: "The combatants exchange attacks and damage."}, reasonCode: ReasonCodeActiveCombatNotClassified, guidance: "Return kind: combat"},
|
||||
{name: "remove combat", kind: dnd.SceneKindCombat, response: classificationResponse{Classification: "non_combat", Explanation: "The group only plans for a possible fight."}, reasonCode: ReasonCodeCombatClassificationUnsupported, guidance: "narrative, recap, or meta"},
|
||||
{name: "unknown classification", kind: dnd.SceneKindNarrative, response: classificationResponse{Classification: "uncertain", Explanation: "The group only plans."}, wantError: "unsupported"},
|
||||
{name: "blank explanation", kind: dnd.SceneKindNarrative, response: classificationResponse{Classification: "non_combat", Explanation: " "}, wantError: "blank"},
|
||||
{name: "trim explanation", kind: dnd.SceneKindNarrative, response: classificationResponse{Classification: "combat", Explanation: " The combatants exchange attacks.\n"}, reasonCode: ReasonCodeActiveCombatNotClassified, correction: "Return kind: combat for this scene. The combatants exchange attacks."},
|
||||
{name: "oversized explanation", kind: dnd.SceneKindNarrative, response: classificationResponse{Classification: "non_combat", Explanation: strings.Repeat("界", 513)}, wantError: "exceeds"},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
result, err := interpretResponse(test.response, test.kind)
|
||||
if test.wantError != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), test.wantError) {
|
||||
t.Fatalf("Validate() error = %v, want %q", err, test.wantError)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil || result.Approved != test.approved || result.ReasonCode != test.reasonCode || !strings.Contains(result.CorrectionGuidance, test.guidance) {
|
||||
t.Fatalf("Validate() = %#v, %v", result, err)
|
||||
}
|
||||
if test.correction != "" && result.CorrectionGuidance != test.correction {
|
||||
t.Fatalf("CorrectionGuidance = %q, want %q", result.CorrectionGuidance, test.correction)
|
||||
}
|
||||
if !result.Approved && strings.Contains(result.CorrectionGuidance, result.ReasonCode) {
|
||||
t.Fatalf("correction guidance leaked opaque application detail: %q", result.CorrectionGuidance)
|
||||
}
|
||||
if err := contracts.ValidateValidationResult(result); err != nil {
|
||||
t.Fatalf("ValidateValidationResult(%#v) = %v", result, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
package combat_semantics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestNativeValidatorReturnsOnlyCompletedSemanticJudgments(t *testing.T) {
|
||||
for _, classification := range []string{"combat", "non_combat"} {
|
||||
for _, proposed := range []dnd.SceneKind{dnd.SceneKindCombat, dnd.SceneKindNarrative} {
|
||||
request := validRequest(proposed)
|
||||
client := &fakeCombatSemanticsClient{response: classificationResponse{Classification: classification, Explanation: "The supplied transcript supports this judgment."}}
|
||||
result, err := newValidator(t, client).Validate(context.Background(), request)
|
||||
wantApproval := (classification == "combat") == (proposed == dnd.SceneKindCombat)
|
||||
if err != nil || result.Approved != wantApproval || len(client.requests) != 1 {
|
||||
t.Fatalf("judgment=%#v %v", result, err)
|
||||
}
|
||||
if !wantApproval && (strings.TrimSpace(result.CorrectionGuidance) == "" || strings.Contains(result.CorrectionGuidance, result.ReasonCode)) {
|
||||
t.Fatal("rejection lost actionable guidance or copied diagnostic labels")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeFailuresAreUnavailableRatherThanRejections(t *testing.T) {
|
||||
failure := errors.New("provider unavailable")
|
||||
for _, test := range []struct {
|
||||
name, raw string
|
||||
failure error
|
||||
}{
|
||||
{"provider failure", "", failure},
|
||||
{"invalid JSON", "{", nil},
|
||||
{"invalid classification", `{"classification":"uncertain","explanation":"Uncertain."}`, nil},
|
||||
{"uninterpretable explanation", `{"classification":"combat","explanation":" "}`, nil},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
client := &fakeCombatSemanticsClient{raw: test.raw, err: test.failure}
|
||||
request := validRequest(dnd.SceneKindCombat)
|
||||
prepared := 0
|
||||
request.Execution.OnPrepared = func(context.Context, promptkit.PreparedRun) { prepared++ }
|
||||
result, err := newValidator(t, client).Validate(context.Background(), request)
|
||||
var unavailable *contracts.ValidationUnavailableError
|
||||
if !errors.As(err, &unavailable) || result.Approved || result.ReasonCode != "" || result.CorrectionGuidance != "" || prepared != 1 {
|
||||
t.Fatalf("unavailable assessment=%#v %v, native executions=%d", result, err, prepared)
|
||||
}
|
||||
if test.failure != nil && (!errors.Is(err, failure) || len(client.requests) != 1) {
|
||||
t.Fatalf("provider failure was flattened or retried: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNilErrorFailedNativeValidationCannotBecomeApproval(t *testing.T) {
|
||||
client := &fakeCombatSemanticsClient{raw: `{"classification":"combat"}`}
|
||||
validator := newValidator(t, client)
|
||||
request := validRequest(dnd.SceneKindCombat)
|
||||
native, err := validator.engine.Run(context.Background(), promptkit.RunRequest{
|
||||
PromptID: PromptID, ProfileID: request.LLMProfile,
|
||||
Inputs: map[string]promptkit.ArtifactRef{"transcript": promptkit.Inline(string(request.Chunk.Content))},
|
||||
})
|
||||
if err != nil || native == nil || native.Validation.Status != promptkit.ValidationFailed || native.Validation.IsValid {
|
||||
t.Fatalf("fixture did not exercise nil-error structural failure: %#v %v", native, err)
|
||||
}
|
||||
result, err := validator.Validate(context.Background(), request)
|
||||
var unavailable *contracts.ValidationUnavailableError
|
||||
if !errors.As(err, &unavailable) || !errors.Is(err, promptkit.ErrValidation) || result.Approved || result.CorrectionGuidance != "" {
|
||||
t.Fatalf("failed native contract accepted/interpreted: %#v %v", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNestedValidatorUsesTheReleasedNativeBackendAdmission(t *testing.T) {
|
||||
client := &fakeCombatSemanticsClient{response: classificationResponse{Classification: "non_combat", Explanation: "No active encounter occurs."}}
|
||||
validator := newValidator(t, client)
|
||||
request := validRequest(dnd.SceneKindNarrative)
|
||||
assessments := 0
|
||||
operation := contracts.ProducerOperation[dnd.SceneDescriptionList]{
|
||||
Controls: contracts.ProducerControls{Mode: contracts.ModeStrict}, ReuseEligible: true,
|
||||
Assess: func(ctx context.Context, candidate dnd.SceneDescriptionList) (contracts.CandidateAssessment, error) {
|
||||
assessments++
|
||||
nested := request
|
||||
nested.Value = candidate
|
||||
judgment, err := validator.Validate(ctx, nested)
|
||||
if err != nil {
|
||||
return contracts.CandidateAssessment{}, err
|
||||
}
|
||||
decision := contracts.AssessmentRejected
|
||||
if judgment.Approved {
|
||||
decision = contracts.AssessmentApproved
|
||||
}
|
||||
return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{ValidatorName: Key, Decision: decision, Result: judgment}}}, nil
|
||||
},
|
||||
}
|
||||
_, outcome, err := pipeline.RunProducer(context.Background(), validator.engine, operation, promptkit.RunRequest{
|
||||
PromptID: PromptID, ProfileID: request.LLMProfile,
|
||||
Inputs: map[string]promptkit.ArtifactRef{"transcript": promptkit.Inline(string(request.Chunk.Content))},
|
||||
}, func(context.Context, *promptkit.RunResult) (*dnd.SceneDescriptionList, []string, error) {
|
||||
return &request.Value, nil, nil
|
||||
}, nil, nil)
|
||||
if err != nil || outcome.Disposition != contracts.ProducerAccepted || assessments != 1 || len(client.requests) != 2 {
|
||||
t.Fatalf("nested native admission=%#v %v, assessments=%d calls=%d", outcome, err, assessments, len(client.requests))
|
||||
}
|
||||
for _, call := range client.requests {
|
||||
if call.Target.BackendID != "validator-backend" {
|
||||
t.Fatal("fixture did not exercise the same limited backend")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCapacityFailureAndCancellationNeverBecomeSemanticGuidance(t *testing.T) {
|
||||
t.Run("capacity unavailable", func(t *testing.T) {
|
||||
var validator *Validator
|
||||
var nestedErr error
|
||||
request := validRequest(dnd.SceneKindNarrative)
|
||||
client := &fakeCombatSemanticsClient{response: classificationResponse{Classification: "non_combat", Explanation: "No encounter."}}
|
||||
client.onGenerate = func(ctx context.Context, _ promptkit.GenerateRequest) {
|
||||
// The outer fake provider still holds the single active native admission.
|
||||
// The configured zero queue makes this a capacity error, not a waiting test.
|
||||
_, nestedErr = validator.Validate(ctx, request)
|
||||
}
|
||||
validator = newValidator(t, client)
|
||||
result, err := validator.Validate(context.Background(), request)
|
||||
var unavailable *contracts.ValidationUnavailableError
|
||||
if err != nil || !result.Approved || !errors.As(nestedErr, &unavailable) || !errors.Is(nestedErr, promptkit.ErrCapacityExceeded) || len(client.requests) != 1 {
|
||||
t.Fatalf("capacity assessment=%#v %v, nested=%v", result, err, nestedErr)
|
||||
}
|
||||
})
|
||||
t.Run("canceled delivery", func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
client := &fakeCombatSemanticsClient{response: classificationResponse{Classification: "non_combat", Explanation: "No encounter."},
|
||||
onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }}
|
||||
result, err := newValidator(t, client).Validate(ctx, validRequest(dnd.SceneKindNarrative))
|
||||
if !errors.Is(err, context.Canceled) || result.Approved || result.CorrectionGuidance != "" || len(client.requests) != 1 {
|
||||
t.Fatalf("cancellation=%#v %v", result, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -1,72 +0,0 @@
|
||||
package combat_semantics
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"sync"
|
||||
|
||||
rootassets "gitea.maximumdirect.net/eric/notarius/assets"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||
)
|
||||
|
||||
const promptAssetRoot = "assets/prompts"
|
||||
|
||||
var promptAssetManifest = shared.PromptAssetManifest{
|
||||
ModuleDir: PromptID,
|
||||
ModuleFiles: []promptfs.ModulePromptFile{
|
||||
{Name: "prompt.yaml", Path: "prompts/prompt.yaml"},
|
||||
{Name: "instructions.md", Path: "prompts/instructions.md"},
|
||||
},
|
||||
SharedFiles: []string{
|
||||
"common-dnd-system.md",
|
||||
"common-dnd-scene-combat-policy.md",
|
||||
"common-dnd-transcript-chunk.md",
|
||||
},
|
||||
}
|
||||
|
||||
func moduleAssetFS() (fs.FS, error) {
|
||||
assets, err := fs.Sub(rootassets.FS(), "dnd/scene-descriptions/validate/combat-semantics")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scope scene combat-semantics validator assets: %w", err)
|
||||
}
|
||||
return assets, nil
|
||||
}
|
||||
|
||||
func RegisterPromptAssets(registry *llm.AssetRegistry) error {
|
||||
assets, err := moduleAssetFS()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
promptFS, err := promptAssetManifest.PromptFS(assets)
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare scene combat-semantics validator prompt assets: %w", err)
|
||||
}
|
||||
if err := registry.RegisterPromptFS(promptFS, promptAssetRoot); err != nil {
|
||||
return err
|
||||
}
|
||||
schemas, err := fs.Sub(assets, "schemas")
|
||||
if err != nil {
|
||||
return fmt.Errorf("scope scene combat-semantics validator schemas: %w", err)
|
||||
}
|
||||
return registry.RegisterSchemaFS(schemas, ".")
|
||||
}
|
||||
|
||||
func promptAssetMetadata() (string, error) {
|
||||
promptAssetHashOnce.Do(func() {
|
||||
assets, err := moduleAssetFS()
|
||||
if err != nil {
|
||||
promptAssetHashErr = err
|
||||
return
|
||||
}
|
||||
promptAssetHash, promptAssetHashErr = promptAssetManifest.Hash(assets)
|
||||
})
|
||||
return promptAssetHash, promptAssetHashErr
|
||||
}
|
||||
|
||||
var (
|
||||
promptAssetHashOnce sync.Once
|
||||
promptAssetHash string
|
||||
promptAssetHashErr error
|
||||
)
|
||||
@@ -1,90 +0,0 @@
|
||||
package combat_semantics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestRegisterPromptAssetsPreparesCombatSemanticsPrompt(t *testing.T) {
|
||||
registry := llm.NewAssetRegistry()
|
||||
if err := RegisterPromptAssets(registry); err != nil {
|
||||
t.Fatalf("RegisterPromptAssets() error = %v, want nil", err)
|
||||
}
|
||||
engine := newCombatSemanticsPromptEngine(t, registry)
|
||||
inputs := map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.InlineWithURI("file:///session.json", `{"sentinel":"combat-semantics-transcript"}`),
|
||||
}
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "combat-semantics-test-profile", Inputs: inputs,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Prepare() error = %v, want nil", err)
|
||||
}
|
||||
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_scene_combat_semantics_llm.v1.json" {
|
||||
t.Fatalf("prepared prompt = %#v, want combat-semantics prompt identity and schema wiring", prepared)
|
||||
}
|
||||
if len(prepared.Messages) != 4 {
|
||||
t.Fatalf("prepared message count = %d, want 4", len(prepared.Messages))
|
||||
}
|
||||
for index, message := range prepared.Messages {
|
||||
wantEphemeral := index == 3
|
||||
gotEphemeral := message.CacheControl != nil && message.CacheControl.Type == promptkit.CacheControlEphemeral
|
||||
if gotEphemeral != wantEphemeral {
|
||||
t.Fatalf("message %d cache control = %#v, want ephemeral=%t", index, message.CacheControl, wantEphemeral)
|
||||
}
|
||||
}
|
||||
assertRenderedExactlyOnce(t, prepared.Messages, "substantive active combat materially organizes the", 1)
|
||||
assertRenderedExactlyOnce(t, prepared.Messages, "combat-semantics-transcript", 3)
|
||||
for name := range inputs {
|
||||
missing := make(map[string]promptkit.ArtifactRef, len(inputs)-1)
|
||||
for inputName, input := range inputs {
|
||||
if inputName != name {
|
||||
missing[inputName] = input
|
||||
}
|
||||
}
|
||||
if _, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "combat-semantics-test-profile", Inputs: missing,
|
||||
}); err == nil {
|
||||
t.Fatalf("Prepare() without required %q input = nil error", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptAssetMetadataIsContentSafe(t *testing.T) {
|
||||
hash, err := promptAssetMetadata()
|
||||
if err != nil || !strings.HasPrefix(hash, "sha256:") {
|
||||
t.Fatalf("promptAssetMetadata() = %q, %v; want hash", hash, err)
|
||||
}
|
||||
if strings.Contains(hash, "non_combat") {
|
||||
t.Fatalf("prompt asset metadata leaked raw prompt content: %q", hash)
|
||||
}
|
||||
}
|
||||
|
||||
func newCombatSemanticsPromptEngine(t *testing.T, registry *llm.AssetRegistry) *promptkit.Engine {
|
||||
t.Helper()
|
||||
return promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
|
||||
"combat-semantics-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "combat-semantics-test-model"),
|
||||
})
|
||||
}
|
||||
|
||||
func assertRenderedExactlyOnce(t *testing.T, messages []promptkit.RenderedMessage, sentinel string, wantIndex int) {
|
||||
t.Helper()
|
||||
index := -1
|
||||
occurrences := 0
|
||||
for messageIndex, message := range messages {
|
||||
count := strings.Count(message.Content, sentinel)
|
||||
if count > 0 {
|
||||
index = messageIndex
|
||||
occurrences += count
|
||||
}
|
||||
}
|
||||
if occurrences != 1 || index != wantIndex {
|
||||
t.Fatalf("sentinel %q rendered %d times at message %d, want once at message %d", sentinel, occurrences, index, wantIndex)
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
package combat_semantics
|
||||
|
||||
import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
|
||||
const (
|
||||
PromptID = "dnd.scene_descriptions.validate_combat"
|
||||
ResponseSchemaKey = llm.ResponseSchemaKey("dnd_scene_combat_semantics_llm")
|
||||
ResponseSchemaID = "notarius.dnd.scene_descriptions.combat_semantics.llm"
|
||||
ResponseSchemaName = "notarius_dnd_scene_combat_semantics_llm_v1"
|
||||
SchemaVersion = "v1"
|
||||
)
|
||||
|
||||
func loadResponseSchema() (llm.ResponseSchema, error) {
|
||||
assets, err := moduleAssetFS()
|
||||
if err != nil {
|
||||
return llm.ResponseSchema{}, err
|
||||
}
|
||||
return llm.LoadResponseSchema(assets, llm.ResponseSchemaDefinition{
|
||||
Key: ResponseSchemaKey,
|
||||
ID: ResponseSchemaID,
|
||||
Version: SchemaVersion,
|
||||
Name: ResponseSchemaName,
|
||||
AssetPath: "schemas/dnd_scene_combat_semantics_llm.v1.json",
|
||||
})
|
||||
}
|
||||
@@ -1,86 +0,0 @@
|
||||
package combat_semantics
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/santhosh-tekuri/jsonschema/v6"
|
||||
)
|
||||
|
||||
func TestLoadResponseSchemaUsesStrictCombatSemanticsContract(t *testing.T) {
|
||||
schema, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
|
||||
}
|
||||
if schema.Key != ResponseSchemaKey || schema.ID != ResponseSchemaID || schema.Version != SchemaVersion || schema.Name != ResponseSchemaName || !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) {
|
||||
t.Fatalf("schema = %#v, want private combat-semantics schema identity", schema)
|
||||
}
|
||||
for _, classification := range []string{"combat", "non_combat"} {
|
||||
if err := validateCombatSemanticsSchema(map[string]any{"classification": classification, "explanation": "The chunk contains sustained attack exchanges."}, schema.JSONSchema); err != nil {
|
||||
t.Fatalf("valid %q classification rejected: %v", classification, err)
|
||||
}
|
||||
}
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
response map[string]any
|
||||
}{
|
||||
{name: "missing explanation", response: map[string]any{"classification": "combat"}},
|
||||
{name: "missing classification", response: map[string]any{"explanation": "The chunk contains active combat."}},
|
||||
{name: "unknown property", response: map[string]any{"classification": "combat", "explanation": "The chunk contains active combat.", "confidence": 1}},
|
||||
{name: "wrong classification type", response: map[string]any{"classification": 1, "explanation": "The chunk contains active combat."}},
|
||||
{name: "wrong explanation type", response: map[string]any{"classification": "combat", "explanation": 1}},
|
||||
{name: "unsupported classification", response: map[string]any{"classification": "uncertain", "explanation": "The classification is uncertain."}},
|
||||
{name: "blank explanation", response: map[string]any{"classification": "combat", "explanation": ""}},
|
||||
{name: "oversized explanation", response: map[string]any{"classification": "combat", "explanation": strings.Repeat("a", 513)}},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if err := validateCombatSemanticsSchema(test.response, schema.JSONSchema); err == nil {
|
||||
t.Fatal("invalid private response accepted")
|
||||
}
|
||||
})
|
||||
}
|
||||
if bytes.Contains(schema.JSONSchema, []byte("uniqueItems")) {
|
||||
t.Fatalf("schema included unsupported uniqueItems constraint: %s", schema.JSONSchema)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResponseSchemaIsMutationSafeAndDiagnosticsRedactContent(t *testing.T) {
|
||||
first, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first.JSONSchema[0] = '['
|
||||
second, err := loadResponseSchema()
|
||||
if err != nil || !strings.HasPrefix(second.SHA256, "sha256:") || !json.Valid(second.JSONSchema) || bytes.Equal(first.JSONSchema, second.JSONSchema) {
|
||||
t.Fatalf("second schema = %s, %v; want defensive copy", second.JSONSchema, err)
|
||||
}
|
||||
if diagnostics := second.DiagnosticsMap(); diagnostics["json_schema"] != nil {
|
||||
t.Fatalf("schema diagnostics included raw content: %#v", diagnostics)
|
||||
}
|
||||
}
|
||||
|
||||
func validateCombatSemanticsSchema(instance map[string]any, schemaContent []byte) error {
|
||||
instanceContent, err := json.Marshal(instance)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
parsedInstance, err := jsonschema.UnmarshalJSON(bytes.NewReader(instanceContent))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
parsedSchema, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaContent))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
compiler := jsonschema.NewCompiler()
|
||||
if err := compiler.AddResource("schema.json", parsedSchema); err != nil {
|
||||
return err
|
||||
}
|
||||
compiled, err := compiler.Compile("schema.json")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return compiled.Validate(parsedInstance)
|
||||
}
|
||||
@@ -2,9 +2,10 @@ package combat_semantics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"go.maximumdirect.net/promptkit"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
@@ -14,44 +15,20 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/shape"
|
||||
)
|
||||
|
||||
const (
|
||||
Key = "extract/dnd/scene-descriptions/combat_semantics"
|
||||
ReasonCodeActiveCombatNotClassified = "scene_active_combat_not_classified"
|
||||
ReasonCodeCombatClassificationUnsupported = "scene_combat_classification_unsupported"
|
||||
combatPolicy = "dnd.scene_descriptions.combat_semantics.v1"
|
||||
maximumExplanationRunes = 512
|
||||
)
|
||||
const Key = "extract/dnd/scene-descriptions/combat_semantics"
|
||||
const PromptID = "notarius/dnd.scene_descriptions.validate_combat"
|
||||
|
||||
type Options struct{}
|
||||
|
||||
type completionResponse struct {
|
||||
Classification string `json:"classification"`
|
||||
Explanation string `json:"explanation"`
|
||||
}
|
||||
|
||||
type Validator struct {
|
||||
llm contracts.StructuredLLMClient
|
||||
promptSHA string
|
||||
responseSchemaSHA string
|
||||
}
|
||||
type Validator struct{ engine *promptkit.Engine }
|
||||
|
||||
var _ contracts.TypedValidator[dnd.SceneDescriptionList] = (*Validator)(nil)
|
||||
var _ contracts.ManifestMetadataProvider = (*Validator)(nil)
|
||||
var _ pipeline.CheckpointFingerprintProvider = (*Validator)(nil)
|
||||
|
||||
func New(llmClient contracts.StructuredLLMClient, _ Options) (*Validator, error) {
|
||||
if llmClient == nil {
|
||||
return nil, validatorErrorf("LLM client must not be nil")
|
||||
func New(engine *promptkit.Engine, _ Options) (*Validator, error) {
|
||||
if engine == nil {
|
||||
return nil, validatorErrorf("native engine must not be nil")
|
||||
}
|
||||
promptSHA, err := promptAssetMetadata()
|
||||
if err != nil {
|
||||
return nil, validatorErrorf("load prompt metadata: %w", err)
|
||||
}
|
||||
responseSchema, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
return nil, validatorErrorf("load response schema: %w", err)
|
||||
}
|
||||
return &Validator{llm: llmClient, promptSHA: promptSHA, responseSchemaSHA: responseSchema.SHA256}, nil
|
||||
return &Validator{engine: engine}, nil
|
||||
}
|
||||
|
||||
func (v *Validator) Name() string { return Key }
|
||||
@@ -60,62 +37,46 @@ func (v *Validator) ExecutionClass() contracts.ExecutionClass {
|
||||
return contracts.ExecutionClassLLMBacked
|
||||
}
|
||||
|
||||
func (v *Validator) ManifestMetadata() map[string]any {
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
return map[string]any{
|
||||
"prompt_id": PromptID,
|
||||
"prompt_version": SchemaVersion,
|
||||
"prompt_sha256": v.promptSHA,
|
||||
"response_schema_key": string(ResponseSchemaKey),
|
||||
"response_schema_id": ResponseSchemaID,
|
||||
"response_schema_name": ResponseSchemaName,
|
||||
"response_schema_version": SchemaVersion,
|
||||
"response_schema_sha256": v.responseSchemaSHA,
|
||||
"combat_policy": combatPolicy,
|
||||
}
|
||||
}
|
||||
|
||||
func (v *Validator) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
return []pipeline.CheckpointFingerprint{
|
||||
{Name: "prompt", Value: v.promptSHA},
|
||||
{Name: "response_schema", Value: v.responseSchemaSHA},
|
||||
{Name: "combat_policy", Value: combatPolicy},
|
||||
}
|
||||
}
|
||||
|
||||
func (v *Validator) Validate(ctx context.Context, req contracts.TypedValidationRequest[dnd.SceneDescriptionList]) (contracts.ValidationResult, error) {
|
||||
if v == nil {
|
||||
return contracts.ValidationResult{}, validatorErrorf("validator must not be nil")
|
||||
}
|
||||
if v.llm == nil {
|
||||
return contracts.ValidationResult{}, validatorErrorf("LLM client must not be nil")
|
||||
if v.engine == nil {
|
||||
return contracts.ValidationResult{}, validatorErrorf("native engine must not be nil")
|
||||
}
|
||||
sourceInput, scene, err := validateRequest(ctx, req)
|
||||
if err != nil {
|
||||
return contracts.ValidationResult{}, err
|
||||
}
|
||||
|
||||
var response completionResponse
|
||||
_, err = v.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||
StageName: Key,
|
||||
PromptID: PromptID,
|
||||
PromptVersion: SchemaVersion,
|
||||
ProfileID: req.LLMProfile,
|
||||
SessionID: req.SessionID,
|
||||
StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts,
|
||||
Inputs: contracts.LLMInputSet{
|
||||
"transcript": shared.TranscriptPromptMaterial(sourceInput),
|
||||
},
|
||||
}, &response)
|
||||
if err != nil {
|
||||
return contracts.ValidationResult{}, validatorErrorf("complete structured output: %w", err)
|
||||
material := shared.TranscriptPromptMaterial(sourceInput)
|
||||
result, err := v.engine.Run(ctx, promptkit.RunRequest{
|
||||
PromptID: PromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID,
|
||||
Inputs: map[string]promptkit.ArtifactRef{"transcript": promptkit.InlineWithURI(material.OriginURI, string(material.Content))},
|
||||
Execution: req.Execution.Execution, TransientRetries: req.Execution.TransientRetries,
|
||||
Observer: req.Execution.Observer, OnPrepared: req.Execution.OnPrepared,
|
||||
})
|
||||
if canceled := ctx.Err(); canceled != nil {
|
||||
return contracts.ValidationResult{}, canceled
|
||||
}
|
||||
return interpretResponse(response, scene.Kind)
|
||||
if err != nil {
|
||||
return contracts.ValidationResult{}, unavailable(err)
|
||||
}
|
||||
if result == nil || result.Validation.Status == promptkit.ValidationFailed || !result.Validation.IsValid {
|
||||
return contracts.ValidationResult{}, unavailable(validatorErrorf("native output contract failed: %w", promptkit.ErrValidation))
|
||||
}
|
||||
var response classificationResponse
|
||||
if err := json.Unmarshal([]byte(result.RawOutput), &response); err != nil {
|
||||
return contracts.ValidationResult{}, unavailable(validatorErrorf("decode native output: %w", err))
|
||||
}
|
||||
assessment, err := interpretResponse(response, scene.Kind)
|
||||
if canceled := ctx.Err(); canceled != nil {
|
||||
return contracts.ValidationResult{}, canceled
|
||||
}
|
||||
if err != nil {
|
||||
return contracts.ValidationResult{}, unavailable(err)
|
||||
}
|
||||
return assessment, nil
|
||||
}
|
||||
|
||||
func validateRequest(ctx context.Context, req contracts.TypedValidationRequest[dnd.SceneDescriptionList]) (contracts.LLMInputMaterial, dnd.SceneDescription, error) {
|
||||
@@ -128,7 +89,7 @@ func validateRequest(ctx context.Context, req contracts.TypedValidationRequest[d
|
||||
if req.Stage != string(pipeline.StageExtract) {
|
||||
return contracts.LLMInputMaterial{}, dnd.SceneDescription{}, validatorErrorf("validation stage must be extract, got %q", req.Stage)
|
||||
}
|
||||
sourceInput, err := shared.PrepareChunkExtraction(ctx, contracts.TypedExtractionRequest{Source: req.Source, Chunk: req.Chunk, SourceInput: req.SourceInput})
|
||||
sourceInput, err := shared.PrepareChunkExtraction(ctx, contracts.TypedExtractionRequest[dnd.SceneDescriptionList]{Source: req.Source, Chunk: req.Chunk, SourceInput: req.SourceInput})
|
||||
if err != nil {
|
||||
return contracts.LLMInputMaterial{}, dnd.SceneDescription{}, validatorErrorf("prepare chunk source input: %w", err)
|
||||
}
|
||||
@@ -154,53 +115,12 @@ func validateRequest(ctx context.Context, req contracts.TypedValidationRequest[d
|
||||
return sourceInput, scene, nil
|
||||
}
|
||||
|
||||
func interpretResponse(response completionResponse, proposedKind dnd.SceneKind) (contracts.ValidationResult, error) {
|
||||
explanation, err := validateExplanation(response.Explanation)
|
||||
if err != nil {
|
||||
return contracts.ValidationResult{}, err
|
||||
}
|
||||
switch response.Classification {
|
||||
case "combat":
|
||||
if proposedKind == dnd.SceneKindCombat {
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
}
|
||||
return contracts.ValidationResult{
|
||||
Approved: false,
|
||||
ReasonCode: ReasonCodeActiveCombatNotClassified,
|
||||
Message: "The current chunk contains substantive active combat that is not classified as combat.",
|
||||
CorrectionGuidance: "Return kind: combat for this scene. " + explanation,
|
||||
}, nil
|
||||
case "non_combat":
|
||||
if proposedKind != dnd.SceneKindCombat {
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
}
|
||||
return contracts.ValidationResult{
|
||||
Approved: false,
|
||||
ReasonCode: ReasonCodeCombatClassificationUnsupported,
|
||||
Message: "The current chunk does not support a combat classification.",
|
||||
CorrectionGuidance: "Choose the appropriate narrative, recap, or meta kind for this scene. " + explanation,
|
||||
}, nil
|
||||
default:
|
||||
return contracts.ValidationResult{}, validatorErrorf("unsupported combat-semantics classification %q", response.Classification)
|
||||
}
|
||||
}
|
||||
|
||||
func validateExplanation(value string) (string, error) {
|
||||
if !utf8.ValidString(value) {
|
||||
return "", validatorErrorf("validator explanation must be valid UTF-8")
|
||||
}
|
||||
trimmed := strings.TrimSpace(value)
|
||||
if trimmed == "" {
|
||||
return "", validatorErrorf("validator explanation must not be blank")
|
||||
}
|
||||
if utf8.RuneCountInString(trimmed) > maximumExplanationRunes {
|
||||
return "", validatorErrorf("validator explanation exceeds %d Unicode code points", maximumExplanationRunes)
|
||||
}
|
||||
return trimmed, nil
|
||||
func unavailable(cause error) error {
|
||||
return &contracts.ValidationUnavailableError{Cause: validatorErrorf("native assessment unavailable: %w", cause)}
|
||||
}
|
||||
|
||||
func Spec() pipeline.ValidatorSpec {
|
||||
return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassLLMBacked}
|
||||
return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{PromptID}}
|
||||
}
|
||||
|
||||
func Register(registry *pipeline.ValidatorRegistry) error {
|
||||
@@ -209,7 +129,7 @@ func Register(registry *pipeline.ValidatorRegistry) error {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return New(request.Dependencies.LLM, options)
|
||||
return New(request.Dependencies.Engine, options)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -221,7 +141,3 @@ func DecodeOptions(options map[string]any) (Options, error) {
|
||||
}
|
||||
|
||||
func validateOptions(options map[string]any) error { _, err := DecodeOptions(options); return err }
|
||||
|
||||
func validatorErrorf(format string, args ...any) error {
|
||||
return fmt.Errorf("scene combat-semantics validator: "+format, args...)
|
||||
}
|
||||
|
||||
@@ -4,75 +4,31 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/assets"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestValidatorInterpretsCombatSemanticsResponses(t *testing.T) {
|
||||
request := validRequest(dnd.SceneKindNarrative)
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
kind dnd.SceneKind
|
||||
response completionResponse
|
||||
approved bool
|
||||
reasonCode string
|
||||
guidance string
|
||||
correction string
|
||||
wantError string
|
||||
}{
|
||||
{name: "approve non-combat", kind: dnd.SceneKindNarrative, response: completionResponse{Classification: "non_combat", Explanation: "No active encounter occurs."}, approved: true},
|
||||
{name: "approve combat", kind: dnd.SceneKindCombat, response: completionResponse{Classification: "combat", Explanation: "Initiative and attacks organize the chunk."}, approved: true},
|
||||
{name: "add combat", kind: dnd.SceneKindNarrative, response: completionResponse{Classification: "combat", Explanation: "The combatants exchange attacks and damage."}, reasonCode: ReasonCodeActiveCombatNotClassified, guidance: "Return kind: combat"},
|
||||
{name: "remove combat", kind: dnd.SceneKindCombat, response: completionResponse{Classification: "non_combat", Explanation: "The group only plans for a possible fight."}, reasonCode: ReasonCodeCombatClassificationUnsupported, guidance: "narrative, recap, or meta"},
|
||||
{name: "unknown classification", kind: dnd.SceneKindNarrative, response: completionResponse{Classification: "uncertain", Explanation: "The group only plans."}, wantError: "unsupported"},
|
||||
{name: "blank explanation", kind: dnd.SceneKindNarrative, response: completionResponse{Classification: "non_combat", Explanation: " "}, wantError: "blank"},
|
||||
{name: "trim explanation", kind: dnd.SceneKindNarrative, response: completionResponse{Classification: "combat", Explanation: " The combatants exchange attacks.\n"}, reasonCode: ReasonCodeActiveCombatNotClassified, correction: "Return kind: combat for this scene. The combatants exchange attacks."},
|
||||
{name: "oversized explanation", kind: dnd.SceneKindNarrative, response: completionResponse{Classification: "non_combat", Explanation: strings.Repeat("界", 513)}, wantError: "exceeds"},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
request.Value.Scenes[0].Kind = test.kind
|
||||
client := &fakeCombatSemanticsClient{response: test.response}
|
||||
validator := newValidator(t, client)
|
||||
result, err := validator.Validate(context.Background(), request)
|
||||
if test.wantError != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), test.wantError) {
|
||||
t.Fatalf("Validate() error = %v, want %q", err, test.wantError)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil || result.Approved != test.approved || result.ReasonCode != test.reasonCode || !strings.Contains(result.CorrectionGuidance, test.guidance) {
|
||||
t.Fatalf("Validate() = %#v, %v", result, err)
|
||||
}
|
||||
if test.correction != "" && result.CorrectionGuidance != test.correction {
|
||||
t.Fatalf("CorrectionGuidance = %q, want %q", result.CorrectionGuidance, test.correction)
|
||||
}
|
||||
if !result.Approved && (strings.Contains(result.CorrectionGuidance, Key) || strings.Contains(result.CorrectionGuidance, result.ReasonCode) || strings.Contains(result.CorrectionGuidance, request.Chunk.ID)) {
|
||||
t.Fatalf("correction guidance leaked opaque application detail: %q", result.CorrectionGuidance)
|
||||
}
|
||||
if err := contracts.ValidateValidationResult(result); err != nil {
|
||||
t.Fatalf("ValidateValidationResult(%#v) = %v", result, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidatorUsesOnlyChunkTranscript(t *testing.T) {
|
||||
request := validRequest(dnd.SceneKindNarrative)
|
||||
repairAttempts := 2
|
||||
request.LLMProfile = "validator-profile"
|
||||
request.SessionID = "validator-session"
|
||||
request.StructuredOutputRepairAttempts = &repairAttempts
|
||||
request.References = contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{"players": {Items: []contracts.ReferenceItem{{Content: []byte("hidden reference")}}}}}
|
||||
client := &fakeCombatSemanticsClient{response: completionResponse{Classification: "non_combat", Explanation: "No active encounter occurs."}}
|
||||
client := &fakeCombatSemanticsClient{response: classificationResponse{Classification: "non_combat", Explanation: "No active encounter occurs."}}
|
||||
validator := newValidator(t, client)
|
||||
model := "validator-override"
|
||||
request.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: model}
|
||||
var prepared promptkit.PreparedRun
|
||||
preparedCalls := 0
|
||||
request.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = value; preparedCalls++ }
|
||||
if result, err := validator.Validate(context.Background(), request); err != nil || !result.Approved {
|
||||
t.Fatalf("Validate() = %#v, %v", result, err)
|
||||
}
|
||||
@@ -80,19 +36,22 @@ func TestValidatorUsesOnlyChunkTranscript(t *testing.T) {
|
||||
t.Fatalf("completion count = %d, want 1", len(client.requests))
|
||||
}
|
||||
completed := client.requests[0]
|
||||
if completed.StageName != Key || completed.PromptID != PromptID || completed.PromptVersion != SchemaVersion || completed.ProfileID != request.LLMProfile || completed.SessionID != request.SessionID || completed.StructuredOutputRepairAttempts == request.StructuredOutputRepairAttempts || *completed.StructuredOutputRepairAttempts != repairAttempts {
|
||||
t.Fatalf("completion request = %#v", completed)
|
||||
if preparedCalls != 1 || prepared.PromptID != PromptID || prepared.SelectedProfileID != request.LLMProfile || completed.Prompt.SessionID != request.SessionID || completed.Target.Model != model {
|
||||
t.Fatal("native execution selection/override missing")
|
||||
}
|
||||
if completed.Correction != nil || !reflect.DeepEqual(sortedInputNames(completed.Inputs), []string{"transcript"}) || !reflect.DeepEqual(completed.Inputs["transcript"].Content, request.Chunk.Content) {
|
||||
t.Fatalf("completion inputs = %#v", completed.Inputs)
|
||||
var messages string
|
||||
for _, message := range completed.Prompt.Messages {
|
||||
messages += message.Content
|
||||
}
|
||||
if strings.Count(messages, string(request.Chunk.Content)) != 1 {
|
||||
t.Fatal("chunk transcript not rendered exactly once")
|
||||
}
|
||||
for _, forbidden := range []string{request.Value.Scenes[0].ID, request.Value.Scenes[0].Title, request.Value.Scenes[0].Summary, "hidden reference"} {
|
||||
for name, input := range completed.Inputs {
|
||||
if strings.Contains(string(input.Content), forbidden) {
|
||||
t.Fatalf("input %q leaked forbidden model-visible value %q", name, forbidden)
|
||||
}
|
||||
if strings.Contains(messages, forbidden) {
|
||||
t.Fatalf("model-visible context leaked %q", forbidden)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestValidatorRejectsInvalidRequestsAndCompletionFailures(t *testing.T) {
|
||||
@@ -130,7 +89,7 @@ func TestValidatorRejectsInvalidRequestsAndCompletionFailures(t *testing.T) {
|
||||
if test.mutate != nil {
|
||||
test.mutate(&request)
|
||||
}
|
||||
client := &fakeCombatSemanticsClient{response: completionResponse{Classification: "non_combat", Explanation: "No active encounter occurs."}}
|
||||
client := &fakeCombatSemanticsClient{response: classificationResponse{Classification: "non_combat", Explanation: "No active encounter occurs."}}
|
||||
_, err := newValidator(t, client).Validate(test.ctx, request)
|
||||
if err == nil || !strings.Contains(err.Error(), test.wantErr) || len(client.requests) != 0 {
|
||||
t.Fatalf("Validate() error = %v, calls = %d; want %q and no calls", err, len(client.requests), test.wantErr)
|
||||
@@ -138,29 +97,26 @@ func TestValidatorRejectsInvalidRequestsAndCompletionFailures(t *testing.T) {
|
||||
})
|
||||
}
|
||||
client := &fakeCombatSemanticsClient{err: errors.New("provider unavailable")}
|
||||
if _, err := newValidator(t, client).Validate(context.Background(), base); err == nil || !strings.Contains(err.Error(), "complete structured output") {
|
||||
if _, err := newValidator(t, client).Validate(context.Background(), base); err == nil || !strings.Contains(err.Error(), "assessment unavailable") {
|
||||
t.Fatalf("completion failure error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidatorConstructionOptionsAndMetadata(t *testing.T) {
|
||||
if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "LLM client") {
|
||||
func TestValidatorConstructionOptionsAndDependencies(t *testing.T) {
|
||||
if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "native engine") {
|
||||
t.Fatalf("New(nil) error = %v", err)
|
||||
}
|
||||
if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil {
|
||||
t.Fatal("DecodeOptions() accepted unknown options")
|
||||
}
|
||||
validator := newValidator(t, &fakeCombatSemanticsClient{})
|
||||
if validator.Name() != Key || validator.ExecutionClass() != contracts.ExecutionClassLLMBacked || Spec().CorrectionProtocol != "" {
|
||||
if validator.Name() != Key || validator.ExecutionClass() != contracts.ExecutionClassLLMBacked || !reflect.DeepEqual(Spec().Prompts, []string{PromptID}) {
|
||||
t.Fatalf("validator contract = %#v / %#v", validator, Spec())
|
||||
}
|
||||
metadata, err := json.Marshal(validator.ManifestMetadata())
|
||||
if err != nil || !strings.Contains(string(metadata), ResponseSchemaID) || strings.Contains(string(metadata), "non_combat") {
|
||||
t.Fatalf("manifest metadata = %s, %v", metadata, err)
|
||||
}
|
||||
fingerprints := validator.CheckpointFingerprints()
|
||||
if len(fingerprints) != 3 || fingerprints[0].Name != "prompt" || fingerprints[1].Name != "response_schema" || fingerprints[2] != (pipeline.CheckpointFingerprint{Name: "combat_policy", Value: combatPolicy}) {
|
||||
t.Fatalf("fingerprints = %#v", fingerprints)
|
||||
selected := Spec()
|
||||
selected.Prompts[0] = "mutated"
|
||||
if !reflect.DeepEqual(Spec().Prompts, []string{PromptID}) {
|
||||
t.Fatal("prompt dependencies alias caller memory")
|
||||
}
|
||||
registry := pipeline.NewValidatorRegistry()
|
||||
if err := Register(registry); err != nil {
|
||||
@@ -171,60 +127,36 @@ func TestValidatorConstructionOptionsAndMetadata(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluationCasesAreStrictAndCoverClassifications(t *testing.T) {
|
||||
content, err := os.ReadFile("testdata/evaluation_cases.json")
|
||||
func newValidator(t *testing.T, client promptkit.LLMClient) *Validator {
|
||||
t.Helper()
|
||||
type settings struct {
|
||||
PromptKit promptkit.Settings `yaml:"promptkit"`
|
||||
}
|
||||
definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit })
|
||||
filesystem := configkit.NewFilesystem("Notarius assets", assets.FS())
|
||||
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("validator fixture", []byte(`promptkit:
|
||||
backends:
|
||||
- id: validator-backend
|
||||
endpoint: https://example.invalid/v1
|
||||
concurrency_limit: 1
|
||||
queue_capacity: 0
|
||||
profiles:
|
||||
- id: validator-profile
|
||||
backend: validator-backend
|
||||
model: validator-test-model
|
||||
api_key_required: false
|
||||
`))}, configkit.LayeredOptions[settings]{Definition: definition, Underlays: []configkit.Underlay{
|
||||
promptkit.BundledOpenRouterUnderlay(),
|
||||
{Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var cases []struct {
|
||||
Name string `json:"name"`
|
||||
TranscriptUnits []struct {
|
||||
ID int `json:"id"`
|
||||
Text string `json:"text"`
|
||||
} `json:"transcript_units"`
|
||||
ProposedKind string `json:"proposed_kind"`
|
||||
ExpectedClassification string `json:"expected_classification"`
|
||||
ReviewerRationale string `json:"reviewer_rationale"`
|
||||
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
decoder := json.NewDecoder(strings.NewReader(string(content)))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(&cases); err != nil {
|
||||
t.Fatalf("decode evaluation cases: %v", err)
|
||||
}
|
||||
if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) {
|
||||
t.Fatalf("decode trailing evaluation data: %v, want EOF", err)
|
||||
}
|
||||
seenNames, seenClassifications := map[string]bool{}, map[string]bool{}
|
||||
for _, value := range cases {
|
||||
if strings.TrimSpace(value.Name) == "" || seenNames[value.Name] || len(value.TranscriptUnits) == 0 || strings.TrimSpace(value.ProposedKind) == "" || strings.TrimSpace(value.ReviewerRationale) == "" {
|
||||
t.Fatalf("invalid evaluation case: %#v", value)
|
||||
}
|
||||
seenNames[value.Name] = true
|
||||
switch value.ProposedKind {
|
||||
case "combat", "narrative", "recap", "meta":
|
||||
default:
|
||||
t.Fatalf("unsupported proposed kind %q", value.ProposedKind)
|
||||
}
|
||||
switch value.ExpectedClassification {
|
||||
case "combat", "non_combat":
|
||||
seenClassifications[value.ExpectedClassification] = true
|
||||
default:
|
||||
t.Fatalf("unsupported classification %q", value.ExpectedClassification)
|
||||
}
|
||||
for _, unit := range value.TranscriptUnits {
|
||||
if unit.ID <= 0 || strings.TrimSpace(unit.Text) == "" {
|
||||
t.Fatalf("invalid transcript unit: %#v", unit)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(seenClassifications) != 2 {
|
||||
t.Fatalf("evaluation classifications = %#v, want both classes", seenClassifications)
|
||||
}
|
||||
}
|
||||
|
||||
func newValidator(t *testing.T, client contracts.StructuredLLMClient) *Validator {
|
||||
t.Helper()
|
||||
validator, err := New(client, Options{})
|
||||
validator, err := New(engine, Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
@@ -234,7 +166,7 @@ func newValidator(t *testing.T, client contracts.StructuredLLMClient) *Validator
|
||||
func validRequest(kind dnd.SceneKind) contracts.TypedValidationRequest[dnd.SceneDescriptionList] {
|
||||
document := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{{ID: 1, Text: "The foes attack."}, {ID: 2, Text: "The party responds."}}}
|
||||
chunk := &source.Chunk{ID: "session:chunk:0", SourceID: document.ID, Ref: source.SourceRef{SourceID: document.ID, StartUnitID: 1, EndUnitID: 2}, Content: []byte(`{"units":[1,2]}`), MediaType: "application/json", Units: append([]source.SourceUnit(nil), document.Units...)}
|
||||
return contracts.TypedValidationRequest[dnd.SceneDescriptionList]{Stage: string(pipeline.StageExtract), Source: document, Chunk: chunk, SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session.json"), Value: dnd.SceneDescriptionList{Scenes: []dnd.SceneDescription{{ID: chunk.ID, SourceRef: chunk.Ref, Kind: kind, Title: "A Scene", Summary: "A valid scene."}}}}
|
||||
return contracts.TypedValidationRequest[dnd.SceneDescriptionList]{Stage: string(pipeline.StageExtract), LLMProfile: "validator-profile", Source: document, Chunk: chunk, SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session.json"), Value: dnd.SceneDescriptionList{Scenes: []dnd.SceneDescription{{ID: chunk.ID, SourceRef: chunk.Ref, Kind: kind, Title: "A Scene", Summary: "A valid scene."}}}}
|
||||
}
|
||||
|
||||
func cloneValidationRequest(request contracts.TypedValidationRequest[dnd.SceneDescriptionList]) contracts.TypedValidationRequest[dnd.SceneDescriptionList] {
|
||||
@@ -247,40 +179,28 @@ func cloneValidationRequest(request contracts.TypedValidationRequest[dnd.SceneDe
|
||||
return request
|
||||
}
|
||||
|
||||
func sortedInputNames(inputs contracts.LLMInputSet) []string {
|
||||
names := make([]string, 0, len(inputs))
|
||||
for name := range inputs {
|
||||
names = append(names, name)
|
||||
}
|
||||
if len(names) == 2 && names[0] == "transcript" {
|
||||
names[0], names[1] = names[1], names[0]
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
type fakeCombatSemanticsClient struct {
|
||||
response completionResponse
|
||||
err error
|
||||
requests []contracts.StructuredCompletionRequest
|
||||
response classificationResponse
|
||||
raw string
|
||||
err error
|
||||
requests []promptkit.GenerateRequest
|
||||
onGenerate func(context.Context, promptkit.GenerateRequest)
|
||||
}
|
||||
|
||||
func (client *fakeCombatSemanticsClient) CompleteStructured(_ context.Context, request contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
cloned, err := contracts.CloneStructuredCompletionRequest(request)
|
||||
if err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
func (client *fakeCombatSemanticsClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
client.requests = append(client.requests, request)
|
||||
if client.onGenerate != nil {
|
||||
client.onGenerate(ctx, request)
|
||||
}
|
||||
client.requests = append(client.requests, cloned)
|
||||
if client.err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, client.err
|
||||
return nil, client.err
|
||||
}
|
||||
response, ok := out.(*completionResponse)
|
||||
if !ok {
|
||||
return contracts.StructuredCompletionResponse{}, errors.New("unexpected completion target")
|
||||
if client.raw != "" {
|
||||
return &promptkit.GenerateResponse{Content: client.raw}, nil
|
||||
}
|
||||
*response = client.response
|
||||
content, err := json.Marshal(client.response)
|
||||
if err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
return nil, err
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{Content: content}, nil
|
||||
return &promptkit.GenerateResponse{Content: string(content)}, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user