Migrate combat semantics validation to native execution and unavailable assessments
This commit is contained in:
102
assets/combat_semantics_test.go
Normal file
102
assets/combat_semantics_test.go
Normal file
@@ -0,0 +1,102 @@
|
||||
package assets_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"gitea.maximumdirect.net/eric/notarius/assets"
|
||||
"io/fs"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/promptkit"
|
||||
|
||||
"github.com/santhosh-tekuri/jsonschema/v6"
|
||||
)
|
||||
|
||||
func TestCombatSemanticsPromptKeepsItsSceneLocalPolicyAndInput(t *testing.T) {
|
||||
_, engine := loadNativeAssets(t)
|
||||
const transcript = `{"sentinel":"combat-semantics-transcript"}`
|
||||
handle, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "notarius/dnd.scene_descriptions.validate_combat", APIKey: "test-only",
|
||||
Inputs: map[string]promptkit.ArtifactRef{"transcript": promptkit.Inline(transcript)},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer handle.Discard()
|
||||
details := handle.Details()
|
||||
if details.SelectedProfileID != "notarius/dnd-extraction" || details.StructuredOutput == nil {
|
||||
t.Fatal("native validator defaults/output contract lost")
|
||||
}
|
||||
var all string
|
||||
for _, message := range details.Messages {
|
||||
all += message.Content
|
||||
if strings.Contains(message.Content, transcript) && (message.CacheControl == nil || message.CacheControl.Type != promptkit.CacheControlEphemeral) {
|
||||
t.Fatal("chunk transcript native cache hint lost")
|
||||
}
|
||||
}
|
||||
for _, content := range []string{"substantive active combat materially organizes the", "Do not judge the scene title", transcript} {
|
||||
if strings.Count(all, content) != 1 {
|
||||
t.Fatalf("scene-local context not rendered once: %q", content)
|
||||
}
|
||||
}
|
||||
if _, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "notarius/dnd.scene_descriptions.validate_combat", APIKey: "test-only"}); err == nil {
|
||||
t.Fatal("native validator accepted missing transcript")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCombatSemanticsResponseSchemaRetainsClassificationAndExplanationRules(t *testing.T) {
|
||||
schema, err := fs.ReadFile(assets.FS(), "dnd/scene-descriptions/validate/combat-semantics/schemas/dnd_scene_combat_semantics_llm.v1.json")
|
||||
if err != nil {
|
||||
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
|
||||
}
|
||||
for _, classification := range []string{"combat", "non_combat"} {
|
||||
if err := validateCombatSemanticsSchema(map[string]any{"classification": classification, "explanation": "The chunk contains sustained attack exchanges."}, schema); err != nil {
|
||||
t.Fatalf("valid %q classification rejected: %v", classification, err)
|
||||
}
|
||||
}
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
response map[string]any
|
||||
}{
|
||||
{name: "missing explanation", response: map[string]any{"classification": "combat"}},
|
||||
{name: "missing classification", response: map[string]any{"explanation": "The chunk contains active combat."}},
|
||||
{name: "unknown property", response: map[string]any{"classification": "combat", "explanation": "The chunk contains active combat.", "confidence": 1}},
|
||||
{name: "wrong classification type", response: map[string]any{"classification": 1, "explanation": "The chunk contains active combat."}},
|
||||
{name: "wrong explanation type", response: map[string]any{"classification": "combat", "explanation": 1}},
|
||||
{name: "unsupported classification", response: map[string]any{"classification": "uncertain", "explanation": "The classification is uncertain."}},
|
||||
{name: "blank explanation", response: map[string]any{"classification": "combat", "explanation": ""}},
|
||||
{name: "oversized explanation", response: map[string]any{"classification": "combat", "explanation": strings.Repeat("a", 513)}},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if err := validateCombatSemanticsSchema(test.response, schema); err == nil {
|
||||
t.Fatal("invalid private response accepted")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func validateCombatSemanticsSchema(instance map[string]any, schemaContent []byte) error {
|
||||
instanceContent, err := json.Marshal(instance)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
parsedInstance, err := jsonschema.UnmarshalJSON(bytes.NewReader(instanceContent))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
parsedSchema, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaContent))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
compiler := jsonschema.NewCompiler()
|
||||
if err := compiler.AddResource("schema.json", parsedSchema); err != nil {
|
||||
return err
|
||||
}
|
||||
compiled, err := compiler.Compile("schema.json")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return compiled.Validate(parsedInstance)
|
||||
}
|
||||
Reference in New Issue
Block a user