Make production registration asset-free and verify direct deterministic execution
This commit is contained in:
81
internal/modules/dnd/register/dependencies_test.go
Normal file
81
internal/modules/dnd/register/dependencies_test.go
Normal file
@@ -0,0 +1,81 @@
|
||||
package register
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/assets"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestRegisteredDependenciesUseNativeEmbeddedPrompts(t *testing.T) {
|
||||
registries := completeRegistries()
|
||||
if err := Register(registries); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
type document struct {
|
||||
PromptKit promptkit.Settings `yaml:"promptkit"`
|
||||
}
|
||||
filesystem := configkit.NewFilesystem("Notarius assets", assets.FS())
|
||||
definition := promptkit.Configure(configkit.Definition[document]{}, func(value *document) *promptkit.Settings { return &value.PromptKit })
|
||||
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.NoSource()}, configkit.LayeredOptions[document]{
|
||||
Definition: definition,
|
||||
Underlays: []configkit.Underlay{
|
||||
promptkit.BundledOpenRouterUnderlay(),
|
||||
{Inputs: []configkit.Input{{
|
||||
Source: filesystem.File("notarius.yml"), Imports: true,
|
||||
Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."},
|
||||
}}},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
checkPrompts := func(key string, class contracts.ExecutionClass, ids []string, expected int) {
|
||||
t.Helper()
|
||||
if class == contracts.ExecutionClassDeterministic {
|
||||
expected = 0
|
||||
}
|
||||
if len(ids) != expected {
|
||||
t.Fatalf("%s declares %v; want %d static prompt dependencies", key, ids, expected)
|
||||
}
|
||||
for _, id := range ids {
|
||||
if _, err := engine.InspectPrompt(context.Background(), id); err != nil {
|
||||
t.Fatalf("%s prompt %s: %v", key, id, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, key := range registries.Chunkers.RegisteredKeys() {
|
||||
spec, ok := registries.Chunkers.Spec(key)
|
||||
if !ok {
|
||||
t.Fatalf("missing chunker spec %s", key)
|
||||
}
|
||||
checkPrompts(key, spec.ExecutionClass, spec.Prompts, 2)
|
||||
}
|
||||
for _, key := range registries.Extractors.RegisteredKeys() {
|
||||
spec, ok := registries.Extractors.Spec(key)
|
||||
if !ok {
|
||||
t.Fatalf("missing extractor spec %s", key)
|
||||
}
|
||||
checkPrompts(key, spec.ExecutionClass, spec.Prompts, 2)
|
||||
}
|
||||
for _, key := range registries.Normalizers.RegisteredKeys() {
|
||||
for _, kind := range registries.Normalizers.RegisteredArtifactKinds(key) {
|
||||
spec, ok := registries.Normalizers.SpecForArtifact(key, kind)
|
||||
if !ok {
|
||||
t.Fatalf("missing normalizer spec %s/%s", key, kind)
|
||||
}
|
||||
checkPrompts(key, spec.ExecutionClass, spec.Prompts, 2)
|
||||
}
|
||||
}
|
||||
for _, spec := range registries.Validators.RegisteredSpecs() {
|
||||
checkPrompts(spec.Key, spec.ExecutionClass, spec.Prompts, 1)
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,6 @@
|
||||
package register
|
||||
|
||||
import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/chunk/scenes"
|
||||
@@ -35,7 +34,6 @@ import (
|
||||
npcnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/npcregistry"
|
||||
scenedescriptionnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/scenedescriptions"
|
||||
spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells"
|
||||
combatsemantics "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/combat_semantics"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop"
|
||||
)
|
||||
@@ -144,23 +142,3 @@ func registerModules(registries pipeline.Registries) error {
|
||||
}},
|
||||
})
|
||||
}
|
||||
|
||||
func registerPromptAssets(assets *llm.AssetRegistry) error {
|
||||
return runRegistrations([]registration{
|
||||
{name: "scenes prompt assets", register: func() error { return scenes.RegisterPromptAssets(assets) }},
|
||||
{name: "spells prompt assets", register: func() error { return spellextract.RegisterPromptAssets(assets) }},
|
||||
{name: "npc registry prompt assets", register: func() error { return npcextract.RegisterPromptAssets(assets) }},
|
||||
{name: "npc normalization prompt assets", register: func() error { return npcnormalize.RegisterPromptAssets(assets) }},
|
||||
{name: "combat turns prompt assets", register: func() error { return combatextract.RegisterPromptAssets(assets) }},
|
||||
{name: "enemy events prompt assets", register: func() error { return enemyeventextract.RegisterPromptAssets(assets) }},
|
||||
{name: "item occurrences prompt assets", register: func() error { return itemoccurrenceextract.RegisterPromptAssets(assets) }},
|
||||
{name: "item registry prompt assets", register: func() error { return itemregistryextract.RegisterPromptAssets(assets) }},
|
||||
{name: "item registry normalization prompt assets", register: func() error { return itemregistrynormalize.RegisterPromptAssets(assets) }},
|
||||
{name: "npc occurrences prompt assets", register: func() error { return occurrenceextract.RegisterPromptAssets(assets) }},
|
||||
{name: "scene descriptions prompt assets", register: func() error { return scenedescriptionextract.RegisterPromptAssets(assets) }},
|
||||
{name: "scene combat-semantics validator prompt assets", register: func() error { return combatsemantics.RegisterPromptAssets(assets) }},
|
||||
{name: "location registry prompt assets", register: func() error { return locationextract.RegisterPromptAssets(assets) }},
|
||||
{name: "location normalization prompt assets", register: func() error { return locationnormalize.RegisterPromptAssets(assets) }},
|
||||
{name: "location occurrences prompt assets", register: func() error { return locationoccurrenceextract.RegisterPromptAssets(assets) }},
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1,17 +0,0 @@
|
||||
package register
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
|
||||
rootassets "gitea.maximumdirect.net/eric/notarius/assets"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
)
|
||||
|
||||
func registerFallbackProfiles(assets *llm.AssetRegistry) error {
|
||||
profiles, err := fs.Sub(rootassets.FS(), "dnd/profiles")
|
||||
if err != nil {
|
||||
return fmt.Errorf("scope D&D fallback profiles: %w", err)
|
||||
}
|
||||
return assets.RegisterFallbackProfileFS(profiles, ".")
|
||||
}
|
||||
@@ -1,216 +0,0 @@
|
||||
package register
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
combatextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/combatturns"
|
||||
enemyeventextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/enemyevents"
|
||||
itemoccurrenceextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/itemoccurrences"
|
||||
itemregistryextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/itemregistry"
|
||||
locationoccurrenceextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/locationoccurrences"
|
||||
locationextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/locationregistry"
|
||||
occurrenceextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/npcoccurrences"
|
||||
npcextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/npcregistry"
|
||||
scenedescriptionextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/scenedescriptions"
|
||||
spellextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestExtractionPromptComposition(t *testing.T) {
|
||||
const (
|
||||
transcriptSentinel = "shared-transcript-sentinel"
|
||||
playersSentinel = "shared-players-sentinel"
|
||||
partySentinel = "shared-party-sentinel"
|
||||
glossarySentinel = "shared-glossary-sentinel"
|
||||
npcSentinel = "npc-registry-sentinel"
|
||||
catalogSentinel = "spell-catalog-sentinel"
|
||||
evidenceSentinel = "Transcript units are the only evidence"
|
||||
endpointOrderSentinel = "according to transcript order"
|
||||
combatPolicySentinel = "substantive active combat materially organizes the"
|
||||
)
|
||||
registry := llm.NewAssetRegistry()
|
||||
if err := registerPromptAssets(registry); err != nil {
|
||||
t.Fatalf("registerPromptAssets() error = %v", err)
|
||||
}
|
||||
engine := newPromptCacheEngine(t, registry)
|
||||
commonInputs := map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.InlineWithURI("file:///session.json", `{"sentinel":"`+transcriptSentinel+`"}`),
|
||||
"players": promptkit.Inline(playersSentinel),
|
||||
"party": promptkit.Inline(partySentinel),
|
||||
"glossary": promptkit.Inline(glossarySentinel),
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
promptID string
|
||||
promptVersion string
|
||||
inputs map[string]promptkit.ArtifactRef
|
||||
suffixGroups [][]string
|
||||
middleSentinel string
|
||||
}{
|
||||
{name: "npcs", promptID: npcextract.PromptID, promptVersion: npcextract.SchemaVersion, inputs: commonInputs, suffixGroups: [][]string{{evidenceSentinel, endpointOrderSentinel}}},
|
||||
{name: "locations", promptID: locationextract.PromptID, promptVersion: locationextract.SchemaVersion, inputs: commonInputs, suffixGroups: [][]string{{evidenceSentinel, endpointOrderSentinel}}},
|
||||
{name: "item occurrences", promptID: itemoccurrenceextract.PromptID, promptVersion: itemoccurrenceextract.SchemaVersion, inputs: withPromptInputs(commonInputs, map[string]promptkit.ArtifactRef{
|
||||
"item_registry": promptkit.Inline(`{"items":[{"id":"item-registry-sentinel","name":"Torch"}]}`),
|
||||
}), suffixGroups: [][]string{{evidenceSentinel, endpointOrderSentinel}, {"item-registry-sentinel"}}},
|
||||
{name: "item registry", promptID: itemregistryextract.PromptID, promptVersion: itemregistryextract.SchemaVersion, inputs: commonInputs, suffixGroups: [][]string{{evidenceSentinel, endpointOrderSentinel}}},
|
||||
{name: "scene descriptions", promptID: scenedescriptionextract.PromptID, promptVersion: scenedescriptionextract.SchemaVersion, inputs: commonInputs, middleSentinel: combatPolicySentinel},
|
||||
{
|
||||
name: "combat turns",
|
||||
promptID: combatextract.PromptID,
|
||||
promptVersion: combatextract.SchemaVersion,
|
||||
inputs: withPromptInputs(commonInputs, map[string]promptkit.ArtifactRef{
|
||||
"npc_registry": promptkit.Inline(`{"sentinel":"` + npcSentinel + `"}`),
|
||||
}),
|
||||
suffixGroups: [][]string{{evidenceSentinel, endpointOrderSentinel}, {npcSentinel}},
|
||||
},
|
||||
{
|
||||
name: "enemy events",
|
||||
promptID: enemyeventextract.PromptID,
|
||||
promptVersion: enemyeventextract.SchemaVersion,
|
||||
inputs: withPromptInputs(commonInputs, map[string]promptkit.ArtifactRef{
|
||||
"npc_registry": promptkit.Inline(`{"sentinel":"` + npcSentinel + `"}`),
|
||||
"combat_turns": promptkit.Inline(`{"sentinel":"combat-turns-sentinel"}`),
|
||||
"npc_occurrences": promptkit.Inline(`{"sentinel":"npc-occurrences-sentinel"}`),
|
||||
}),
|
||||
suffixGroups: [][]string{
|
||||
{evidenceSentinel, endpointOrderSentinel},
|
||||
{npcSentinel},
|
||||
{"combat-turns-sentinel", "npc-occurrences-sentinel"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "npc occurrences",
|
||||
promptID: occurrenceextract.PromptID,
|
||||
promptVersion: occurrenceextract.SchemaVersion,
|
||||
inputs: withPromptInputs(commonInputs, map[string]promptkit.ArtifactRef{
|
||||
"npc_registry": promptkit.Inline(`{"sentinel":"` + npcSentinel + `"}`),
|
||||
}),
|
||||
suffixGroups: [][]string{{evidenceSentinel, endpointOrderSentinel}, {npcSentinel}},
|
||||
},
|
||||
{
|
||||
name: "location occurrences",
|
||||
promptID: locationoccurrenceextract.PromptID,
|
||||
promptVersion: locationoccurrenceextract.SchemaVersion,
|
||||
inputs: withPromptInputs(commonInputs, map[string]promptkit.ArtifactRef{
|
||||
"location_registry": promptkit.Inline(`{"sentinel":"location-registry-sentinel"}`),
|
||||
}),
|
||||
suffixGroups: [][]string{{evidenceSentinel, endpointOrderSentinel}, {"location-registry-sentinel"}},
|
||||
},
|
||||
{
|
||||
name: "spells",
|
||||
promptID: spellextract.PromptID,
|
||||
promptVersion: spellextract.SchemaVersion,
|
||||
inputs: withPromptInputs(commonInputs, map[string]promptkit.ArtifactRef{
|
||||
"npc_registry": promptkit.Inline(`{"sentinel":"` + npcSentinel + `"}`),
|
||||
"spell_catalog": promptkit.Inline(`{"sentinel":"` + catalogSentinel + `"}`),
|
||||
}),
|
||||
suffixGroups: [][]string{{evidenceSentinel, endpointOrderSentinel}, {npcSentinel}, {catalogSentinel}},
|
||||
},
|
||||
}
|
||||
|
||||
var sharedPrefix []promptkit.RenderedMessage
|
||||
for _, testCase := range cases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: testCase.promptID,
|
||||
PromptVersion: testCase.promptVersion,
|
||||
ProfileID: "prompt-cache-test-profile",
|
||||
Inputs: testCase.inputs,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Prepare() error = %v", err)
|
||||
}
|
||||
transcriptIndex := renderedInputMessageIndex(t, prepared.Messages, transcriptSentinel)
|
||||
referenceIndex := renderedInputMessageIndex(t, prepared.Messages, playersSentinel)
|
||||
if referenceIndex >= transcriptIndex {
|
||||
t.Fatalf("campaign references rendered at message %d, want before transcript message %d", referenceIndex, transcriptIndex)
|
||||
}
|
||||
assertEphemeralCache(t, prepared.Messages, referenceIndex)
|
||||
assertEphemeralCache(t, prepared.Messages, transcriptIndex)
|
||||
prefix := prepared.Messages[:transcriptIndex+1]
|
||||
if len(prepared.Messages) <= len(prefix) {
|
||||
t.Fatalf("prepared prompt has %d messages, want lane-specific suffix after transcript", len(prepared.Messages))
|
||||
}
|
||||
if sharedPrefix == nil {
|
||||
sharedPrefix = append([]promptkit.RenderedMessage(nil), prefix...)
|
||||
} else if !reflect.DeepEqual(prefix, sharedPrefix) {
|
||||
t.Fatalf("rendered prefix = %#v, want %#v", prefix, sharedPrefix)
|
||||
}
|
||||
previousIndex := transcriptIndex
|
||||
for _, group := range testCase.suffixGroups {
|
||||
groupIndex := -1
|
||||
for _, sentinel := range group {
|
||||
inputIndex := renderedInputMessageIndex(t, prepared.Messages, sentinel)
|
||||
if groupIndex < 0 {
|
||||
groupIndex = inputIndex
|
||||
} else if inputIndex != groupIndex {
|
||||
t.Fatalf("input sentinel %q rendered at message %d, want grouped at message %d", sentinel, inputIndex, groupIndex)
|
||||
}
|
||||
}
|
||||
if groupIndex <= previousIndex {
|
||||
t.Fatalf("prompt suffix group rendered at message %d, want after message %d", groupIndex, previousIndex)
|
||||
}
|
||||
previousIndex = groupIndex
|
||||
}
|
||||
instructionIndex := len(prepared.Messages) - 1
|
||||
if testCase.middleSentinel != "" {
|
||||
policyIndex := renderedInputMessageIndex(t, prepared.Messages, testCase.middleSentinel)
|
||||
if policyIndex <= transcriptIndex || policyIndex >= instructionIndex {
|
||||
t.Fatalf("combat policy rendered at message %d, want after transcript message %d and before instructions message %d", policyIndex, transcriptIndex, instructionIndex)
|
||||
}
|
||||
}
|
||||
if instructionIndex <= previousIndex {
|
||||
t.Fatalf("instructions rendered at message %d, want after lane input message %d", instructionIndex, previousIndex)
|
||||
}
|
||||
assertEphemeralCache(t, prepared.Messages, instructionIndex)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func newPromptCacheEngine(t *testing.T, registry *llm.AssetRegistry) *promptkit.Engine {
|
||||
t.Helper()
|
||||
return promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
|
||||
"prompt-cache-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "prompt-cache-test-model"),
|
||||
})
|
||||
}
|
||||
|
||||
func withPromptInputs(inputs, extras map[string]promptkit.ArtifactRef) map[string]promptkit.ArtifactRef {
|
||||
merged := make(map[string]promptkit.ArtifactRef, len(inputs)+len(extras))
|
||||
for name, input := range inputs {
|
||||
merged[name] = input
|
||||
}
|
||||
for name, input := range extras {
|
||||
merged[name] = input
|
||||
}
|
||||
return merged
|
||||
}
|
||||
|
||||
func assertEphemeralCache(t *testing.T, messages []promptkit.RenderedMessage, index int) {
|
||||
t.Helper()
|
||||
if cache := messages[index].CacheControl; cache == nil || cache.Type != promptkit.CacheControlEphemeral {
|
||||
t.Fatalf("message %d cache control = %#v, want ephemeral", index, cache)
|
||||
}
|
||||
}
|
||||
|
||||
func renderedInputMessageIndex(t *testing.T, messages []promptkit.RenderedMessage, sentinel string) int {
|
||||
t.Helper()
|
||||
index := -1
|
||||
occurrences := 0
|
||||
for messageIndex, message := range messages {
|
||||
count := strings.Count(message.Content, sentinel)
|
||||
if count > 0 {
|
||||
index = messageIndex
|
||||
occurrences += count
|
||||
}
|
||||
}
|
||||
if occurrences != 1 {
|
||||
t.Fatalf("input sentinel %q rendered %d times, want exactly once", sentinel, occurrences)
|
||||
}
|
||||
return index
|
||||
}
|
||||
@@ -4,7 +4,6 @@ package register
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
@@ -13,9 +12,9 @@ type registration struct {
|
||||
register func() error
|
||||
}
|
||||
|
||||
// Register adds all production D&D modules, validators, policy, and assets.
|
||||
func Register(registries pipeline.Registries, assets *llm.AssetRegistry) error {
|
||||
if err := validateRegistries(registries, assets); err != nil {
|
||||
// Register adds all production D&D modules, validators, and policy.
|
||||
func Register(registries pipeline.Registries) error {
|
||||
if err := validateRegistries(registries); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := registerModules(registries); err != nil {
|
||||
@@ -27,12 +26,6 @@ func Register(registries pipeline.Registries, assets *llm.AssetRegistry) error {
|
||||
if err := registerValidators(registries); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := registerPromptAssets(assets); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := registerFallbackProfiles(assets); err != nil {
|
||||
return err
|
||||
}
|
||||
return registerDefaultChains(registries.ValidatorChains)
|
||||
}
|
||||
|
||||
@@ -45,7 +38,7 @@ func runRegistrations(registrations []registration) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateRegistries(registries pipeline.Registries, assets *llm.AssetRegistry) error {
|
||||
func validateRegistries(registries pipeline.Registries) error {
|
||||
switch {
|
||||
case registries.Chunkers == nil:
|
||||
return fmt.Errorf("dnd registrar: chunker registry must not be nil")
|
||||
@@ -63,8 +56,6 @@ func validateRegistries(registries pipeline.Registries, assets *llm.AssetRegistr
|
||||
return fmt.Errorf("dnd registrar: validator registry must not be nil")
|
||||
case registries.ValidatorChains == nil:
|
||||
return fmt.Errorf("dnd registrar: validator chain registry must not be nil")
|
||||
case assets == nil:
|
||||
return fmt.Errorf("dnd registrar: asset registry must not be nil")
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,14 +1,12 @@
|
||||
package register
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
combatextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/combatturns"
|
||||
@@ -36,49 +34,9 @@ import (
|
||||
|
||||
func TestRegisterAddsDNDFamily(t *testing.T) {
|
||||
registries := completeRegistries()
|
||||
assets := llm.NewAssetRegistry()
|
||||
if err := Register(registries, assets); err != nil {
|
||||
if err := Register(registries); err != nil {
|
||||
t.Fatalf("Register() error = %v, want nil", err)
|
||||
}
|
||||
promptFS, err := assets.PromptFS()
|
||||
if err != nil {
|
||||
t.Fatalf("PromptFS() error = %v", err)
|
||||
}
|
||||
if _, err := fs.ReadFile(promptFS, "dnd.npc_registry.normalize/prompt.yaml"); err != nil {
|
||||
t.Fatalf("normalization prompt asset = %v, want registered private prompt", err)
|
||||
}
|
||||
for _, name := range []string{
|
||||
"dnd.scenes/prompt.yaml",
|
||||
"dnd.spells/prompt.yaml",
|
||||
"dnd.npc_registry/prompt.yaml",
|
||||
"dnd.combat_turns/prompt.yaml",
|
||||
"dnd.enemy_events/prompt.yaml",
|
||||
"dnd.item_occurrences/prompt.yaml",
|
||||
"dnd.item_registry/prompt.yaml",
|
||||
"dnd.item_registry.normalize/prompt.yaml",
|
||||
"dnd.npc_occurrences/prompt.yaml",
|
||||
"dnd.scene_descriptions/prompt.yaml",
|
||||
"dnd.scene_descriptions.validate_combat/prompt.yaml",
|
||||
"dnd.npc_registry.normalize/prompt.yaml",
|
||||
"dnd.location_registry/prompt.yaml",
|
||||
"dnd.location_registry.normalize/prompt.yaml",
|
||||
"dnd.location_occurrences/prompt.yaml",
|
||||
} {
|
||||
content, err := fs.ReadFile(promptFS, name)
|
||||
if err != nil {
|
||||
t.Fatalf("read prompt asset %q: %v", name, err)
|
||||
}
|
||||
if !strings.Contains(string(content), "default_profile: dnd-extraction") {
|
||||
t.Fatalf("prompt asset %q does not select dnd-extraction", name)
|
||||
}
|
||||
}
|
||||
fallbackFS, err := assets.FallbackProfileFS()
|
||||
if err != nil {
|
||||
t.Fatalf("FallbackProfileFS() error = %v", err)
|
||||
}
|
||||
if _, err := fs.ReadFile(fallbackFS, "dnd-extraction.yaml"); err != nil {
|
||||
t.Fatalf("fallback profile asset = %v, want registered D&D profile", err)
|
||||
}
|
||||
assertContainsKeys(t, "chunkers", registries.Chunkers.RegisteredKeys(), []string{"dnd/scenes"})
|
||||
assertContainsKeys(t, "extractors", registries.Extractors.RegisteredKeys(), []string{"dnd/spells", npcextract.Key, combatextract.Key, enemyeventextract.Key, itemoccurrenceextract.Key, itemregistryextract.Key, occurrenceextract.Key, scenedescriptionextract.Key, locationextract.Key, locationoccurrenceextract.Key})
|
||||
assertContainsKeys(t, "normalizers", registries.Normalizers.RegisteredKeys(), []string{spellnormalize.Key, npcnormalize.Key, combatnormalize.Key, enemyeventnormalize.Key, itemoccurrencenormalize.Key, itemregistrynormalize.Key, occurrencenormalize.Key, scenedescriptionnormalize.Key, locationnormalize.Key, locationoccurrencenormalize.Key, pipeline.DefaultNormalizeModule})
|
||||
@@ -767,25 +725,23 @@ func TestAppendListsPreserveNestedSourceReferencePresence(t *testing.T) {
|
||||
func TestRegisterRejectsMissingDNDDependenciesBeforeMutation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
remove func(*pipeline.Registries, **llm.AssetRegistry)
|
||||
remove func(*pipeline.Registries)
|
||||
wantErr string
|
||||
}{
|
||||
{name: "chunkers", remove: func(r *pipeline.Registries, _ **llm.AssetRegistry) { r.Chunkers = nil }, wantErr: "chunker registry"},
|
||||
{name: "artifact codecs", remove: func(r *pipeline.Registries, _ **llm.AssetRegistry) { r.ArtifactCodecs = nil }, wantErr: "artifact codec registry"},
|
||||
{name: "artifact evidence", remove: func(r *pipeline.Registries, _ **llm.AssetRegistry) { r.ArtifactEvidence = nil }, wantErr: "artifact evidence registry"},
|
||||
{name: "extractors", remove: func(r *pipeline.Registries, _ **llm.AssetRegistry) { r.Extractors = nil }, wantErr: "extractor registry"},
|
||||
{name: "mergers", remove: func(r *pipeline.Registries, _ **llm.AssetRegistry) { r.Mergers = nil }, wantErr: "merger registry"},
|
||||
{name: "normalizers", remove: func(r *pipeline.Registries, _ **llm.AssetRegistry) { r.Normalizers = nil }, wantErr: "normalizer registry"},
|
||||
{name: "validators", remove: func(r *pipeline.Registries, _ **llm.AssetRegistry) { r.Validators = nil }, wantErr: "validator registry"},
|
||||
{name: "validator chains", remove: func(r *pipeline.Registries, _ **llm.AssetRegistry) { r.ValidatorChains = nil }, wantErr: "validator chain registry"},
|
||||
{name: "assets", remove: func(_ *pipeline.Registries, assets **llm.AssetRegistry) { *assets = nil }, wantErr: "asset registry"},
|
||||
{name: "chunkers", remove: func(r *pipeline.Registries) { r.Chunkers = nil }, wantErr: "chunker registry"},
|
||||
{name: "artifact codecs", remove: func(r *pipeline.Registries) { r.ArtifactCodecs = nil }, wantErr: "artifact codec registry"},
|
||||
{name: "artifact evidence", remove: func(r *pipeline.Registries) { r.ArtifactEvidence = nil }, wantErr: "artifact evidence registry"},
|
||||
{name: "extractors", remove: func(r *pipeline.Registries) { r.Extractors = nil }, wantErr: "extractor registry"},
|
||||
{name: "mergers", remove: func(r *pipeline.Registries) { r.Mergers = nil }, wantErr: "merger registry"},
|
||||
{name: "normalizers", remove: func(r *pipeline.Registries) { r.Normalizers = nil }, wantErr: "normalizer registry"},
|
||||
{name: "validators", remove: func(r *pipeline.Registries) { r.Validators = nil }, wantErr: "validator registry"},
|
||||
{name: "validator chains", remove: func(r *pipeline.Registries) { r.ValidatorChains = nil }, wantErr: "validator chain registry"},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
registries := completeRegistries()
|
||||
assets := llm.NewAssetRegistry()
|
||||
test.remove(®istries, &assets)
|
||||
err := Register(registries, assets)
|
||||
test.remove(®istries)
|
||||
err := Register(registries)
|
||||
if err == nil || !strings.Contains(err.Error(), test.wantErr) {
|
||||
t.Fatalf("Register() error = %v, want %q", err, test.wantErr)
|
||||
}
|
||||
@@ -798,11 +754,10 @@ func TestRegisterRejectsMissingDNDDependenciesBeforeMutation(t *testing.T) {
|
||||
|
||||
func TestRegisterReportsDuplicateDNDRegistration(t *testing.T) {
|
||||
registries := completeRegistries()
|
||||
assets := llm.NewAssetRegistry()
|
||||
if err := Register(registries, assets); err != nil {
|
||||
if err := Register(registries); err != nil {
|
||||
t.Fatalf("first Register() error = %v, want nil", err)
|
||||
}
|
||||
err := Register(registries, assets)
|
||||
err := Register(registries)
|
||||
if err == nil || !strings.Contains(err.Error(), "register dnd spells codec") || !strings.Contains(err.Error(), "already registered") {
|
||||
t.Fatalf("second Register() error = %v, want contextual duplicate error", err)
|
||||
}
|
||||
|
||||
@@ -1,85 +0,0 @@
|
||||
package register
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io/fs"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
)
|
||||
|
||||
func TestRegisteredResponseSchemasRequireEveryObjectProperty(t *testing.T) {
|
||||
assets := llm.NewAssetRegistry()
|
||||
if err := Register(completeRegistries(), assets); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
schemaFS, err := assets.SchemaFS()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
count := 0
|
||||
err = fs.WalkDir(schemaFS, ".", func(path string, entry fs.DirEntry, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
if entry.IsDir() || !strings.HasSuffix(path, ".json") {
|
||||
return nil
|
||||
}
|
||||
content, err := fs.ReadFile(schemaFS, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var schema any
|
||||
if err := json.Unmarshal(content, &schema); err != nil {
|
||||
t.Errorf("decode response schema %q: %v", path, err)
|
||||
return nil
|
||||
}
|
||||
count++
|
||||
assertRequiredObjectProperties(t, path, "$", schema)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count == 0 {
|
||||
t.Fatal("no registered response schemas found")
|
||||
}
|
||||
}
|
||||
|
||||
func assertRequiredObjectProperties(t *testing.T, schemaPath, nodePath string, value any) {
|
||||
t.Helper()
|
||||
switch node := value.(type) {
|
||||
case map[string]any:
|
||||
if properties, ok := node["properties"].(map[string]any); ok {
|
||||
required := make(map[string]struct{})
|
||||
if values, ok := node["required"].([]any); ok {
|
||||
for _, value := range values {
|
||||
if name, ok := value.(string); ok {
|
||||
required[name] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
propertyNames := make([]string, 0, len(properties))
|
||||
for name := range properties {
|
||||
propertyNames = append(propertyNames, name)
|
||||
}
|
||||
sort.Strings(propertyNames)
|
||||
for _, name := range propertyNames {
|
||||
if _, ok := required[name]; !ok {
|
||||
t.Errorf("response schema %q object %s declares property %q without requiring it", schemaPath, nodePath, name)
|
||||
}
|
||||
}
|
||||
}
|
||||
for name, child := range node {
|
||||
assertRequiredObjectProperties(t, schemaPath, nodePath+"."+name, child)
|
||||
}
|
||||
case []any:
|
||||
for index, child := range node {
|
||||
assertRequiredObjectProperties(t, schemaPath, nodePath+"["+strconv.Itoa(index)+"]", child)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user