929 lines
43 KiB
Go
929 lines
43 KiB
Go
package cli
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io/fs"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"reflect"
|
|
"runtime"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkmap"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/chunk/scenes"
|
|
combatcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/combatturns"
|
|
enemyeventcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/enemyevents"
|
|
itemoccurrencecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/itemoccurrences"
|
|
itemregistrycodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/itemregistry"
|
|
locationoccurrencecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/locationoccurrences"
|
|
locationcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/locationregistry"
|
|
npcoccurrencecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcoccurrences"
|
|
npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcregistry"
|
|
scenecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/scenedescriptions"
|
|
spellcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/spells"
|
|
combatextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/combatturns"
|
|
enemyeventextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/enemyevents"
|
|
itemoccurrenceextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/itemoccurrences"
|
|
itemregistryextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/itemregistry"
|
|
locationoccurrenceextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/locationoccurrences"
|
|
locationextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/locationregistry"
|
|
npcoccurrenceextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/npcoccurrences"
|
|
npcextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/npcregistry"
|
|
sceneextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/scenedescriptions"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
|
|
combatnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/combatturns"
|
|
enemyeventnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/enemyevents"
|
|
itemoccurrencenormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/itemoccurrences"
|
|
itemregistrynormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/itemregistry"
|
|
locationoccurrencenormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/locationoccurrences"
|
|
locationnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/locationregistry"
|
|
npcoccurrencenormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/npcoccurrences"
|
|
npcnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/npcregistry"
|
|
scenenormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/scenedescriptions"
|
|
spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop"
|
|
"go.maximumdirect.net/ckconfig"
|
|
"go.maximumdirect.net/commandkit"
|
|
"go.maximumdirect.net/promptkit"
|
|
)
|
|
|
|
func TestProductionCatalogCoversMaintainedConfigurations(t *testing.T) {
|
|
components := productionTestComponents(t)
|
|
registries := components.registries
|
|
|
|
assertProductionContains(t, "inputs", registries.Inputs.RegisteredKeys(), []string{"seriatim"})
|
|
assertProductionContains(t, "chunkers", registries.Chunkers.RegisteredKeys(), []string{"dnd/scenes", "generic"})
|
|
assertProductionContains(t, "extractors", registries.Extractors.RegisteredKeys(), []string{spells.Key, npcextract.Key, combatextract.Key, itemoccurrenceextract.Key, itemregistryextract.Key, npcoccurrenceextract.Key, sceneextract.Key, locationextract.Key, locationoccurrenceextract.Key, enemyeventextract.Key})
|
|
assertProductionContains(t, "mergers", registries.Mergers.RegisteredKeys(), []string{"appendorder"})
|
|
assertProductionContains(t, "normalizers", registries.Normalizers.RegisteredKeys(), []string{pipeline.DefaultNormalizeModule, spellnormalize.Key, npcnormalize.Key, combatnormalize.Key, itemoccurrencenormalize.Key, itemregistrynormalize.Key, npcoccurrencenormalize.Key, scenenormalize.Key, locationnormalize.Key, locationoccurrencenormalize.Key, enemyeventnormalize.Key})
|
|
assertProductionContains(t, "outputs", registries.Outputs.RegisteredKeys(), []string{"json"})
|
|
assertProductionContains(t, "validators", registries.Validators.RegisteredKeys(), []string{
|
|
"extract/dnd/spells/catalog",
|
|
"extract/dnd/spells/shape",
|
|
"extract/dnd/spells/source_refs",
|
|
"extract/dnd/spells/source_relatedness",
|
|
"extract/dnd/combat-turns/shape",
|
|
"extract/dnd/combat-turns/source_refs",
|
|
"extract/dnd/combat-turns/source_relatedness",
|
|
"normalize/dnd/combat-turns/invariants",
|
|
"extract/dnd/item-occurrences/shape",
|
|
"extract/dnd/item-occurrences/registry",
|
|
"extract/dnd/item-occurrences/source_refs",
|
|
"extract/dnd/item-occurrences/source_relatedness",
|
|
"normalize/dnd/item-occurrences/invariants",
|
|
"extract/dnd/item-registry/shape",
|
|
"extract/dnd/item-registry/source_refs",
|
|
"extract/dnd/item-registry/source_relatedness",
|
|
"normalize/dnd/item-registry/identity",
|
|
"extract/dnd/npc-registry/shape",
|
|
"extract/dnd/npc-registry/source_refs",
|
|
"extract/dnd/npc-registry/source_relatedness",
|
|
"normalize/dnd/npc-registry/identity",
|
|
"extract/dnd/npc-occurrences/shape",
|
|
"extract/dnd/npc-occurrences/registry",
|
|
"extract/dnd/npc-occurrences/source_refs",
|
|
"extract/dnd/npc-occurrences/source_relatedness",
|
|
"normalize/dnd/npc-occurrences/invariants",
|
|
"extract/dnd/scene-descriptions/shape",
|
|
"extract/dnd/scene-descriptions/source_refs",
|
|
"extract/dnd/scene-descriptions/source_relatedness",
|
|
"normalize/dnd/scene-descriptions/invariants",
|
|
"extract/dnd/location-registry/shape",
|
|
"extract/dnd/location-registry/source_refs",
|
|
"extract/dnd/location-registry/source_relatedness",
|
|
"normalize/dnd/location-registry/identity",
|
|
"extract/dnd/location-occurrences/shape",
|
|
"extract/dnd/location-occurrences/registry",
|
|
"extract/dnd/location-occurrences/source_refs",
|
|
"extract/dnd/location-occurrences/source_relatedness",
|
|
"normalize/dnd/location-occurrences/invariants",
|
|
"extract/dnd/enemy-events/shape",
|
|
"extract/dnd/enemy-events/engagements",
|
|
"extract/dnd/enemy-events/source_refs",
|
|
"extract/dnd/enemy-events/source_relatedness",
|
|
"normalize/dnd/enemy-events/invariants",
|
|
"generic/always_accept",
|
|
"generic/always_reject",
|
|
"generic/valid_json",
|
|
"generic/valid_json_schema",
|
|
})
|
|
allArtifactKinds := []contracts.ArtifactKind{dnd.SpellListKind, dnd.NPCRegistryKind, dnd.NPCOccurrenceListKind, dnd.CombatTurnListKind, dnd.SceneDescriptionListKind, dnd.LocationRegistryKind, dnd.LocationOccurrenceListKind, dnd.ItemRegistryKind, dnd.ItemOccurrenceListKind, dnd.EnemyEventListKind}
|
|
assertProductionContains(t, "artifact codec kinds", registries.ArtifactCodecs.RegisteredKinds(), allArtifactKinds)
|
|
assertProductionContains(t, "merger variants", registries.Mergers.RegisteredArtifactKinds(pipeline.DefaultMergeModule), allArtifactKinds)
|
|
assertProductionContains(t, "normalizer variants", registries.Normalizers.RegisteredArtifactKinds(pipeline.DefaultNormalizeModule), allArtifactKinds)
|
|
assertProductionContains(t, "spell normalizer variants", registries.Normalizers.RegisteredArtifactKinds(spellnormalize.Key), []contracts.ArtifactKind{dnd.SpellListKind})
|
|
assertProductionContains(t, "combat normalizer variants", registries.Normalizers.RegisteredArtifactKinds(combatnormalize.Key), []contracts.ArtifactKind{dnd.CombatTurnListKind})
|
|
assertProductionContains(t, "item occurrence normalizer variants", registries.Normalizers.RegisteredArtifactKinds(itemoccurrencenormalize.Key), []contracts.ArtifactKind{dnd.ItemOccurrenceListKind})
|
|
assertProductionContains(t, "item registry normalizer variants", registries.Normalizers.RegisteredArtifactKinds(itemregistrynormalize.Key), []contracts.ArtifactKind{dnd.ItemRegistryKind})
|
|
assertProductionContains(t, "NPC registry normalizer variants", registries.Normalizers.RegisteredArtifactKinds(npcnormalize.Key), []contracts.ArtifactKind{dnd.NPCRegistryKind})
|
|
assertProductionContains(t, "NPC occurrence normalizer variants", registries.Normalizers.RegisteredArtifactKinds(npcoccurrencenormalize.Key), []contracts.ArtifactKind{dnd.NPCOccurrenceListKind})
|
|
assertProductionContains(t, "scene description normalizer variants", registries.Normalizers.RegisteredArtifactKinds(scenenormalize.Key), []contracts.ArtifactKind{dnd.SceneDescriptionListKind})
|
|
assertProductionContains(t, "location registry normalizer variants", registries.Normalizers.RegisteredArtifactKinds(locationnormalize.Key), []contracts.ArtifactKind{dnd.LocationRegistryKind})
|
|
assertProductionContains(t, "location occurrence normalizer variants", registries.Normalizers.RegisteredArtifactKinds(locationoccurrencenormalize.Key), []contracts.ArtifactKind{dnd.LocationOccurrenceListKind})
|
|
assertProductionContains(t, "enemy event normalizer variants", registries.Normalizers.RegisteredArtifactKinds(enemyeventnormalize.Key), []contracts.ArtifactKind{dnd.EnemyEventListKind})
|
|
|
|
catalog := catalogFromRegistries(registries)
|
|
for _, test := range []struct {
|
|
stage pipeline.ModuleStage
|
|
key string
|
|
want contracts.ExecutionClass
|
|
}{
|
|
{stage: pipeline.StageInput, key: "seriatim", want: contracts.ExecutionClassDeterministic},
|
|
{stage: pipeline.StageChunk, key: "generic", want: contracts.ExecutionClassDeterministic},
|
|
{stage: pipeline.StageChunk, key: "dnd/scenes", want: contracts.ExecutionClassLLMBacked},
|
|
{stage: pipeline.StageExtract, key: "dnd/spells", want: contracts.ExecutionClassLLMBacked},
|
|
{stage: pipeline.StageExtract, key: "dnd/npc-registry", want: contracts.ExecutionClassLLMBacked},
|
|
{stage: pipeline.StageExtract, key: "dnd/location-registry", want: contracts.ExecutionClassLLMBacked},
|
|
{stage: pipeline.StageExtract, key: "dnd/item-registry", want: contracts.ExecutionClassLLMBacked},
|
|
{stage: pipeline.StageExtract, key: "dnd/combat-turns", want: contracts.ExecutionClassLLMBacked},
|
|
{stage: pipeline.StageExtract, key: "dnd/item-occurrences", want: contracts.ExecutionClassLLMBacked},
|
|
{stage: pipeline.StageExtract, key: "dnd/npc-occurrences", want: contracts.ExecutionClassLLMBacked},
|
|
{stage: pipeline.StageExtract, key: "dnd/location-occurrences", want: contracts.ExecutionClassLLMBacked},
|
|
{stage: pipeline.StageExtract, key: "dnd/scene-descriptions", want: contracts.ExecutionClassLLMBacked},
|
|
{stage: pipeline.StageExtract, key: enemyeventextract.Key, want: contracts.ExecutionClassLLMBacked},
|
|
{stage: pipeline.StageMerge, key: "appendorder", want: contracts.ExecutionClassDeterministic},
|
|
{stage: pipeline.StageNormalize, key: "noop", want: contracts.ExecutionClassDeterministic},
|
|
{stage: pipeline.StageNormalize, key: "dnd/spells", want: contracts.ExecutionClassDeterministic},
|
|
{stage: pipeline.StageNormalize, key: "dnd/npc-registry", want: contracts.ExecutionClassLLMBacked},
|
|
{stage: pipeline.StageNormalize, key: "dnd/location-registry", want: contracts.ExecutionClassLLMBacked},
|
|
{stage: pipeline.StageNormalize, key: "dnd/item-registry", want: contracts.ExecutionClassLLMBacked},
|
|
{stage: pipeline.StageNormalize, key: "dnd/combat-turns", want: contracts.ExecutionClassDeterministic},
|
|
{stage: pipeline.StageNormalize, key: "dnd/item-occurrences", want: contracts.ExecutionClassDeterministic},
|
|
{stage: pipeline.StageNormalize, key: "dnd/npc-occurrences", want: contracts.ExecutionClassDeterministic},
|
|
{stage: pipeline.StageNormalize, key: "dnd/location-occurrences", want: contracts.ExecutionClassDeterministic},
|
|
{stage: pipeline.StageNormalize, key: "dnd/scene-descriptions", want: contracts.ExecutionClassDeterministic},
|
|
{stage: pipeline.StageNormalize, key: enemyeventnormalize.Key, want: contracts.ExecutionClassDeterministic},
|
|
{stage: pipeline.StageOutput, key: "json", want: contracts.ExecutionClassDeterministic},
|
|
} {
|
|
got, ok := catalog.ExecutionClass(test.stage, test.key)
|
|
if !ok || got != test.want {
|
|
t.Fatalf("production execution class for %s/%s = %q, %t; want %q, true", test.stage, test.key, got, ok, test.want)
|
|
}
|
|
}
|
|
converted := registriesFromCatalog(catalog)
|
|
if converted.ArtifactCodecs != registries.ArtifactCodecs || converted.ArtifactEvidence != registries.ArtifactEvidence || converted.ValidatorChains != registries.ValidatorChains {
|
|
t.Fatal("catalog/registry conversion did not preserve artifact and validator registries")
|
|
}
|
|
codecSpec, ok := catalog.ArtifactCodecs.Spec(dnd.SpellListKind)
|
|
if !ok || codecSpec.Kind != dnd.SpellListKind || codecSpec.Schema.ID != spellcodec.SchemaID {
|
|
t.Fatalf("catalog codec spec = %#v, ok=%t, want typed D&D spell codec", codecSpec, ok)
|
|
}
|
|
combatCodecSpec, ok := catalog.ArtifactCodecs.Spec(dnd.CombatTurnListKind)
|
|
if !ok || combatCodecSpec.Kind != dnd.CombatTurnListKind || combatCodecSpec.Schema.ID != combatcodec.SchemaID {
|
|
t.Fatalf("combat codec spec = %#v, ok=%t, want typed D&D combat codec", combatCodecSpec, ok)
|
|
}
|
|
itemOccurrenceCodecSpec, ok := catalog.ArtifactCodecs.Spec(dnd.ItemOccurrenceListKind)
|
|
if !ok || itemOccurrenceCodecSpec.Kind != dnd.ItemOccurrenceListKind || itemOccurrenceCodecSpec.Schema.ID != itemoccurrencecodec.SchemaID {
|
|
t.Fatalf("item occurrence codec spec = %#v, ok=%t, want typed D&D item-occurrence codec", itemOccurrenceCodecSpec, ok)
|
|
}
|
|
for _, test := range []struct {
|
|
name string
|
|
kind contracts.ArtifactKind
|
|
id string
|
|
}{
|
|
{name: "NPC registry", kind: dnd.NPCRegistryKind, id: npccodec.SchemaID},
|
|
{name: "NPC occurrences", kind: dnd.NPCOccurrenceListKind, id: npcoccurrencecodec.SchemaID},
|
|
{name: "item registry", kind: dnd.ItemRegistryKind, id: itemregistrycodec.SchemaID},
|
|
{name: "scene descriptions", kind: dnd.SceneDescriptionListKind, id: scenecodec.SchemaID},
|
|
{name: "location registry", kind: dnd.LocationRegistryKind, id: locationcodec.SchemaID},
|
|
{name: "location occurrences", kind: dnd.LocationOccurrenceListKind, id: locationoccurrencecodec.SchemaID},
|
|
} {
|
|
spec, found := catalog.ArtifactCodecs.Spec(test.kind)
|
|
if !found || spec.Kind != test.kind || spec.Schema.ID != test.id {
|
|
t.Fatalf("%s codec spec = %#v, found=%t, want typed D&D codec", test.name, spec, found)
|
|
}
|
|
}
|
|
enemyEventCodecSpec, ok := catalog.ArtifactCodecs.Spec(dnd.EnemyEventListKind)
|
|
if !ok || enemyEventCodecSpec.Kind != dnd.EnemyEventListKind || enemyEventCodecSpec.Schema.ID != enemyeventcodec.SchemaID {
|
|
t.Fatalf("enemy event codec spec = %#v, ok=%t, want typed D&D enemy-event codec", enemyEventCodecSpec, ok)
|
|
}
|
|
}
|
|
|
|
func TestDefaultCLICompositionValidatesRepresentativeConfiguration(t *testing.T) {
|
|
path := writeProductionSpellCatalogContractConfig(t)
|
|
var stdout, stderr strings.Builder
|
|
code := RunWithOptions([]string{"pipeline", "validate", "dnd-session", "--config", path}, &stdout, &stderr, Options{})
|
|
if code != 0 {
|
|
t.Fatalf("validate production composition: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
}
|
|
|
|
func TestProductionAssetsResolveDNDExtractionProfile(t *testing.T) {
|
|
for _, test := range []struct {
|
|
name, companion string
|
|
invalid bool
|
|
}{
|
|
{name: "embedded defaults"},
|
|
{name: "operator override", companion: `collections:
|
|
notarius:
|
|
profiles:
|
|
- id: dnd-extraction
|
|
endpoint: https://operator.example.invalid/v1
|
|
model: operator-model
|
|
timeout_seconds: 75
|
|
`},
|
|
{name: "invalid operator override", companion: `collections:
|
|
notarius:
|
|
profiles:
|
|
- id: dnd-extraction
|
|
model: [invalid]
|
|
`, invalid: true},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
path := writeProductionContractConfig(t, "version: 6\n")
|
|
if test.companion != "" {
|
|
if err := os.WriteFile(filepath.Join(filepath.Dir(path), "prompts.yml"), []byte(test.companion), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
loaders := &nativeLoaders{lookupEnv: emptyLookup}
|
|
engine, err := loaders.engine(context.Background(), productionSelection(path))
|
|
if test.invalid {
|
|
if err == nil {
|
|
t.Fatal("invalid operator resource fell back to embedded defaults")
|
|
}
|
|
return
|
|
}
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
profile, err := engine.InspectProfile(context.Background(), "notarius/dnd-extraction")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if test.companion == "" && (profile.Backend == nil || *profile.Backend != "openrouter" || profile.ServiceTier == nil || *profile.ServiceTier != "flex" || profile.ReasoningEffort != nil) {
|
|
t.Fatalf("lost embedded profile defaults: %#v", profile)
|
|
}
|
|
model, timeout := "openai/gpt-5.6-luna", 240
|
|
if test.companion != "" {
|
|
model, timeout = "operator-model", 75
|
|
if profile.Endpoint == nil || *profile.Endpoint != "https://operator.example.invalid/v1" {
|
|
t.Fatalf("operator endpoint=%#v", profile.Endpoint)
|
|
}
|
|
}
|
|
if profile.Model == nil || *profile.Model != model || profile.TimeoutSeconds == nil || *profile.TimeoutSeconds != timeout {
|
|
t.Fatalf("profile=%#v", profile)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestProductionModulesPrepareWithoutProviderCalls(t *testing.T) {
|
|
components := productionTestComponents(t)
|
|
cfg := config.Default()
|
|
cfg.Pipelines["dnd-scenes"] = pipeline.PipelineProfile{
|
|
ID: "dnd-scenes", Input: pipeline.Binding("seriatim"), Chunk: pipeline.Binding("dnd/scenes"),
|
|
Artifacts: map[string]pipeline.ArtifactLaneProfile{"spells": {Extract: pipeline.Binding("dnd/spells")}},
|
|
}
|
|
effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-scenes", Catalog: catalogFromRegistries(components.registries)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
client := &productionFakeLLMClient{}
|
|
if _, err := pipeline.Prepare(effective.ResolvedPipeline, components.registries, pipeline.ModuleDependencies{Engine: productionNativeEngine(t, client)}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(client.requestsSnapshot()) != 0 {
|
|
t.Fatal("module preparation invoked provider")
|
|
}
|
|
}
|
|
|
|
func TestProductionSpellValidatorsPrepareFromMaterializedCatalog(t *testing.T) {
|
|
components := productionTestComponents(t)
|
|
configPath := writeProductionSpellCatalogContractConfig(t)
|
|
effective, err := loadMaintainedExample(t, configPath).Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalogFromRegistries(components.registries)})
|
|
if err != nil {
|
|
t.Fatalf("resolve production spell configuration: %v", err)
|
|
}
|
|
materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{
|
|
ConfigPath: configPath,
|
|
WorkingDir: filepath.Dir(configPath),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("materialize production spell references: %v", err)
|
|
}
|
|
extractItems := materialized.Steps[0].ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["spell_catalog"].Items
|
|
normalizeItems := materialized.Steps[0].ArtifactLanes[0].NormalizeReferences.ReferenceSet.Slots["spell_catalog"].Items
|
|
if len(extractItems) != 1 || extractItems[0].MediaType != "application/json" || len(extractItems[0].Content) == 0 {
|
|
t.Fatalf("materialized extract spell catalog items = %#v, want one JSON item", extractItems)
|
|
}
|
|
if len(normalizeItems) != 1 || normalizeItems[0].MediaType != "application/json" || !reflect.DeepEqual(normalizeItems[0].Content, extractItems[0].Content) {
|
|
t.Fatalf("materialized normalize spell catalog items = %#v, want an independent binding of the extract catalog", normalizeItems)
|
|
}
|
|
if _, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{Engine: productionNativeEngine(t, &productionFakeLLMClient{})}); err != nil {
|
|
t.Fatalf("prepare production spell pipeline from materialized catalog: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestProductionSpellNormalizerRejectsInvalidCatalogReferencesBeforeExecution(t *testing.T) {
|
|
components := productionTestComponents(t)
|
|
configPath := writeProductionSpellCatalogContractConfig(t)
|
|
resolve := func(t *testing.T) pipeline.ResolvedPipeline {
|
|
t.Helper()
|
|
effective, err := loadMaintainedExample(t, configPath).Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalogFromRegistries(components.registries)})
|
|
if err != nil {
|
|
t.Fatalf("resolve production spell configuration: %v", err)
|
|
}
|
|
return effective.ResolvedPipeline
|
|
}
|
|
materialize := func(resolved pipeline.ResolvedPipeline) (pipeline.ResolvedPipeline, error) {
|
|
materialized, _, err := pipeline.MaterializeReferences(resolved, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{
|
|
ConfigPath: configPath,
|
|
WorkingDir: filepath.Dir(configPath),
|
|
})
|
|
return materialized, err
|
|
}
|
|
|
|
t.Run("malformed catalog fails preparation", func(t *testing.T) {
|
|
catalogPath := filepath.Join(t.TempDir(), "malformed.json")
|
|
if err := os.WriteFile(catalogPath, []byte(`{"schema_version":`), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
resolved := resolve(t)
|
|
setNormalizeSpellCatalogSource(t, &resolved, catalogPath)
|
|
materialized, err := materialize(resolved)
|
|
if err != nil {
|
|
t.Fatalf("MaterializeReferences() error = %v, want malformed JSON to reach preparation", err)
|
|
}
|
|
_, err = pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{Engine: productionNativeEngine(t, &productionFakeLLMClient{})})
|
|
for _, fragment := range []string{`pipeline "dnd-session"`, `lane "spells"`, "normalize", `module "dnd/spells"`, "decode spell catalog overlay"} {
|
|
if err == nil || !strings.Contains(err.Error(), fragment) {
|
|
t.Fatalf("Prepare() error = %v, want context fragment %q", err, fragment)
|
|
}
|
|
}
|
|
})
|
|
|
|
t.Run("multiple catalog items fail preparation", func(t *testing.T) {
|
|
materialized, err := materialize(resolve(t))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
slot := materialized.Steps[0].ArtifactLanes[0].NormalizeReferences.ReferenceSet.Slots["spell_catalog"]
|
|
slot.Items = append(slot.Items, slot.Items[0])
|
|
materialized.Steps[0].ArtifactLanes[0].NormalizeReferences.ReferenceSet.Slots["spell_catalog"] = slot
|
|
_, err = pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{Engine: productionNativeEngine(t, &productionFakeLLMClient{})})
|
|
for _, fragment := range []string{"normalize", `module "dnd/spells"`, "zero or one item"} {
|
|
if err == nil || !strings.Contains(err.Error(), fragment) {
|
|
t.Fatalf("Prepare() error = %v, want context fragment %q", err, fragment)
|
|
}
|
|
}
|
|
})
|
|
|
|
t.Run("oversized catalog fails materialization", func(t *testing.T) {
|
|
catalogPath := filepath.Join(t.TempDir(), "oversized.json")
|
|
if err := os.WriteFile(catalogPath, []byte(strings.Repeat("x", 1048577)), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
content := productionSpellCatalogContractConfig(t)
|
|
catalogSource := repositoryPath("examples", "dnd-complete", "spell-catalog.json")
|
|
content = strings.Replace(content, catalogSource, "__extract_catalog__", 1)
|
|
content = replaceRequiredOnce(t, content, catalogSource, catalogPath)
|
|
content = replaceRequiredOnce(t, content, "__extract_catalog__", catalogSource)
|
|
configFile := writeProductionContractConfig(t, content)
|
|
client := &productionFakeLLMClient{}
|
|
var stdout, stderr strings.Builder
|
|
code := RunWithOptions([]string{"run", "dnd-session", "--config", configFile, "--input-file", repositoryPath("examples", "dnd-minimal", "transcript.json"), "--cache", "bypass"}, &stdout, &stderr, productionRunOptions(t, client))
|
|
for _, fragment := range []string{"normalize", `lane "spells"`, `reference slot "spell_catalog"`, "1048577 bytes", "limit 1048576"} {
|
|
if code == 0 || !strings.Contains(stderr.String(), fragment) {
|
|
t.Fatalf("status=%d stderr=%q, want %q", code, stderr.String(), fragment)
|
|
}
|
|
}
|
|
if len(client.requestsSnapshot()) != 0 || stdout.Len() != 0 {
|
|
t.Fatal("invalid reference reached provider or completion reporting")
|
|
}
|
|
if _, err := os.Stat(filepath.Join(filepath.Dir(configFile), "notarius-output")); !errors.Is(err, fs.ErrNotExist) {
|
|
t.Fatalf("output allocation before invalid reference: %v", err)
|
|
}
|
|
|
|
})
|
|
}
|
|
|
|
func setNormalizeSpellCatalogSource(t *testing.T, resolved *pipeline.ResolvedPipeline, sourcePath string) {
|
|
t.Helper()
|
|
if resolved == nil || len(resolved.Steps[0].ArtifactLanes) != 1 {
|
|
t.Fatalf("resolved pipeline = %#v, want one artifact lane", resolved)
|
|
}
|
|
bindings := resolved.Steps[0].ArtifactLanes[0].NormalizeReferences.Bindings
|
|
matches := 0
|
|
for index := range bindings {
|
|
if bindings[index].SlotName == "spell_catalog" {
|
|
bindings[index].Source = sourcePath
|
|
matches++
|
|
}
|
|
}
|
|
if matches != 1 {
|
|
t.Fatalf("normalize reference bindings = %#v, want exactly one spell_catalog binding", bindings)
|
|
}
|
|
resolved.Steps[0].ArtifactLanes[0].NormalizeReferences.Bindings = bindings
|
|
}
|
|
|
|
func TestProductionNativeEngineUsesConfiguredBackend(t *testing.T) {
|
|
var providerCalls atomic.Int32
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
providerCalls.Add(1)
|
|
if r.URL.Path != "/v1/chat/completions" {
|
|
t.Errorf("provider path=%q", r.URL.Path)
|
|
}
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_, _ = w.Write([]byte(`{"choices":[{"message":{"role":"assistant","content":"{\"ok\":true}"}}],"usage":{"prompt_tokens":3,"completion_tokens":4,"total_tokens":7}}`))
|
|
}))
|
|
defer server.Close()
|
|
path := writeProductionContractConfig(t, fmt.Sprintf(`version: 6
|
|
promptkit:
|
|
backends:
|
|
- id: operator
|
|
endpoint: %q
|
|
concurrency_limit: 2
|
|
profiles:
|
|
- id: local-profile
|
|
backend: operator
|
|
model: local-model
|
|
api_key_required: false
|
|
prompts:
|
|
- id: local-request
|
|
default_profile: local-profile
|
|
inputs: [{name: transcript, required: true}]
|
|
messages: [{role: user, content: '{{input "transcript"}}'}]
|
|
output: {format: json, validation_mode: json}
|
|
`, server.URL+"/v1"))
|
|
loaders := &nativeLoaders{lookupEnv: emptyLookup}
|
|
engine, err := loaders.engine(context.Background(), productionSelection(path))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "local-request", Inputs: map[string]promptkit.ArtifactRef{"transcript": promptkit.Inline("local request")}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result == nil || !result.Validation.IsValid || strings.TrimSpace(result.RawOutput) != `{"ok":true}` || providerCalls.Load() != 1 {
|
|
t.Fatalf("native result=%#v calls=%d", result, providerCalls.Load())
|
|
}
|
|
}
|
|
|
|
func TestProductionNativeLoadHonorsCanceledContext(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
loaders := &nativeLoaders{lookupEnv: emptyLookup}
|
|
engine, err := loaders.engine(ctx, productionSelection(filepath.Join(t.TempDir(), "missing.yml")))
|
|
if !errors.Is(err, context.Canceled) || engine != nil {
|
|
t.Fatalf("engine=%v error=%v", engine, err)
|
|
}
|
|
}
|
|
|
|
func TestProductionConfigValidationCoversModuleAndVariantFailures(t *testing.T) {
|
|
base := productionSpellCatalogContractConfig(t)
|
|
validPath := writeProductionContractConfig(t, base)
|
|
options := productionCLIOptions(t)
|
|
var stdout, stderr strings.Builder
|
|
if code := RunWithOptions([]string{"pipeline", "validate", "dnd-session", "--config", validPath}, &stdout, &stderr, options); code != 0 {
|
|
t.Fatalf("valid production config: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
tests := []struct {
|
|
name string
|
|
content string
|
|
options Options
|
|
fragments []string
|
|
}{
|
|
{
|
|
name: "unknown module",
|
|
content: replaceRequiredOnce(t, base, " input:\n module: seriatim\n", " input:\n module: missing/input\n"),
|
|
options: productionCLIOptions(t),
|
|
fragments: []string{"pipeline \"dnd-session\"", "input", "missing/input"},
|
|
},
|
|
{
|
|
name: "unknown validator",
|
|
content: replaceRequiredOnce(t, base, " extract:\n module: dnd/spells\n", " extract:\n module: dnd/spells\n validators:\n - module: missing/validator\n"),
|
|
options: productionCLIOptions(t),
|
|
fragments: []string{"validator", "missing/validator"},
|
|
},
|
|
{
|
|
name: "invalid artifact variant",
|
|
content: replaceRequiredOnce(t, base, " normalize:\n module: dnd/spells\n", " normalize:\n module: noop\n"),
|
|
options: productionCLIOptionsWithoutSpellNormalizer(t),
|
|
fragments: []string{"normalizer", "noop", string(dnd.SpellListKind)},
|
|
},
|
|
{
|
|
name: "deterministic validator with profile",
|
|
content: replaceRequiredOnce(t, base, " extract:\n module: dnd/spells\n", " extract:\n module: dnd/spells\n validators:\n - module: generic/valid_json\n llm_profile: forbidden-profile\n"),
|
|
options: productionCLIOptions(t),
|
|
fragments: []string{"deterministic validator", "llm_profile"},
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
path := writeProductionContractConfig(t, tt.content)
|
|
var stdout, stderr strings.Builder
|
|
code := RunWithOptions([]string{"pipeline", "validate", "dnd-session", "--config", path}, &stdout, &stderr, tt.options)
|
|
if code != 2 {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
for _, fragment := range tt.fragments {
|
|
if !strings.Contains(stderr.String(), fragment) {
|
|
t.Fatalf("stderr=%q, want %q", stderr.String(), fragment)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestProductionNormalizeValidatorOverrideRemainsAuthoritative(t *testing.T) {
|
|
base := productionSpellCatalogContractConfig(t)
|
|
content := replaceRequiredOnce(t, base, " normalize:\n module: dnd/spells\n", " normalize:\n module: dnd/spells\n validators:\n - module: generic/always_accept\n - module: generic/valid_json\n")
|
|
path := writeProductionContractConfig(t, content)
|
|
components := productionTestComponents(t)
|
|
effective, err := loadMaintainedExample(t, path).Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalogFromRegistries(components.registries)})
|
|
if err != nil {
|
|
t.Fatalf("resolve normalize override: %v", err)
|
|
}
|
|
for _, chain := range effective.ResolvedPipeline.ValidatorChains {
|
|
if chain.Stage != pipeline.StageNormalize || chain.ModuleKey != spellnormalize.Key {
|
|
continue
|
|
}
|
|
if len(chain.Validators) != 2 || chain.Validators[0].Binding.Module != "generic/always_accept" || chain.Validators[1].Binding.Module != "generic/valid_json" {
|
|
t.Fatalf("normalize validator chain = %#v, want explicit validator order", chain)
|
|
}
|
|
return
|
|
}
|
|
t.Fatalf("resolved validator chains = %#v, want normalize chain for %q", effective.ResolvedPipeline.ValidatorChains, spellnormalize.Key)
|
|
}
|
|
|
|
func TestProductionSceneRunPublishesAnnotationFreeChunkPlan(t *testing.T) {
|
|
outputRoot := filepath.Join(t.TempDir(), "output")
|
|
path := writeProductionContractConfig(t, productionRunConfig(outputRoot, "dnd/scenes"))
|
|
client := &productionFakeLLMClient{}
|
|
var stdout, stderr strings.Builder
|
|
code := RunWithOptions([]string{"run", "dnd-session", "--config", path, "--input-file", repositoryPath("examples", "dnd-minimal", "transcript.json"), "--cache", "bypass", "--llm-session-id", "offline-session", "--format", "json"}, &stdout, &stderr, productionRunOptions(t, client))
|
|
if code != 0 {
|
|
t.Fatalf("status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
var report runResult
|
|
if err := json.Unmarshal([]byte(stdout.String()), &report); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if report.ArtifactCount != 4 || report.FallbackCount != 0 || report.OutputDirectory != filepath.Join(outputRoot, productionRunID) {
|
|
t.Fatalf("completion=%#v", report)
|
|
}
|
|
content, err := os.ReadFile(filepath.Join(report.OutputDirectory, "chunk-map.json"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
value, err := chunkmap.New().Decode(content)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if value.SourceID != "session-alpha" || value.SourceDigest == "" || value.PlanDigest == "" || value.RequestedChunker != scenes.Key || value.Producer.InputModule != "seriatim" || value.Producer.ChunkModule != scenes.Key {
|
|
t.Fatalf("chunk map=%#v", value)
|
|
}
|
|
if len(value.Chunks) != 1 || value.Chunks[0].ID != "chunk-000001" || value.Chunks[0].Index != 0 || value.Chunks[0].SourceRef.SourceID != "session-alpha" || value.Chunks[0].SourceRef.StartUnitID != 1 || value.Chunks[0].SourceRef.EndUnitID != 2 || value.Chunks[0].UnitCount != 2 {
|
|
t.Fatalf("chunks=%#v", value.Chunks)
|
|
}
|
|
if len(value.PlanAnnotations) != 0 || len(value.Chunks[0].Annotations) != 0 {
|
|
t.Fatalf("unexpected annotations: %#v", value)
|
|
}
|
|
for _, property := range []string{"scenes", "spell_casts", "occurrences"} {
|
|
if len(client.requestsWithProperty(property)) != 1 {
|
|
t.Fatalf("%s provider calls=%d", property, len(client.requestsWithProperty(property)))
|
|
}
|
|
}
|
|
for _, name := range []string{"manifest.json", "index.json", "warnings.json"} {
|
|
if _, err := os.Stat(filepath.Join(report.OutputDirectory, name)); !errors.Is(err, fs.ErrNotExist) {
|
|
t.Fatalf("retired reporting sidecar %s: %v", name, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
type maintainedExample struct {
|
|
name string
|
|
path string
|
|
transcriptPath string
|
|
pipelineIDs []string
|
|
}
|
|
|
|
func maintainedExampleFiles(t *testing.T) []maintainedExample {
|
|
t.Helper()
|
|
return []maintainedExample{
|
|
{name: "minimal", path: repositoryPath("examples", "dnd-minimal", "config.yml"), transcriptPath: repositoryPath("examples", "dnd-minimal", "transcript.json"), pipelineIDs: []string{"dnd-session"}},
|
|
{name: "complete", path: repositoryPath("examples", "dnd-complete", "config.yml"), transcriptPath: repositoryPath("examples", "dnd-complete", "transcript.json"), pipelineIDs: []string{"dnd-session", "dnd-session-development"}},
|
|
}
|
|
}
|
|
|
|
func productionSpellCatalogContractConfig(t *testing.T) string {
|
|
t.Helper()
|
|
return fmt.Sprintf(`version: 6
|
|
pipelines:
|
|
dnd-session:
|
|
input:
|
|
module: seriatim
|
|
references:
|
|
party:
|
|
path: %q
|
|
glossary:
|
|
path: %q
|
|
artifacts:
|
|
spells:
|
|
extract:
|
|
module: dnd/spells
|
|
references:
|
|
spell_catalog:
|
|
path: %q
|
|
normalize:
|
|
module: dnd/spells
|
|
references:
|
|
spell_catalog:
|
|
path: %q
|
|
`, repositoryPath("examples", "dnd-complete", "party.txt"), repositoryPath("examples", "dnd-complete", "glossary.txt"), repositoryPath("examples", "dnd-complete", "spell-catalog.json"), repositoryPath("examples", "dnd-complete", "spell-catalog.json"))
|
|
}
|
|
|
|
func writeProductionSpellCatalogContractConfig(t *testing.T) string {
|
|
t.Helper()
|
|
return writeProductionContractConfig(t, productionSpellCatalogContractConfig(t))
|
|
}
|
|
|
|
func loadMaintainedExample(t *testing.T, path string) config.Config {
|
|
t.Helper()
|
|
loaded, err := config.Load(path, config.LoadOptions{})
|
|
if err != nil {
|
|
t.Fatalf("load maintained config %q: %v", path, err)
|
|
}
|
|
cfg := loaded.RuntimeConfig()
|
|
if err := cfg.Validate(); err != nil {
|
|
t.Fatalf("validate maintained config %q: %v", path, err)
|
|
}
|
|
return cfg
|
|
}
|
|
|
|
func productionTestComponents(t *testing.T) productionComponents {
|
|
t.Helper()
|
|
components, err := newProductionComponents()
|
|
if err != nil {
|
|
t.Fatalf("new production components: %v", err)
|
|
}
|
|
return components
|
|
}
|
|
|
|
func productionCLIOptions(t *testing.T) Options {
|
|
t.Helper()
|
|
components := productionTestComponents(t)
|
|
return productionOptionsFromComponents(components)
|
|
}
|
|
|
|
func productionOptionsFromComponents(components productionComponents) Options {
|
|
return Options{
|
|
Catalog: catalogFromRegistries(components.registries),
|
|
Registries: components.registries,
|
|
LookupEnv: emptyLookup,
|
|
}
|
|
}
|
|
|
|
func productionCLIOptionsWithoutSpellNormalizer(t *testing.T) Options {
|
|
t.Helper()
|
|
components := productionTestComponents(t)
|
|
registries := components.registries
|
|
registries.Normalizers = pipeline.NewNormalizerRegistry()
|
|
if err := noop.RegisterTyped[dnd.SpellList](registries.Normalizers, contracts.ArtifactKind("test/other")); err != nil {
|
|
t.Fatalf("register mismatched normalizer: %v", err)
|
|
}
|
|
return productionOptionsFromComponents(productionComponents{registries: registries})
|
|
}
|
|
|
|
const productionRunID = "run-1700000000000000000-0123456789abcdef0123456789abcdef"
|
|
|
|
func productionRunOptions(t *testing.T, client promptkit.LLMClient) Options {
|
|
t.Helper()
|
|
options := productionCLIOptions(t)
|
|
options.Now = func() time.Time { return time.Unix(1700000000, 0).UTC() }
|
|
options.RunIDGenerator = func(time.Time) (string, error) { return productionRunID, nil }
|
|
options.LLMClient = client
|
|
return options
|
|
}
|
|
|
|
func productionRunConfig(outputRoot, chunkModule string) string {
|
|
return fmt.Sprintf(`version: 6
|
|
output:
|
|
directory: %q
|
|
debug:
|
|
directory: %q
|
|
pipelines:
|
|
dnd-session:
|
|
cache: bypass
|
|
input:
|
|
module: seriatim
|
|
chunk:
|
|
module: %s
|
|
output:
|
|
module: json
|
|
options:
|
|
include_chunk_map: true
|
|
steps:
|
|
- id: identify-items
|
|
artifacts:
|
|
item-registry:
|
|
extract:
|
|
module: dnd/item-registry
|
|
normalize:
|
|
module: dnd/item-registry
|
|
- id: extract-occurrences
|
|
references:
|
|
item_registry:
|
|
artifact:
|
|
step: identify-items
|
|
lane: item-registry
|
|
artifacts:
|
|
spells:
|
|
extract:
|
|
module: dnd/spells
|
|
item-occurrences:
|
|
extract:
|
|
module: dnd/item-occurrences
|
|
`+productionProviderConfiguration, outputRoot, filepath.Join(filepath.Dir(outputRoot), "debug"), chunkModule)
|
|
}
|
|
|
|
func writeProductionContractConfig(t *testing.T, content string) string {
|
|
t.Helper()
|
|
path := filepath.Join(t.TempDir(), "config.yml")
|
|
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return path
|
|
}
|
|
|
|
func assertProductionContains[T comparable](t *testing.T, name string, got, required []T) {
|
|
t.Helper()
|
|
available := make(map[T]struct{}, len(got))
|
|
for _, entry := range got {
|
|
available[entry] = struct{}{}
|
|
}
|
|
var missing []T
|
|
for _, entry := range required {
|
|
if _, ok := available[entry]; !ok {
|
|
missing = append(missing, entry)
|
|
}
|
|
}
|
|
if len(missing) > 0 {
|
|
t.Fatalf("%s missing required entries %#v; registered entries are %#v", name, missing, got)
|
|
}
|
|
}
|
|
|
|
func readProductionJSON[T any](t *testing.T, path string) T {
|
|
t.Helper()
|
|
data, err := os.ReadFile(path)
|
|
if err != nil {
|
|
t.Fatalf("read %s: %v", path, err)
|
|
}
|
|
var value T
|
|
if err := json.Unmarshal(data, &value); err != nil {
|
|
t.Fatalf("decode %s: %v", path, err)
|
|
}
|
|
return value
|
|
}
|
|
|
|
const productionProviderConfiguration = `
|
|
promptkit:
|
|
collections:
|
|
notarius:
|
|
profiles:
|
|
- id: dnd-extraction
|
|
endpoint: https://example.invalid/v1
|
|
model: production-test-model
|
|
api_key_required: false
|
|
`
|
|
|
|
func productionSelection(path string) ckconfig.Selection {
|
|
return ckconfig.Selection{Config: commandkit.Supplied[string]{Set: true, Value: path}}
|
|
}
|
|
|
|
func productionNativeEngine(t *testing.T, client promptkit.LLMClient, profileIDs ...string) *promptkit.Engine {
|
|
t.Helper()
|
|
profiles := make([]map[string]any, len(profileIDs))
|
|
for index, id := range profileIDs {
|
|
profiles[index] = map[string]any{"id": id, "endpoint": "https://example.invalid/v1", "model": id + "-model", "api_key_required": false}
|
|
}
|
|
content, err := json.Marshal(map[string]any{"version": 6, "promptkit": map[string]any{
|
|
"profiles": profiles,
|
|
"collections": map[string]any{"notarius": map[string]any{"profiles": []map[string]any{{"id": "dnd-extraction", "endpoint": "https://example.invalid/v1", "model": "production-test-model", "api_key_required": false}}}},
|
|
}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
path := writeProductionContractConfig(t, string(content))
|
|
loaders := &nativeLoaders{lookupEnv: emptyLookup, engineOptions: []promptkit.Option{promptkit.WithLLMClient(client)}}
|
|
engine, err := loaders.engine(context.Background(), productionSelection(path))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return engine
|
|
}
|
|
|
|
func productionJSONInput(request promptkit.GenerateRequest, field string) ([]byte, error) {
|
|
for _, message := range request.Prompt.Messages {
|
|
for offset := range len(message.Content) {
|
|
if message.Content[offset] != '{' {
|
|
continue
|
|
}
|
|
decoder := json.NewDecoder(strings.NewReader(message.Content[offset:]))
|
|
var object map[string]json.RawMessage
|
|
if decoder.Decode(&object) == nil && object[field] != nil {
|
|
return []byte(message.Content[offset : offset+int(decoder.InputOffset())]), nil
|
|
}
|
|
}
|
|
}
|
|
return nil, fmt.Errorf("native request has no JSON input with field %q", field)
|
|
}
|
|
|
|
type productionFakeLLMClient struct {
|
|
mu sync.Mutex
|
|
requests []promptkit.GenerateRequest
|
|
spellResponse string
|
|
}
|
|
|
|
func (client *productionFakeLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
client.mu.Lock()
|
|
defer client.mu.Unlock()
|
|
client.requests = append(client.requests, request)
|
|
var content string
|
|
switch {
|
|
case productionSchemaHasProperty(request, "scenes"):
|
|
content = `{"scenes":[{"start_unit_id":1,"end_unit_id":2}]}`
|
|
case productionSchemaHasProperty(request, "spell_casts"):
|
|
content = client.spellResponse
|
|
if content == "" {
|
|
content = `{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`
|
|
}
|
|
case productionSchemaHasProperty(request, "occurrences"):
|
|
content = `{"occurrences":[]}`
|
|
case productionSchemaHasProperty(request, "items"):
|
|
content = `{"items":[]}`
|
|
case productionSchemaHasProperty(request, "duplicate_groups"):
|
|
content = `{"duplicate_groups":[]}`
|
|
default:
|
|
return nil, fmt.Errorf("unexpected native response contract")
|
|
}
|
|
return &promptkit.GenerateResponse{Content: content}, nil
|
|
}
|
|
|
|
func (client *productionFakeLLMClient) requestsSnapshot() []promptkit.GenerateRequest {
|
|
client.mu.Lock()
|
|
defer client.mu.Unlock()
|
|
return append([]promptkit.GenerateRequest(nil), client.requests...)
|
|
}
|
|
|
|
func (client *productionFakeLLMClient) requestsWithProperty(property string) []promptkit.GenerateRequest {
|
|
var requests []promptkit.GenerateRequest
|
|
for _, request := range client.requestsSnapshot() {
|
|
if productionSchemaHasProperty(request, property) {
|
|
requests = append(requests, request)
|
|
}
|
|
}
|
|
return requests
|
|
}
|
|
|
|
func productionSchemaHasProperty(request promptkit.GenerateRequest, property string) bool {
|
|
if request.StructuredOutput == nil || request.StructuredOutput.JSONSchema == nil {
|
|
return false
|
|
}
|
|
content, err := json.Marshal(request.StructuredOutput.JSONSchema.Schema)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
var schema struct {
|
|
Properties map[string]json.RawMessage `json:"properties"`
|
|
}
|
|
if json.Unmarshal(content, &schema) != nil {
|
|
return false
|
|
}
|
|
_, ok := schema.Properties[property]
|
|
return ok
|
|
}
|
|
|
|
var _ promptkit.LLMClient = (*productionFakeLLMClient)(nil)
|
|
|
|
func repositoryPath(parts ...string) string {
|
|
_, file, _, _ := runtime.Caller(0)
|
|
return filepath.Join(append([]string{filepath.Dir(file), "..", ".."}, parts...)...)
|
|
}
|
|
|
|
func readRepositoryFile(t *testing.T, parts ...string) []byte {
|
|
t.Helper()
|
|
data, err := os.ReadFile(repositoryPath(parts...))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return data
|
|
}
|