Migrate maintained D&D examples to native configuration
This commit is contained in:
@@ -3,9 +3,11 @@ package cli
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -20,6 +22,8 @@ import (
|
||||
locationnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/locationregistry"
|
||||
spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript"
|
||||
"go.maximumdirect.net/commandkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestMaintainedExamplesLoadResolveAndList(t *testing.T) {
|
||||
@@ -28,8 +32,8 @@ func TestMaintainedExamplesLoadResolveAndList(t *testing.T) {
|
||||
for _, example := range maintainedExampleFiles(t) {
|
||||
t.Run(example.name, func(t *testing.T) {
|
||||
cfg := loadMaintainedExample(t, example.path)
|
||||
if example.name == "complete" && cfg.Concurrency.Workers != 2 {
|
||||
t.Fatalf("complete example concurrency = %#v, want an explicit shared worker limit of 2", cfg.Concurrency)
|
||||
if example.name == "complete" && cfg.Concurrency.Workers != 4 {
|
||||
t.Fatalf("complete example concurrency = %#v, want four shared workers independent of backend admission", cfg.Concurrency)
|
||||
}
|
||||
raw, err := os.ReadFile(example.transcriptPath)
|
||||
if err != nil {
|
||||
@@ -59,6 +63,9 @@ func TestMaintainedExamplesLoadResolveAndList(t *testing.T) {
|
||||
}
|
||||
if example.name == "complete" {
|
||||
if pipelineID == "dnd-session-development" {
|
||||
if materialized.ChunkControls.MaxAdditionalTurns != 0 || materialized.Chunk.LLMProfile != "notarius/dnd-extraction-development" {
|
||||
t.Fatalf("development native controls=%#v profile=%q", materialized.ChunkControls, materialized.Chunk.LLMProfile)
|
||||
}
|
||||
parent, found := completePipelines["dnd-session"]
|
||||
if !found {
|
||||
t.Fatal("complete parent pipeline was not resolved before its development variant")
|
||||
@@ -137,56 +144,104 @@ func TestMaintainedExamplesLoadResolveAndList(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaintainedConfigurationExampleSet(t *testing.T) {
|
||||
entries, err := os.ReadDir(repositoryPath("examples"))
|
||||
func TestMaintainedExamplesIsolateCompanionsAndInheritNativeResources(t *testing.T) {
|
||||
client := &productionFakeLLMClient{}
|
||||
loaders := &nativeLoaders{lookupEnv: emptyLookup, engineOptions: []promptkit.Option{promptkit.WithLLMClient(client)}}
|
||||
ctx := context.Background()
|
||||
completePath := repositoryPath("examples", "dnd-complete", "config.yml")
|
||||
complete, err := loaders.engine(ctx, nativeSelection(completePath))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var names []string
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".config.yml") {
|
||||
names = append(names, entry.Name())
|
||||
base, err := complete.InspectProfile(ctx, "notarius/dnd-extraction")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
example, err := complete.InspectProfile(ctx, "notarius/dnd-extraction-example")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
development, err := complete.InspectProfile(ctx, "notarius/dnd-extraction-development")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, profile := range []*promptkit.ProfileInspection{example, development} {
|
||||
if !reflect.DeepEqual(profile.Model, base.Model) || !reflect.DeepEqual(profile.TimeoutSeconds, base.TimeoutSeconds) || !reflect.DeepEqual(profile.ServiceTier, base.ServiceTier) || profile.Backend == nil || *profile.Backend != "limited-openrouter" {
|
||||
t.Fatalf("inherited profile=%#v, embedded parent=%#v", profile, base)
|
||||
}
|
||||
}
|
||||
sort.Strings(names)
|
||||
if got := strings.Join(names, ","); got != "dnd-complete.config.yml,dnd-minimal.config.yml" {
|
||||
t.Fatalf("maintained configuration examples = %q, want only the minimal and complete D&D examples", got)
|
||||
if development.Temperature == nil || *development.Temperature != 0.2 || !reflect.DeepEqual(example.Temperature, base.Temperature) {
|
||||
t.Fatalf("development customization leaked into parent: %#v %#v", development, example)
|
||||
}
|
||||
bundled, err := complete.InspectBackend(ctx, "openrouter")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
limited, err := complete.InspectBackend(ctx, "limited-openrouter")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if limited.ConcurrencyLimit == nil || *limited.ConcurrencyLimit != 2 || limited.Endpoint != bundled.Endpoint || !reflect.DeepEqual(limited.APIKeyEnv, bundled.APIKeyEnv) || !reflect.DeepEqual(limited.SmokeTests, bundled.SmokeTests) {
|
||||
t.Fatalf("limited backend=%#v, bundled parent=%#v", limited, bundled)
|
||||
}
|
||||
minimal, err := loaders.engine(ctx, nativeSelection(repositoryPath("examples", "dnd-minimal", "config.yml")))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := minimal.InspectProfile(ctx, "notarius/dnd-extraction-example"); !errors.Is(err, promptkit.ErrProfileNotFound) {
|
||||
t.Fatalf("complete example profile leaked into minimal selection: %v", err)
|
||||
}
|
||||
if _, err := minimal.InspectBackend(ctx, "limited-openrouter"); !errors.Is(err, promptkit.ErrBackendNotFound) {
|
||||
t.Fatalf("complete example backend leaked into minimal selection: %v", err)
|
||||
}
|
||||
if len(client.requestsSnapshot()) != 0 {
|
||||
t.Fatal("example inspection called provider")
|
||||
}
|
||||
}
|
||||
|
||||
overlayEntries, err := os.ReadDir(repositoryPath("examples", "overlays"))
|
||||
func TestMaintainedDevelopmentOverlayResolvesNativeControlsAndRelativeOutput(t *testing.T) {
|
||||
path := repositoryPath("examples", "dnd-complete", "config.yml")
|
||||
selection := nativeSelection(path)
|
||||
selection.Overlay = commandkit.Supplied[string]{Set: true, Value: "development"}
|
||||
loaders := &nativeLoaders{lookupEnv: emptyLookup}
|
||||
result, err := loaders.application(context.Background(), selection)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
names = names[:0]
|
||||
for _, entry := range overlayEntries {
|
||||
if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".yml") {
|
||||
names = append(names, entry.Name())
|
||||
}
|
||||
}
|
||||
sort.Strings(names)
|
||||
if got := strings.Join(names, ","); got != "development.yml" {
|
||||
t.Fatalf("maintained configuration overlays = %q, want development.yml", got)
|
||||
}
|
||||
llmProfiles, err := os.ReadDir(repositoryPath("examples", "llm-profiles"))
|
||||
loaded, err := config.FromResult(result)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(llmProfiles) != 1 || llmProfiles[0].Name() != "dnd-extraction.yml" {
|
||||
t.Fatalf("maintained LLM profile files = %#v, want the single combined profile file", llmProfiles)
|
||||
cfg := loaded.RuntimeConfig()
|
||||
if cfg.Concurrency.Workers != 1 || cfg.Output.Directory != filepath.Join(filepath.Dir(path), "notarius-output-development") {
|
||||
t.Fatalf("development application controls=%#v %#v", cfg.Concurrency, cfg.Output)
|
||||
}
|
||||
resolved, err := cfg.Resolve(resolveInputForMaintainedExample(productionTestComponents(t), "dnd-session"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if resolved.ResolvedPipeline.Chunk.LLMProfile != "notarius/dnd-extraction-development" || resolved.ResolvedPipeline.ChunkControls != (contracts.ProducerControls{Mode: contracts.ModeStrict, MaxAdditionalTurns: 0, Cache: contracts.CacheBypass}) {
|
||||
t.Fatalf("development chunk selection=%#v controls=%#v", resolved.ResolvedPipeline.Chunk, resolved.ResolvedPipeline.ChunkControls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaintainedExamplesValidateEffectiveProfilesOffline(t *testing.T) {
|
||||
t.Chdir(repositoryPath())
|
||||
t.Setenv("OPENROUTER_API_KEY", "")
|
||||
client := &productionFakeLLMClient{}
|
||||
options := Options{LLMClient: client}
|
||||
for _, example := range maintainedExampleFiles(t) {
|
||||
t.Run(example.name, func(t *testing.T) {
|
||||
var configOut, configErr strings.Builder
|
||||
if code := RunWithOptions([]string{"config", "validate", "--config", example.path}, &configOut, &configErr, options); code != 0 {
|
||||
t.Fatalf("configuration validation: status=%d stdout=%q stderr=%q", code, configOut.String(), configErr.String())
|
||||
}
|
||||
for _, pipelineID := range example.pipelineIDs {
|
||||
t.Run(pipelineID, func(t *testing.T) {
|
||||
var stdout, stderr strings.Builder
|
||||
code := RunWithOptions([]string{
|
||||
"pipeline", "validate", pipelineID, "--config", example.path,
|
||||
}, &stdout, &stderr, Options{})
|
||||
}, &stdout, &stderr, options)
|
||||
if code != 0 || stderr.Len() != 0 || !strings.Contains(stdout.String(), pipelineID) {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
@@ -198,11 +253,14 @@ func TestMaintainedExamplesValidateEffectiveProfilesOffline(t *testing.T) {
|
||||
var stdout, stderr strings.Builder
|
||||
code := RunWithOptions([]string{
|
||||
"promptkit", "show", "profile", "notarius/dnd-extraction-development",
|
||||
"--config", repositoryPath("examples", "dnd-complete.config.yml"), "--format", "json",
|
||||
}, &stdout, &stderr, Options{})
|
||||
"--config", repositoryPath("examples", "dnd-complete", "config.yml"), "--format", "json",
|
||||
}, &stdout, &stderr, options)
|
||||
if code != 0 || stderr.Len() != 0 || !json.Valid([]byte(stdout.String())) || !strings.Contains(stdout.String(), "dnd-extraction-development") {
|
||||
t.Fatalf("development profile inspection: status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
if len(client.requestsSnapshot()) != 0 {
|
||||
t.Fatal("offline example validation or inspection called provider")
|
||||
}
|
||||
}
|
||||
|
||||
func exampleStepLaneIDs(resolved pipeline.ResolvedPipeline) []string {
|
||||
@@ -244,7 +302,8 @@ func generatedReferenceBindingDescriptions(resolved pipeline.ResolvedPipeline) [
|
||||
}
|
||||
|
||||
func TestMaintainedMinimalInvocationPublishesDomainArtifacts(t *testing.T) {
|
||||
cfg := loadMaintainedExample(t, repositoryPath("examples", "dnd-minimal.config.yml"))
|
||||
t.Setenv("OPENROUTER_API_KEY", "notarius-test-credential")
|
||||
cfg := loadMaintainedExample(t, repositoryPath("examples", "dnd-minimal", "config.yml"))
|
||||
resolved, err := cfg.Resolve(resolveInputForMaintainedExample(productionTestComponents(t), "dnd-session"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -259,8 +318,8 @@ func TestMaintainedMinimalInvocationPublishesDomainArtifacts(t *testing.T) {
|
||||
var stdout, stderr strings.Builder
|
||||
code := RunWithOptions([]string{
|
||||
"run", "dnd-session",
|
||||
"--config", repositoryPath("examples", "dnd-minimal.config.yml"),
|
||||
"--input-file", repositoryPath("examples", "seriatim-minimal-transcript.json"),
|
||||
"--config", repositoryPath("examples", "dnd-minimal", "config.yml"),
|
||||
"--input-file", repositoryPath("examples", "dnd-minimal", "transcript.json"),
|
||||
"--lane", "spells", "--cache", "bypass", "--output-dir", outputRoot, "--format", "json",
|
||||
}, &stdout, &stderr, options)
|
||||
if code != 0 {
|
||||
@@ -301,7 +360,7 @@ func TestMaintainedMalformedInputOnlyRecordsDebugFailureWhenRequested(t *testing
|
||||
options := productionRunOptions(t, client)
|
||||
args := []string{
|
||||
"run", "dnd-session",
|
||||
"--config", repositoryPath("examples", "dnd-minimal.config.yml"),
|
||||
"--config", repositoryPath("examples", "dnd-minimal", "config.yml"),
|
||||
"--input-file", malformed, "--cache", "bypass", "--output-dir", outputRoot,
|
||||
}
|
||||
if debug {
|
||||
@@ -340,7 +399,7 @@ func resolveInputForMaintainedExample(components productionComponents, pipelineI
|
||||
}
|
||||
|
||||
func TestCompleteDNDSharedCLIReferencesExpandAcrossCompatibleTargets(t *testing.T) {
|
||||
cfg := loadMaintainedExample(t, repositoryPath("examples", "dnd-complete.config.yml"))
|
||||
cfg := loadMaintainedExample(t, repositoryPath("examples", "dnd-complete", "config.yml"))
|
||||
catalog := catalogFromRegistries(productionTestComponents(t).registries)
|
||||
sources := map[string]string{
|
||||
"party": "/references/party.txt",
|
||||
|
||||
Reference in New Issue
Block a user