Migrate LLM fixtures to keyed PromptKit sources

This commit is contained in:
2026-10-04 12:15:26 +00:00
parent 30509f9d33
commit 8a8ca6d83c
7 changed files with 277 additions and 299 deletions

View File

@@ -2,12 +2,14 @@ package llm
import (
"context"
"errors"
"io/fs"
"strings"
"testing"
"testing/fstest"
"time"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/promptkit"
)
@@ -18,7 +20,7 @@ func TestAssetRegistryCombinesPromptAndSchemaSources(t *testing.T) {
"prompts/messages/user.tmpl": {Data: []byte(`Input: {{ input "transcript" }}`)},
"prompts/messages/task.tmpl": {Data: []byte("Return JSON.")},
"schemas/ignored/schema.json": {Data: []byte(`{"type":"object"}`)},
}, "prompts")
}, ".")
mustRegisterSchemaFS(t, registry, fstest.MapFS{
"root/schemas/out.json": {Data: []byte(`{"type":"object"}`)},
}, "root")
@@ -42,7 +44,7 @@ func TestAssetRegistryCombinesPromptAndSchemaSources(t *testing.T) {
}
}
func TestAssetRegistryPrepareFailsForMissingPromptAsset(t *testing.T) {
func TestAssetRegistryConstructionFailsForMissingPromptAsset(t *testing.T) {
registry := NewAssetRegistry()
mustRegisterPromptFS(t, registry, fstest.MapFS{
"test.yaml": {Data: []byte(validPromptYAML("out.json"))},
@@ -51,16 +53,21 @@ func TestAssetRegistryPrepareFailsForMissingPromptAsset(t *testing.T) {
"out.json": {Data: []byte(`{"type":"object"}`)},
}, ".")
engine := newPromptKitAssetTestEngine(t, registry)
_, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: "asset.test",
ProfileID: "asset-test-profile",
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.Inline(`{"ok":true}`),
},
})
if err == nil || !strings.Contains(err.Error(), "content_file") {
t.Fatalf("Prepare() error = %v, want missing content_file error", err)
sources, options, err := registry.PromptKitConfigurationSources()
if err != nil {
t.Fatal(err)
}
sources.Underlays = append(sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{
Source: promptkit.ProfileSource("asset test profile", map[string]promptkit.ProfileConfig{
"asset-test-profile": {
Endpoint: promptKitTestString("http://127.0.0.1:1/v1"),
Model: promptKitTestString("asset-test-model"),
},
}),
}}})
_, err = promptkit.NewEngine(promptkit.Config{Sources: sources}, options...)
if err == nil || !errors.Is(err, promptkit.ErrInvalidConfig) || !strings.Contains(err.Error(), "messages/user.tmpl") {
t.Fatalf("NewEngine() error = %v, want invalid configuration for missing prompt content", err)
}
}
@@ -75,8 +82,23 @@ func TestAssetRegistryPrepareFailsForMissingSchemaAsset(t *testing.T) {
"present.json": {Data: []byte(`{"type":"object"}`)},
}, ".")
engine := newPromptKitAssetTestEngine(t, registry)
_, err := engine.Prepare(context.Background(), promptkit.RunRequest{
sources, options, err := registry.PromptKitConfigurationSources()
if err != nil {
t.Fatal(err)
}
sources.Underlays = append(sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{
Source: promptkit.ProfileSource("asset test profile", map[string]promptkit.ProfileConfig{
"asset-test-profile": {
Endpoint: promptKitTestString("http://127.0.0.1:1/v1"),
Model: promptKitTestString("asset-test-model"),
},
}),
}}})
engine, err := promptkit.NewEngine(promptkit.Config{Sources: sources}, options...)
if err != nil {
t.Fatalf("NewEngine() error = %v, want nil", err)
}
_, err = engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: "asset.test",
ProfileID: "asset-test-profile",
Inputs: map[string]promptkit.ArtifactRef{
@@ -84,14 +106,14 @@ func TestAssetRegistryPrepareFailsForMissingSchemaAsset(t *testing.T) {
},
})
if err == nil || !strings.Contains(err.Error(), "missing.json") {
t.Fatalf("Prepare() error = %v, want missing schema error", err)
t.Fatalf("Prepare() error = %v, want missing schema asset failure", err)
}
}
func TestAssetRegistryRejectsDuplicateAssetPaths(t *testing.T) {
registry := NewAssetRegistry()
mustRegisterPromptFS(t, registry, fstest.MapFS{"one/prompt.yaml": {Data: []byte("id: one")}}, "one")
mustRegisterPromptFS(t, registry, fstest.MapFS{"two/prompt.yaml": {Data: []byte("id: two")}}, "two")
mustRegisterPromptFS(t, registry, fstest.MapFS{"one/prompt.yaml": {Data: []byte("prompts: {}")}}, "one")
mustRegisterPromptFS(t, registry, fstest.MapFS{"two/prompt.yaml": {Data: []byte("prompts: {}")}}, "two")
_, err := registry.PromptFS()
if err == nil || !strings.Contains(err.Error(), "duplicate asset path") {
@@ -102,12 +124,12 @@ func TestAssetRegistryRejectsDuplicateAssetPaths(t *testing.T) {
func TestAssetRegistryCombinesFallbackProfileSources(t *testing.T) {
registry := NewAssetRegistry()
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{
"first/profiles/one.yaml": {Data: []byte("id: one\nmodel: first\n")},
"first/profiles/one.yaml": {Data: []byte("profiles:\n one:\n model: first\n")},
}, "first/profiles"); err != nil {
t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err)
}
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{
"second/two.yaml": {Data: []byte("id: two\nmodel: second\n")},
"second/two.yaml": {Data: []byte("profiles:\n two:\n model: second\n")},
}, "second"); err != nil {
t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err)
}
@@ -144,10 +166,10 @@ func TestAssetRegistryRejectsUnreadableFallbackProfileAssets(t *testing.T) {
func TestAssetRegistryRejectsDuplicateFallbackProfilePaths(t *testing.T) {
registry := NewAssetRegistry()
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{"first/profile.yaml": {Data: []byte("id: first\n")}}, "first"); err != nil {
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{"first/profile.yaml": {Data: []byte("profiles:\n first: {}\n")}}, "first"); err != nil {
t.Fatal(err)
}
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{"second/profile.yaml": {Data: []byte("id: second\n")}}, "second"); err != nil {
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{"second/profile.yaml": {Data: []byte("profiles:\n second: {}\n")}}, "second"); err != nil {
t.Fatal(err)
}
_, err := registry.FallbackProfileFS()
@@ -172,8 +194,8 @@ func TestAssetRegistryFallbackProfileDigestTracksContentWithoutLeakingIt(t *test
return digest
}
first := digestFor("id: fallback\nmodel: model-one\n")
second := digestFor("id: fallback\nmodel: model-two\n")
first := digestFor("profiles:\n fallback:\n model: model-one\n")
second := digestFor("profiles:\n fallback:\n model: model-two\n")
if first == second {
t.Fatalf("fallback profile digests = %q and %q, want content change", first, second)
}
@@ -231,16 +253,19 @@ func TestHashAssetsOmitsRawAssetContent(t *testing.T) {
func newPromptKitAssetTestEngine(t *testing.T, registry *AssetRegistry) *promptkit.Engine {
t.Helper()
options, err := registry.PromptKitOptions()
sources, options, err := registry.PromptKitConfigurationSources()
if err != nil {
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
t.Fatalf("PromptKitConfigurationSources() error = %v, want nil", err)
}
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "asset-test-profile",
Endpoint: "http://127.0.0.1:1/v1",
Model: "asset-test-model",
})))
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
sources.Underlays = append(sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{
Source: promptkit.ProfileSource("asset test profile", map[string]promptkit.ProfileConfig{
"asset-test-profile": {
Endpoint: promptKitTestString("http://127.0.0.1:1/v1"),
Model: promptKitTestString("asset-test-model"),
},
}),
}}})
engine, err := promptkit.NewEngine(promptkit.Config{Sources: sources, Timeout: time.Second}, options...)
if err != nil {
t.Fatalf("NewEngine() error = %v, want nil", err)
}
@@ -262,22 +287,23 @@ func mustRegisterSchemaFS(t *testing.T, registry *AssetRegistry, fsys fstest.Map
}
func validPromptYAML(schemaPath string) string {
return `id: asset.test
version: "v1"
inputs:
- name: transcript
required: true
content_type: application/json
messages:
- role: user
content_file: ./messages/user.tmpl
- role: user
content_file: ./messages/task.tmpl
output:
format: json
validation_mode: json_schema
schema_path: ` + schemaPath + `
repair_attempts: 0
return `prompts:
asset.test:
v1:
inputs:
- name: transcript
required: true
content_type: application/json
messages:
- role: user
content_file: ./messages/user.tmpl
- role: user
content_file: ./messages/task.tmpl
output:
format: json
validation_mode: json_schema
schema_path: ` + schemaPath + `
repair_attempts: 0
`
}

View File

@@ -28,6 +28,7 @@ type PromptKitLocalBackendConfig struct {
type PromptKitClientConfig struct {
ProfileDir string
ProfileFile string
Sources promptkit.ConfigurationSources
TransientRetries *int
LocalBackend *PromptKitLocalBackendConfig
Assets *AssetRegistry
@@ -76,7 +77,7 @@ func NewPromptKitClient(cfg PromptKitClientConfig) (*PromptKitClient, error) {
ProfileDir: cfg.ProfileDir,
ProfileFile: cfg.ProfileFile,
LocalBackend: cfg.LocalBackend,
}, cfg.Assets)
}, cfg.Assets, cfg.Sources)
if err != nil {
return nil, &promptKitClientConfigurationError{err: err}
}

View File

@@ -190,94 +190,74 @@ func TestPromptKitClientRejectsInvalidCorrectionsBeforePromptPreparation(t *test
}
func TestPromptKitClientUsesOnePreparedSnapshotForDebugAndGeneration(t *testing.T) {
const initialPrompt = `id: snapshot.test
version: "v1"
default_profile: snapshot-profile
inputs:
- name: transcript
required: true
content_type: application/json
messages:
- role: user
content: "Snapshot A: {{ input \"transcript\" }}"
output:
format: json
validation_mode: json_schema
schema_path: adapter.schema.json
repair_attempts: 0
const initialPrompt = `prompts:
snapshot.test:
v1:
default_profile: snapshot-profile
inputs:
- name: transcript
required: true
content_type: application/json
messages:
- role: user
content: "Snapshot A: {{ input \"transcript\" }}"
output:
format: json
validation_mode: json_schema
schema_path: adapter.schema.json
repair_attempts: 0
`
const updatedPrompt = `id: snapshot.test
version: "v1"
default_profile: snapshot-profile
inputs:
- name: transcript
required: true
content_type: application/json
messages:
- role: user
content: "Snapshot B: {{ input \"transcript\" }}"
output:
format: json
validation_mode: json_schema
schema_path: adapter.schema.json
repair_attempts: 0
const updatedPrompt = `prompts:
snapshot.test:
v1:
default_profile: snapshot-profile
inputs:
- name: transcript
required: true
content_type: application/json
messages:
- role: user
content: "Snapshot B: {{ input \"transcript\" }}"
output:
format: json
validation_mode: json_schema
schema_path: adapter.schema.json
repair_attempts: 0
`
source := &switchingPromptFS{files: fstest.MapFS{
source := fstest.MapFS{
"snapshot.test.yaml": {Data: []byte(initialPrompt)},
}}
}
fake := &fakePromptKitLLM{content: `{"ok":true}`}
client, err := NewPromptKitClient(PromptKitClientConfig{
Assets: newTestPromptKitAssets(t),
EngineOptions: []promptkit.Option{
promptkit.WithPromptFS(source, "."),
promptkit.WithBackend(promptkit.Backend{ID: "snapshot-backend", Endpoint: "http://promptkit.test/v1"}),
promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "snapshot-profile",
BackendID: "snapshot-backend",
Model: "snapshot-model",
})),
promptkit.WithLLMClient(fake),
},
Sources: promptKitTestSources(
map[string]promptkit.BackendConfig{"snapshot-backend": {Endpoint: promptKitTestString("http://promptkit.test/v1")}},
map[string]promptkit.ProfileConfig{"snapshot-profile": {Backend: promptKitTestString("snapshot-backend"), Model: promptKitTestString("snapshot-model")}},
promptKitTestInputs(t, "snapshot-prompt", source, ".")...,
),
EngineOptions: []promptkit.Option{promptkit.WithLLMClient(fake)},
})
if err != nil {
t.Fatalf("NewPromptKitClient() error = %v, want nil", err)
}
opened := source.holdNextPromptRead()
defer source.resumePromptRead()
type completion struct {
response contracts.StructuredCompletionResponse
err error
source["snapshot.test.yaml"] = &fstest.MapFile{Data: []byte(updatedPrompt)}
var out struct {
OK bool `json:"ok"`
}
completed := make(chan completion, 1)
go func() {
var out struct {
OK bool `json:"ok"`
}
response, callErr := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
PromptID: "snapshot.test",
Inputs: contracts.LLMInputSet{
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
},
}, &out)
completed <- completion{response: response, err: callErr}
}()
select {
case <-opened:
case <-time.After(time.Second):
t.Fatal("PromptKit did not read the prompt source")
result, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
PromptID: "snapshot.test",
Inputs: contracts.LLMInputSet{
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
},
}, &out)
if err != nil {
t.Fatalf("CompleteStructured() error = %v, want nil", err)
}
source.replacePrompt([]byte(updatedPrompt))
source.resumePromptRead()
result := <-completed
if result.err != nil {
t.Fatalf("CompleteStructured() error = %v, want nil", result.err)
if result.Debug == nil || result.Debug.Prompt == nil || len(result.Debug.Prompt.Messages) != 1 {
t.Fatalf("debug prompt = %#v, want one prepared message", result.Debug)
}
if result.response.Debug == nil || result.response.Debug.Prompt == nil || len(result.response.Debug.Prompt.Messages) != 1 {
t.Fatalf("debug prompt = %#v, want one prepared message", result.response.Debug)
}
debugContent := result.response.Debug.Prompt.Messages[0].Content
debugContent := result.Debug.Prompt.Messages[0].Content
generatedContent := fake.lastRequest().Prompt.Messages[0].Content
if debugContent != generatedContent {
t.Fatalf("debug content = %q, generation content = %q, want one snapshot", debugContent, generatedContent)
@@ -399,26 +379,22 @@ func TestPromptKitClientAppliesReasoningEffortOverride(t *testing.T) {
}
func TestNewPromptKitClientReportsAssetAndEngineConstructionFailures(t *testing.T) {
t.Run("assets", func(t *testing.T) {
t.Run("asset read", func(t *testing.T) {
registry := NewAssetRegistry()
for _, root := range []string{"one", "two"} {
if err := registry.RegisterPromptFS(fstest.MapFS{
root + "/prompt.yaml": {Data: []byte("id: duplicate")},
}, root); err != nil {
t.Fatalf("RegisterPromptFS() error = %v", err)
}
if err := registry.RegisterPromptFS(unreadableAssetFS{}, "."); err != nil {
t.Fatalf("RegisterPromptFS() error = %v", err)
}
if _, err := NewPromptKitClient(PromptKitClientConfig{Assets: registry}); err == nil ||
!strings.Contains(err.Error(), "duplicate asset path") {
t.Fatalf("NewPromptKitClient() error = %v, want asset construction failure", err)
!errors.Is(err, fs.ErrPermission) || strings.Contains(err.Error(), "permission denied") {
t.Fatalf("NewPromptKitClient() error = %v, want safe asset read failure", err)
}
})
t.Run("engine", func(t *testing.T) {
if _, err := NewPromptKitClient(PromptKitClientConfig{
Assets: newTestPromptKitAssets(t),
EngineOptions: []promptkit.Option{promptkit.WithProfileFile("")},
}); err == nil || !strings.Contains(err.Error(), "create PromptKit engine") {
EngineOptions: []promptkit.Option{promptkit.WithLLMClient(nil)},
}); err == nil || !errors.Is(err, promptkit.ErrInvalidConfig) || strings.Contains(err.Error(), "nil client") {
t.Fatalf("NewPromptKitClient() error = %v, want engine construction failure", err)
}
})
@@ -429,7 +405,7 @@ func TestPromptKitClientCheckpointFingerprintTracksProfileSource(t *testing.T) {
const credentialEnvironment = "PROMPTKIT_TEST_API_KEY"
writeProfile := func(model string) {
t.Helper()
content := "id: checkpoint-profile\nendpoint: http://promptkit.test/v1\nmodel: " + model + "\napi_key_env: " + credentialEnvironment + "\n"
content := "profiles:\n checkpoint-profile:\n endpoint: http://promptkit.test/v1\n model: " + model + "\n api_key_env: " + credentialEnvironment + "\n"
if err := os.WriteFile(profilePath, []byte(content), 0o600); err != nil {
t.Fatal(err)
}
@@ -508,7 +484,7 @@ func TestPromptKitClientCheckpointFingerprintTracksProfileSource(t *testing.T) {
profileDir := t.TempDir()
firstPath := filepath.Join(profileDir, "first-profile.yaml")
secondPath := filepath.Join(profileDir, "second-profile.yaml")
content := []byte("id: directory-profile\nendpoint: http://promptkit.test/v1\nmodel: directory-model\n")
content := []byte("profiles:\n directory-profile:\n endpoint: http://promptkit.test/v1\n model: directory-model\n")
if err := os.WriteFile(firstPath, content, 0o600); err != nil {
t.Fatal(err)
}
@@ -535,17 +511,17 @@ func TestPromptKitClientCheckpointFingerprintTracksProfileSource(t *testing.T) {
profileDir := t.TempDir()
parentPath := filepath.Join(profileDir, "parent.yaml")
leafPath := filepath.Join(profileDir, "leaf.yaml")
if err := os.WriteFile(parentPath, []byte("id: parent\nendpoint: http://promptkit.test/v1\nmodel: parent-one\n"), 0o600); err != nil {
if err := os.WriteFile(parentPath, []byte("profiles:\n parent:\n endpoint: http://promptkit.test/v1\n model: parent-one\n"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(leafPath, []byte("id: leaf\nbase_profile: parent\nmodel: leaf-model\n"), 0o600); err != nil {
if err := os.WriteFile(leafPath, []byte("profiles:\n leaf:\n extends: parent\n model: leaf-model\n"), 0o600); err != nil {
t.Fatal(err)
}
first, err := promptKitProfileFingerprint(profileDir, "", "")
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(parentPath, []byte("id: parent\nendpoint: http://promptkit.test/v1\nmodel: parent-two\n"), 0o600); err != nil {
if err := os.WriteFile(parentPath, []byte("profiles:\n parent:\n endpoint: http://promptkit.test/v1\n model: parent-two\n"), 0o600); err != nil {
t.Fatal(err)
}
second, err := promptKitProfileFingerprint(profileDir, "", "")
@@ -590,7 +566,7 @@ func TestPromptKitClientUsesFallbackProfilesForExecutionAndInspection(t *testing
assets := newTestPromptKitAssets(t)
const profileID = "fallback-profile"
if err := assets.RegisterFallbackProfileFS(fstest.MapFS{
"profiles/fallback.yaml": {Data: []byte("id: " + profileID + "\nendpoint: http://promptkit.test/v1\nmodel: fallback-model\n")},
"profiles/fallback.yaml": {Data: []byte("profiles:\n " + profileID + ":\n endpoint: http://promptkit.test/v1\n model: fallback-model\n")},
}, "profiles"); err != nil {
t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err)
}
@@ -633,10 +609,10 @@ func TestPromptKitClientUsesFallbackProfilesForExecutionAndInspection(t *testing
func TestPromptKitClientUsesInheritedFilesystemProfileForInspectionAndExecution(t *testing.T) {
profileDir := t.TempDir()
if err := os.WriteFile(filepath.Join(profileDir, "base.yaml"), []byte("id: base\nendpoint: http://promptkit.test/v1\nmodel: base-model\nreasoning_effort: medium\n"), 0o600); err != nil {
if err := os.WriteFile(filepath.Join(profileDir, "base.yaml"), []byte("profiles:\n base:\n endpoint: http://promptkit.test/v1\n model: base-model\n reasoning_effort: medium\n"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(profileDir, "leaf.yaml"), []byte("id: inherited-profile\nbase_profile: base\nmodel: leaf-model\n"), 0o600); err != nil {
if err := os.WriteFile(filepath.Join(profileDir, "leaf.yaml"), []byte("profiles:\n inherited-profile:\n extends: base\n model: leaf-model\n"), 0o600); err != nil {
t.Fatal(err)
}
fake := &fakePromptKitLLM{content: `{"ok":true}`}
@@ -690,7 +666,7 @@ func TestPromptKitClientAllowsMissingOptionalFilesystemCredential(t *testing.T)
defer server.Close()
t.Setenv("NOTARIUS_OPTIONAL_PROFILE_KEY", "")
profilePath := filepath.Join(t.TempDir(), "optional.yaml")
if err := os.WriteFile(profilePath, []byte("id: optional-profile\nendpoint: "+server.URL+"/v1\nmodel: optional-model\napi_key_env: NOTARIUS_OPTIONAL_PROFILE_KEY\n"), 0o600); err != nil {
if err := os.WriteFile(profilePath, []byte("profiles:\n optional-profile:\n endpoint: "+server.URL+"/v1\n model: optional-model\n api_key_env: NOTARIUS_OPTIONAL_PROFILE_KEY\n"), 0o600); err != nil {
t.Fatal(err)
}
client, err := NewPromptKitClient(PromptKitClientConfig{Assets: newTestPromptKitAssets(t), ProfileFile: profilePath})
@@ -713,7 +689,7 @@ func TestPromptKitClientMapsHTTPGenerationErrorToApplicationBoundary(t *testing.
defer server.Close()
t.Setenv("NOTARIUS_GENERATION_TEST_KEY", credential)
profilePath := filepath.Join(t.TempDir(), "profile.yaml")
if err := os.WriteFile(profilePath, []byte("id: generation-profile\nendpoint: "+server.URL+"/v1\nmodel: test\napi_key_env: NOTARIUS_GENERATION_TEST_KEY\n"), 0o600); err != nil {
if err := os.WriteFile(profilePath, []byte("profiles:\n generation-profile:\n endpoint: "+server.URL+"/v1\n model: test\n api_key_env: NOTARIUS_GENERATION_TEST_KEY\n"), 0o600); err != nil {
t.Fatal(err)
}
client, err := NewPromptKitClient(PromptKitClientConfig{Assets: newTestPromptKitAssets(t), ProfileFile: profilePath})
@@ -778,8 +754,8 @@ func TestPromptKitClientCheckpointFingerprintTracksFallbackProfileAssets(t *test
return fingerprints[0]
}
first := fingerprintFor("id: fallback\nendpoint: http://promptkit.test/v1\nmodel: model-one\n")
second := fingerprintFor("id: fallback\nendpoint: http://promptkit.test/v1\nmodel: model-two\n")
first := fingerprintFor("profiles:\n fallback:\n endpoint: http://promptkit.test/v1\n model: model-one\n")
second := fingerprintFor("profiles:\n fallback:\n endpoint: http://promptkit.test/v1\n model: model-two\n")
if first == second {
t.Fatalf("checkpoint fingerprints = %#v and %#v, want fallback asset change", first, second)
}
@@ -904,9 +880,10 @@ func TestPromptKitClientUsesConfiguredLocalBackend(t *testing.T) {
defer server.Close()
profilePath := filepath.Join(t.TempDir(), "profiles.yml")
if err := os.WriteFile(profilePath, []byte(`id: local-profile
backend: local
model: local-model
if err := os.WriteFile(profilePath, []byte(`profiles:
local-profile:
backend: local
model: local-model
`), 0o600); err != nil {
t.Fatal(err)
}
@@ -956,19 +933,11 @@ model: local-model
Assets: newTestPromptKitAssets(t),
ProfileFile: profilePath,
})
if err != nil {
t.Fatalf("NewPromptKitClient() without local registration error = %v, want nil", err)
if clientWithoutRegistration != nil || err == nil || !errors.Is(err, promptkit.ErrInvalidConfig) {
t.Fatalf("NewPromptKitClient() without local registration = (%v, %v), want invalid configuration", clientWithoutRegistration, err)
}
_, err = clientWithoutRegistration.CompleteStructured(context.Background(), request, &out)
if err == nil ||
!strings.Contains(err.Error(), "prepare PromptKit prompt") ||
!strings.Contains(err.Error(), promptkit.BackendLocal) {
t.Fatalf("CompleteStructured() without registration error = %v, want preparation failure with local backend context", err)
}
for _, sentinel := range []error{promptkit.ErrProfileLoad, promptkit.ErrInvalidRequest, promptkit.ErrPromptNotFound} {
if errors.Is(err, sentinel) {
t.Fatalf("preparation failure exposes PromptKit sentinel %v: %v", sentinel, err)
}
if strings.Contains(err.Error(), "local-profile") || strings.Contains(err.Error(), profilePath) {
t.Fatalf("configuration error exposes profile details: %v", err)
}
if providerCalls.Load() != 1 {
t.Fatalf("provider calls after missing-registration failure = %d, want 1", providerCalls.Load())
@@ -1009,7 +978,7 @@ func TestPromptKitClientAppliesConfiguredBuiltInTransientRetries(t *testing.T) {
defer server.Close()
profilePath := filepath.Join(t.TempDir(), "profile.yml")
if err := os.WriteFile(profilePath, []byte("id: retry-profile\nendpoint: "+server.URL+"/v1\nmodel: retry-model\n"), 0o600); err != nil {
if err := os.WriteFile(profilePath, []byte("profiles:\n retry-profile:\n endpoint: "+server.URL+"/v1\n model: retry-model\n"), 0o600); err != nil {
t.Fatal(err)
}
client, err := NewPromptKitClient(PromptKitClientConfig{
@@ -1232,20 +1201,16 @@ func TestPromptKitClientTranslatesBackendCapacityExhaustion(t *testing.T) {
}
client, err := NewPromptKitClient(PromptKitClientConfig{
Assets: newTestPromptKitAssets(t),
EngineOptions: []promptkit.Option{
promptkit.WithBackend(promptkit.Backend{
ID: "limited-backend",
Endpoint: "http://127.0.0.1:1/v1",
ConcurrencyLimit: 1,
QueueCapacity: &queueCapacity,
}),
promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "limited-profile",
BackendID: "limited-backend",
Model: "limited-model",
})),
promptkit.WithLLMClient(fake),
},
Sources: promptKitTestSources(
map[string]promptkit.BackendConfig{"limited-backend": {
Endpoint: promptKitTestString("http://127.0.0.1:1/v1"),
ConcurrencyLimit: promptKitTestInt(1), QueueCapacity: &queueCapacity,
}},
map[string]promptkit.ProfileConfig{"limited-profile": {
Backend: promptKitTestString("limited-backend"), Model: promptKitTestString("limited-model"),
}},
),
EngineOptions: []promptkit.Option{promptkit.WithLLMClient(fake)},
})
if err != nil {
t.Fatalf("NewPromptKitClient() error = %v, want nil", err)
@@ -1364,13 +1329,9 @@ func TestPromptKitClientForwardsConfiguredTransportTimeout(t *testing.T) {
Assets: newTestPromptKitAssets(t),
Timeout: configuredTimeout,
HTTPClient: &http.Client{Transport: transport},
EngineOptions: []promptkit.Option{
promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "default-profile",
Endpoint: "http://promptkit.test/v1",
Model: "default-model",
})),
},
Sources: promptKitTestSources(nil, map[string]promptkit.ProfileConfig{"default-profile": {
Endpoint: promptKitTestString("http://promptkit.test/v1"), Model: promptKitTestString("default-model"),
}}),
})
if err != nil {
t.Fatalf("NewPromptKitClient() error = %v", err)
@@ -1415,9 +1376,9 @@ func TestPromptKitClientClassifiesBuiltInTransportAndMalformedResponses(t *testi
client, err := NewPromptKitClient(PromptKitClientConfig{
Assets: newTestPromptKitAssets(t),
HTTPClient: &http.Client{Transport: roundTripFunc(test.roundTrip)},
EngineOptions: []promptkit.Option{promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "default-profile", Endpoint: "http://promptkit.test/v1", Model: "default-model",
}))},
Sources: promptKitTestSources(nil, map[string]promptkit.ProfileConfig{"default-profile": {
Endpoint: promptKitTestString("http://promptkit.test/v1"), Model: promptKitTestString("default-model"),
}}),
})
if err != nil {
t.Fatal(err)
@@ -1571,27 +1532,20 @@ func newTestPromptKitClientWithReasoning(t *testing.T, fake *fakePromptKitLLM, r
client, err := NewPromptKitClient(PromptKitClientConfig{
Assets: registry,
ReasoningEffort: reasoningEffort,
EngineOptions: []promptkit.Option{
promptkit.WithBackend(promptkit.Backend{
ID: "test-backend",
Endpoint: "http://127.0.0.1:1/v1",
}),
promptkit.WithProfiles(
promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "default-profile",
Endpoint: "http://127.0.0.1:1/v1",
Model: "default-model",
ReasoningEffort: "profile-reasoning",
}),
promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "explicit-profile",
BackendID: "test-backend",
Model: "explicit-model",
ReasoningEffort: "profile-reasoning",
}),
),
promptkit.WithLLMClient(fake),
},
Sources: promptKitTestSources(
map[string]promptkit.BackendConfig{"test-backend": {Endpoint: promptKitTestString("http://127.0.0.1:1/v1")}},
map[string]promptkit.ProfileConfig{
"default-profile": {
Endpoint: promptKitTestString("http://127.0.0.1:1/v1"),
Model: promptKitTestString("default-model"), ReasoningEffort: promptKitTestString("profile-reasoning"),
},
"explicit-profile": {
Backend: promptKitTestString("test-backend"),
Model: promptKitTestString("explicit-model"), ReasoningEffort: promptKitTestString("profile-reasoning"),
},
},
),
EngineOptions: []promptkit.Option{promptkit.WithLLMClient(fake)},
})
if err != nil {
t.Fatalf("NewPromptKitClient() error = %v, want nil", err)
@@ -1603,38 +1557,40 @@ func newTestPromptKitAssets(t *testing.T) *AssetRegistry {
t.Helper()
registry := NewAssetRegistry()
if err := registry.RegisterPromptFS(fstest.MapFS{
"adapter.test.yaml": {Data: []byte(`id: adapter.test
version: "v1"
default_profile: default-profile
session_id: "{{ .session_id }}"
inputs:
- name: transcript
required: true
content_type: application/json
messages:
- role: user
content: "Transcript: {{ input \"transcript\" }} Custom: {{ index . \"custom\" }} Session: {{ .session_id }}"
output:
format: json
validation_mode: json_schema
schema_path: adapter.schema.json
repair_attempts: 0
"adapter.test.yaml": {Data: []byte(`prompts:
adapter.test:
v1:
default_profile: default-profile
session_id: "{{ .session_id }}"
inputs:
- name: transcript
required: true
content_type: application/json
messages:
- role: user
content: "Transcript: {{ input \"transcript\" }} Custom: {{ index . \"custom\" }} Session: {{ .session_id }}"
output:
format: json
validation_mode: json_schema
schema_path: adapter.schema.json
repair_attempts: 0
`)},
"adapter.direct-session.yaml": {Data: []byte(`id: adapter.direct-session
version: "v1"
default_profile: default-profile
inputs:
- name: transcript
required: true
content_type: application/json
messages:
- role: user
content: "Transcript: {{ input \"transcript\" }} Custom: {{ index . \"custom\" }}"
output:
format: json
validation_mode: json_schema
schema_path: adapter.schema.json
repair_attempts: 0
"adapter.direct-session.yaml": {Data: []byte(`prompts:
adapter.direct-session:
v1:
default_profile: default-profile
inputs:
- name: transcript
required: true
content_type: application/json
messages:
- role: user
content: "Transcript: {{ input \"transcript\" }} Custom: {{ index . \"custom\" }}"
output:
format: json
validation_mode: json_schema
schema_path: adapter.schema.json
repair_attempts: 0
`)},
}, "."); err != nil {
t.Fatalf("RegisterPromptFS() error = %v", err)
@@ -1647,61 +1603,6 @@ output:
return registry
}
type switchingPromptFS struct {
mu sync.Mutex
files fstest.MapFS
pauseNextOpen bool
opened chan struct{}
resume chan struct{}
}
func (f *switchingPromptFS) Open(name string) (fs.File, error) {
f.mu.Lock()
file, err := f.files.Open(name)
pause := f.pauseNextOpen && name == "snapshot.test.yaml"
resume := f.resume
if pause {
f.pauseNextOpen = false
close(f.opened)
}
f.mu.Unlock()
if pause {
<-resume
}
return file, err
}
func (f *switchingPromptFS) ReadDir(name string) ([]fs.DirEntry, error) {
f.mu.Lock()
defer f.mu.Unlock()
return f.files.ReadDir(name)
}
func (f *switchingPromptFS) holdNextPromptRead() <-chan struct{} {
f.mu.Lock()
defer f.mu.Unlock()
f.pauseNextOpen = true
f.opened = make(chan struct{})
f.resume = make(chan struct{})
return f.opened
}
func (f *switchingPromptFS) replacePrompt(content []byte) {
f.mu.Lock()
defer f.mu.Unlock()
f.files["snapshot.test.yaml"] = &fstest.MapFile{Data: append([]byte(nil), content...)}
}
func (f *switchingPromptFS) resumePromptRead() {
f.mu.Lock()
resume := f.resume
f.resume = nil
f.mu.Unlock()
if resume != nil {
close(resume)
}
}
type fakePromptKitLLM struct {
content string
responses []promptkit.GenerateResponse

View File

@@ -23,9 +23,9 @@ func newObservedHTTPClient(t *testing.T, handler http.HandlerFunc, retries int,
client, err := NewPromptKitClient(PromptKitClientConfig{
Assets: newTestPromptKitAssets(t), TransientRetries: &retries,
HTTPClient: &http.Client{Transport: transport},
EngineOptions: []promptkit.Option{promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "default-profile", Endpoint: server.URL + "/v1", Model: "test-model",
}))},
Sources: promptKitTestSources(nil, map[string]promptkit.ProfileConfig{"default-profile": {
Endpoint: promptKitTestString(server.URL + "/v1"), Model: promptKitTestString("test-model"),
}}),
})
if err != nil {
t.Fatal(err)

View File

@@ -17,8 +17,9 @@ type PromptKitProfileSourceConfig struct {
}
type PromptKitProfileInspectorConfig struct {
Source PromptKitProfileSourceConfig
Assets *AssetRegistry
Source PromptKitProfileSourceConfig
Assets *AssetRegistry
Sources promptkit.ConfigurationSources
}
func (c PromptKitProfileSourceConfig) localEndpoint() string {
@@ -75,7 +76,7 @@ func (e *promptKitProfileConfigurationError) Unwrap() error {
}
func NewPromptKitProfileInspector(cfg PromptKitProfileInspectorConfig) (*PromptKitProfileInspector, error) {
_, sources, options, err := promptKitEngineConfiguration(cfg.Source, cfg.Assets)
_, sources, options, err := promptKitEngineConfiguration(cfg.Source, cfg.Assets, cfg.Sources)
if err != nil {
return nil, &promptKitProfileConfigurationError{err: err}
}
@@ -112,7 +113,7 @@ func (i *PromptKitProfileInspector) InspectProfile(ctx context.Context, profileI
}, nil
}
func promptKitEngineConfiguration(cfg PromptKitProfileSourceConfig, assets *AssetRegistry) (PromptKitProfileSourceConfig, promptkit.ConfigurationSources, []promptkit.Option, error) {
func promptKitEngineConfiguration(cfg PromptKitProfileSourceConfig, assets *AssetRegistry, configuredSources promptkit.ConfigurationSources) (PromptKitProfileSourceConfig, promptkit.ConfigurationSources, []promptkit.Option, error) {
profileSource, profileSources, err := promptKitProfileConfigurationSources(cfg)
if err != nil {
return PromptKitProfileSourceConfig{}, promptkit.ConfigurationSources{}, nil, err
@@ -127,7 +128,9 @@ func promptKitEngineConfiguration(cfg PromptKitProfileSourceConfig, assets *Asse
sources = maintainedSources
options = append(options, maintainedOptions...)
}
sources.Underlays = append(sources.Underlays, configuredSources.Underlays...)
sources.Underlays = append(sources.Underlays, profileSources.Underlays...)
sources.Main = configuredSources.Main
return profileSource, sources, options, nil
}

View File

@@ -0,0 +1,43 @@
package llm
import (
"testing"
"testing/fstest"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/promptkit"
)
func promptKitTestString(value string) *string { return &value }
func promptKitTestInt(value int) *int { return &value }
func promptKitTestFloat(value float64) *float64 { return &value }
func promptKitTestInputs(t *testing.T, label string, files fstest.MapFS, root string) []configkit.Input {
t.Helper()
inputs, err := promptkit.FilesystemInputs(label, files, root)
if err != nil {
t.Fatalf("FilesystemInputs() error = %v", err)
}
return inputs
}
func promptKitTestSources(
backends map[string]promptkit.BackendConfig,
profiles map[string]promptkit.ProfileConfig,
otherInputs ...configkit.Input,
) promptkit.ConfigurationSources {
inputs := make([]configkit.Input, 0, len(otherInputs)+2)
if len(backends) != 0 {
inputs = append(inputs, configkit.Input{Source: promptkit.BackendSource("test backends", backends)})
}
if len(profiles) != 0 {
inputs = append(inputs, configkit.Input{Source: promptkit.ProfileSource("test profiles", profiles)})
}
inputs = append(inputs, otherInputs...)
if len(inputs) == 0 {
return promptkit.ConfigurationSources{}
}
return promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: inputs}}}
}