Migrate maintained PromptKit definitions and fixtures
This commit is contained in:
@@ -18,11 +18,11 @@ import (
|
||||
func TestFrameworkSourceGroupsComposeAndValidateEffectiveDefinitions(t *testing.T) {
|
||||
conflicting := t.TempDir()
|
||||
writeFrameworkSourceFile(t, conflicting, "first.yml", `profiles:
|
||||
selected:
|
||||
- id: selected
|
||||
model: first
|
||||
`)
|
||||
writeFrameworkSourceFile(t, conflicting, "second.yml", `profiles:
|
||||
selected:
|
||||
- id: selected
|
||||
model: second
|
||||
`)
|
||||
if _, err := newEngine(engineSettings{promptkitSources: []appconfig.SourceSelection{{Directory: conflicting}}}); err == nil {
|
||||
@@ -33,13 +33,13 @@ func TestFrameworkSourceGroupsComposeAndValidateEffectiveDefinitions(t *testing.
|
||||
upper := t.TempDir()
|
||||
empty := t.TempDir()
|
||||
writeFrameworkSourceFile(t, lower, "lower.yml", `profiles:
|
||||
selected:
|
||||
- id: selected
|
||||
endpoint: not-a-url
|
||||
model: lower-model
|
||||
temperature: 10
|
||||
`)
|
||||
writeFrameworkSourceFile(t, upper, "upper.yml", `profiles:
|
||||
selected:
|
||||
- id: selected
|
||||
endpoint: http://127.0.0.1:1/v1
|
||||
model: upper-model
|
||||
temperature: 0.5
|
||||
@@ -60,26 +60,25 @@ func TestFrameworkSourceConstructionUsesCombinedDocumentAndRejectsInvalidSources
|
||||
root := t.TempDir()
|
||||
combined := filepath.Join(root, "combined.yml")
|
||||
writeFrameworkSourceFile(t, root, "combined.yml", `backends:
|
||||
local:
|
||||
- id: local
|
||||
endpoint: http://127.0.0.1:1/v1
|
||||
profiles:
|
||||
selected:
|
||||
- id: selected
|
||||
backend: local
|
||||
model: combined-model
|
||||
prompts:
|
||||
combined.prompt:
|
||||
"1":
|
||||
default_profile: selected
|
||||
messages:
|
||||
- role: user
|
||||
content: hello
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
- id: combined.prompt
|
||||
default_profile: selected
|
||||
messages:
|
||||
- role: user
|
||||
content: hello
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)
|
||||
client := &frameworkSourceCaptureClient{}
|
||||
engine := newFrameworkSourceTestEngine(t, []appconfig.SourceSelection{{File: combined}}, "", promptkit.WithLLMClient(client))
|
||||
if _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "combined.prompt", PromptVersion: "1"}); err != nil {
|
||||
if _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "combined.prompt"}); err != nil {
|
||||
t.Fatalf("run definitions from one combined document: %v", err)
|
||||
}
|
||||
if len(client.requests) != 1 || client.requests[0].Target.BackendID != "local" || client.requests[0].Target.Model != "combined-model" {
|
||||
@@ -94,17 +93,33 @@ prompts:
|
||||
|
||||
invalidUnused := filepath.Join(root, "invalid-unused.yml")
|
||||
writeFrameworkSourceFile(t, root, "invalid-unused.yml", `prompts:
|
||||
unused.prompt:
|
||||
- id: unused.prompt
|
||||
messages:
|
||||
- role: user
|
||||
content: "{{"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)
|
||||
if _, err := newEngine(engineSettings{promptkitSources: []appconfig.SourceSelection{{File: invalidUnused}}}); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||
t.Fatalf("invalid unused definition did not fail as framework configuration: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFrameworkRejectsObsoleteVersionKeyedPrompt(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
path := writeFrameworkSourceFile(t, root, "obsolete.yml", `prompts:
|
||||
old-prompt:
|
||||
"1":
|
||||
messages:
|
||||
- role: user
|
||||
content: "{{"
|
||||
content: hello
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)
|
||||
if _, err := newEngine(engineSettings{promptkitSources: []appconfig.SourceSelection{{File: invalidUnused}}}); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||
t.Fatalf("invalid unused definition did not fail as framework configuration: %v", err)
|
||||
if _, err := newEngine(engineSettings{promptkitSources: []appconfig.SourceSelection{{File: path}}}); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||
t.Fatalf("obsolete version-keyed definition was not rejected by PromptKit: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -119,34 +134,30 @@ func TestFrameworkImportsRetainResourceOriginsAndCaptureAtConstruction(t *testin
|
||||
t.Fatal(err)
|
||||
}
|
||||
mainFile := filepath.Join(root, "main.yml")
|
||||
writeFrameworkSourceFile(t, root, "main.yml", `configkit:
|
||||
writeFrameworkSourceFile(t, root, "main.yml", `config:
|
||||
imports:
|
||||
- path: catalog/base.yml
|
||||
prompts:
|
||||
" team/prompt ":
|
||||
" derived/v2 ":
|
||||
extends: " base/v1 "
|
||||
required: true
|
||||
`)
|
||||
writeFrameworkSourceFile(t, catalogDir, "base.yml", `profiles:
|
||||
" profile/primary ":
|
||||
- id: profile-primary
|
||||
endpoint: http://127.0.0.1:1/v1
|
||||
model: imported-model
|
||||
prompts:
|
||||
" team/prompt ":
|
||||
" base/v1 ":
|
||||
default_profile: " profile/primary "
|
||||
inputs:
|
||||
- name: source
|
||||
required: true
|
||||
messages:
|
||||
- role: user
|
||||
content_file: ./body.md
|
||||
- role: user
|
||||
content: 'Artifact: {{input "source"}}'
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: response.schema.json
|
||||
- id: team-prompt
|
||||
default_profile: profile-primary
|
||||
inputs:
|
||||
- name: source
|
||||
required: true
|
||||
messages:
|
||||
- role: user
|
||||
content_file: ./body.md
|
||||
- role: user
|
||||
content: 'Artifact: {{input "source"}}'
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: response.schema.json
|
||||
`)
|
||||
writeFrameworkSourceFile(t, catalogDir, "body.md", "Captured message for {{.audience}}")
|
||||
artifactPath := filepath.Join(root, "request-artifact.txt")
|
||||
@@ -159,9 +170,9 @@ prompts:
|
||||
writeFrameworkSourceFile(t, root, "request-artifact.txt", "prepared artifact")
|
||||
writeFrameworkSourceFile(t, schemaDir, "response.schema.json", `{"type":"object","title":"prepared schema"}`)
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: " team/prompt ", PromptVersion: " derived/v2 ",
|
||||
Inputs: map[string]promptkit.ArtifactRef{"source": promptkit.File(artifactPath)},
|
||||
Vars: map[string]string{"audience": "reader"},
|
||||
PromptID: "team-prompt",
|
||||
Inputs: map[string]promptkit.ArtifactRef{"source": promptkit.File(artifactPath)},
|
||||
Vars: map[string]string{"audience": "reader"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare inherited imported prompt: %v", err)
|
||||
@@ -169,8 +180,8 @@ prompts:
|
||||
if len(prepared.Messages) != 2 || prepared.Messages[0].Content != "Captured message for reader" || prepared.Messages[1].Content != "Artifact: prepared artifact" {
|
||||
t.Fatalf("resource origin or capture/preparation behavior changed: %+v", prepared.Messages)
|
||||
}
|
||||
if prepared.SelectedProfileID != " profile/primary " || prepared.PromptVersion != " derived/v2 " {
|
||||
t.Fatalf("literal profile/version keys were not preserved: %+v", prepared)
|
||||
if prepared.SelectedProfileID != "profile-primary" {
|
||||
t.Fatalf("profile ID was not preserved: %+v", prepared)
|
||||
}
|
||||
if prepared.StructuredOutput == nil || prepared.StructuredOutput.JSONSchema == nil || prepared.StructuredOutput.JSONSchema.Schema.(map[string]any)["title"] != "prepared schema" {
|
||||
t.Fatalf("schema was not loaded during preparation: %+v", prepared.StructuredOutput)
|
||||
@@ -178,9 +189,9 @@ prompts:
|
||||
|
||||
newEngine := newFrameworkSourceTestEngine(t, selections, schemaDir)
|
||||
newPrepared, err := newEngine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: " team/prompt ", PromptVersion: " derived/v2 ",
|
||||
Inputs: map[string]promptkit.ArtifactRef{"source": promptkit.File(artifactPath)},
|
||||
Vars: map[string]string{"audience": "reader"},
|
||||
PromptID: "team-prompt",
|
||||
Inputs: map[string]promptkit.ArtifactRef{"source": promptkit.File(artifactPath)},
|
||||
Vars: map[string]string{"audience": "reader"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare with newly captured message: %v", err)
|
||||
@@ -192,8 +203,7 @@ prompts:
|
||||
configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\nschema_dir: %q\n", mainFile, schemaDir))
|
||||
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||
"--config", configPath,
|
||||
"--prompt", " team/prompt ",
|
||||
"--prompt-version", " derived/v2 ",
|
||||
"--prompt", "team-prompt",
|
||||
"--input", "source=" + artifactPath,
|
||||
"--var", "audience=reader",
|
||||
"--format", "json",
|
||||
@@ -205,8 +215,8 @@ prompts:
|
||||
if err := json.Unmarshal([]byte(stdout), &rendered); err != nil {
|
||||
t.Fatalf("decode rendered literal-ID fixture: %v", err)
|
||||
}
|
||||
if rendered.PromptID != " team/prompt " || rendered.PromptVersion != " derived/v2 " || rendered.SelectedProfileID != " profile/primary " {
|
||||
t.Fatalf("CLI changed exact prompt/profile/version keys: %+v", rendered)
|
||||
if rendered.PromptID != "team-prompt" || rendered.SelectedProfileID != "profile-primary" {
|
||||
t.Fatalf("CLI changed exact prompt/profile IDs: %+v", rendered)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -214,14 +224,14 @@ func TestNativeProfileInheritanceAndPresenceReachTheClientBoundary(t *testing.T)
|
||||
root := t.TempDir()
|
||||
frameworkFile := filepath.Join(root, "framework.yml")
|
||||
writeFrameworkSourceFile(t, root, "framework.yml", `backends:
|
||||
first:
|
||||
- id: first
|
||||
endpoint: https://first.example/v1
|
||||
api_key_env: FIRST_BACKEND_KEY
|
||||
extra_params:
|
||||
source: first
|
||||
nested:
|
||||
first_backend: true
|
||||
second:
|
||||
- id: second
|
||||
endpoint: https://second.example/v1
|
||||
api_key_env: SECOND_BACKEND_KEY
|
||||
extra_params:
|
||||
@@ -229,7 +239,7 @@ func TestNativeProfileInheritanceAndPresenceReachTheClientBoundary(t *testing.T)
|
||||
nested:
|
||||
second_backend: true
|
||||
profiles:
|
||||
parent:
|
||||
- id: parent
|
||||
backend: first
|
||||
model: parent-model
|
||||
temperature: 0.7
|
||||
@@ -241,12 +251,12 @@ profiles:
|
||||
nested:
|
||||
parent_profile: true
|
||||
retained: "yes"
|
||||
backend-child:
|
||||
extends: parent
|
||||
- id: backend-child
|
||||
extends: {id: parent}
|
||||
backend: second
|
||||
model: child-model
|
||||
presence-child:
|
||||
extends: parent
|
||||
- id: presence-child
|
||||
extends: {id: parent}
|
||||
backend: second
|
||||
api_key_env: null
|
||||
temperature: null
|
||||
@@ -255,29 +265,28 @@ profiles:
|
||||
extra_params:
|
||||
nested:
|
||||
explicit_false: false
|
||||
map-null:
|
||||
- id: map-null
|
||||
backend: second
|
||||
model: map-model
|
||||
extra_params: null
|
||||
map-empty:
|
||||
- id: map-empty
|
||||
backend: second
|
||||
model: map-model
|
||||
extra_params: {}
|
||||
prompts:
|
||||
execution.prompt:
|
||||
"1":
|
||||
messages:
|
||||
- role: user
|
||||
content: hello
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
- id: execution.prompt
|
||||
messages:
|
||||
- role: user
|
||||
content: hello
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)
|
||||
client := &frameworkSourceCaptureClient{}
|
||||
engine := newFrameworkSourceTestEngine(t, []appconfig.SourceSelection{{File: frameworkFile}}, "", promptkit.WithLLMClient(client))
|
||||
|
||||
for _, profileID := range []string{"backend-child", "presence-child", "map-null", "map-empty"} {
|
||||
if _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "execution.prompt", PromptVersion: "1", ProfileID: profileID}); err != nil {
|
||||
if _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "execution.prompt", ProfileID: profileID}); err != nil {
|
||||
t.Fatalf("run profile %q: %v", profileID, err)
|
||||
}
|
||||
}
|
||||
@@ -296,7 +305,7 @@ prompts:
|
||||
|
||||
presenceRequest := client.requests[1]
|
||||
presenceTarget := presenceRequest.Target
|
||||
if presenceTarget.APIKeyEnv != "" || presenceTarget.Temperature != 0 || presenceTarget.MaxTokens != 0 || presenceTarget.TimeoutSeconds != 0 ||
|
||||
if presenceTarget.APIKeyEnv != "SECOND_BACKEND_KEY" || presenceTarget.Temperature != 0 || presenceTarget.MaxTokens != 0 || presenceTarget.TimeoutSeconds != 0 ||
|
||||
!presenceRequest.TargetPresence.MaxTokens || !presenceRequest.TargetPresence.TimeoutSeconds || presenceRequest.TargetPresence.Temperature || !presenceRequest.TargetPresence.TopP {
|
||||
t.Fatalf("explicit null/zero presence did not reach the injected client: target=%+v presence=%+v", presenceTarget, presenceRequest.TargetPresence)
|
||||
}
|
||||
@@ -306,8 +315,8 @@ prompts:
|
||||
if !reflect.DeepEqual(presenceTarget.ExtraParams, wantPresenceParams) {
|
||||
t.Fatalf("recursive provider-map retention changed: %#v", presenceTarget.ExtraParams)
|
||||
}
|
||||
if client.requests[2].Target.ExtraParams != nil {
|
||||
t.Fatalf("whole-map null did not suppress backend defaults: %#v", client.requests[2].Target.ExtraParams)
|
||||
if !reflect.DeepEqual(client.requests[2].Target.ExtraParams, map[string]any{"source": "second", "nested": map[string]any{"second_backend": true}}) {
|
||||
t.Fatalf("whole-map null did not fall back to backend defaults: %#v", client.requests[2].Target.ExtraParams)
|
||||
}
|
||||
if !reflect.DeepEqual(client.requests[3].Target.ExtraParams, map[string]any{"source": "second", "nested": map[string]any{"second_backend": true}}) {
|
||||
t.Fatalf("empty provider map unexpectedly cleared backend defaults: %#v", client.requests[3].Target.ExtraParams)
|
||||
@@ -317,17 +326,16 @@ prompts:
|
||||
func TestCLICompletesPartialFrameworkProfileWithRequestOverrides(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
frameworkFile := writeFrameworkSourceFile(t, root, "partial.yml", `profiles:
|
||||
partial: {}
|
||||
- id: partial
|
||||
prompts:
|
||||
partial.prompt:
|
||||
"1":
|
||||
default_profile: partial
|
||||
messages:
|
||||
- role: user
|
||||
content: hello
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
- id: partial.prompt
|
||||
default_profile: partial
|
||||
messages:
|
||||
- role: user
|
||||
content: hello
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)
|
||||
provider := newTestLLMServer("completed partial profile", nil)
|
||||
defer provider.Close()
|
||||
|
||||
Reference in New Issue
Block a user