Migrate maintained PromptKit definitions and fixtures
This commit is contained in:
@@ -216,6 +216,8 @@ scaffolding, or conversation and coordination features.
|
|||||||
|
|
||||||
## Stage 5: Migrate maintained definitions and existing test fixtures
|
## Stage 5: Migrate maintained definitions and existing test fixtures
|
||||||
|
|
||||||
|
**Status: Complete.**
|
||||||
|
|
||||||
1. Convert selected files under `examples/prompts/`,
|
1. Convert selected files under `examples/prompts/`,
|
||||||
`examples/profiles/`, and `examples/framework/` and inline or
|
`examples/profiles/`, and `examples/framework/` and inline or
|
||||||
on-disk framework fixtures across CLI and HTTP tests. Replace keyed
|
on-disk framework fixtures across CLI and HTTP tests. Replace keyed
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
backends:
|
backends:
|
||||||
local-gpu:
|
- id: local-gpu
|
||||||
endpoint: http://localhost:11434/v1
|
endpoint: http://localhost:11434/v1
|
||||||
api_key_env: LOCAL_GPU_API_KEY
|
api_key_env: LOCAL_GPU_API_KEY
|
||||||
extra_params:
|
extra_params:
|
||||||
|
|||||||
@@ -1,13 +1,13 @@
|
|||||||
backends:
|
backends:
|
||||||
local-base:
|
- id: local-base
|
||||||
endpoint: http://localhost:11434/v1
|
endpoint: http://localhost:11434/v1
|
||||||
api_key_env: LOCAL_GPU_API_KEY
|
api_key_env: LOCAL_GPU_API_KEY
|
||||||
concurrency_limit: 2
|
concurrency_limit: 2
|
||||||
queue_capacity: 4
|
queue_capacity: 4
|
||||||
extra_params:
|
extra_params:
|
||||||
provider_option: true
|
provider_option: true
|
||||||
local-gpu:
|
- id: local-gpu
|
||||||
extends: local-base
|
extends: {id: local-base}
|
||||||
queue_capacity: 0
|
queue_capacity: 0
|
||||||
extra_params:
|
extra_params:
|
||||||
provider_option: false
|
provider_option: false
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
profiles:
|
profiles:
|
||||||
local-fast:
|
- id: local-fast
|
||||||
endpoint: http://localhost:8000/v1
|
endpoint: http://localhost:8000/v1
|
||||||
model: gpt-4o-mini
|
model: gpt-4o-mini
|
||||||
temperature: 0.2
|
temperature: 0.2
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
profiles:
|
profiles:
|
||||||
local-gpu:
|
- id: local-gpu
|
||||||
backend: local-gpu
|
backend: local-gpu
|
||||||
model: local-model
|
model: local-model
|
||||||
reasoning_effort: low
|
reasoning_effort: low
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
profiles:
|
profiles:
|
||||||
local-quality:
|
- id: local-quality
|
||||||
endpoint: http://localhost:8000/v1
|
endpoint: http://localhost:8000/v1
|
||||||
model: gpt-4.1
|
model: gpt-4.1
|
||||||
temperature: 0.0
|
temperature: 0.0
|
||||||
|
|||||||
@@ -1,23 +1,22 @@
|
|||||||
prompts:
|
prompts:
|
||||||
dnd.session_recap:
|
- id: dnd.session_recap
|
||||||
"1.0.0":
|
default_profile: local-quality
|
||||||
default_profile: local-quality
|
description: Example D&D session recap prompt definition for demonstration only.
|
||||||
description: Example D&D session recap prompt definition for demonstration only.
|
inputs:
|
||||||
inputs:
|
- name: transcript
|
||||||
- name: transcript
|
required: true
|
||||||
required: true
|
content_type: text/markdown
|
||||||
content_type: text/markdown
|
description: Session source content
|
||||||
description: Session source content
|
- name: glossary
|
||||||
- name: glossary
|
required: false
|
||||||
required: false
|
content_type: text/yaml
|
||||||
content_type: text/yaml
|
description: Optional glossary context
|
||||||
description: Optional glossary context
|
messages:
|
||||||
messages:
|
- role: system
|
||||||
- role: system
|
content_file: ./dnd.session_recap.system.md
|
||||||
content_file: ./dnd.session_recap.system.md
|
- role: user
|
||||||
- role: user
|
content_file: ./dnd.session_recap.user.md
|
||||||
content_file: ./dnd.session_recap.user.md
|
output:
|
||||||
output:
|
format: markdown
|
||||||
format: markdown
|
validation_mode: basic
|
||||||
validation_mode: basic
|
repair_attempts: 0
|
||||||
repair_attempts: 0
|
|
||||||
|
|||||||
@@ -1,23 +1,22 @@
|
|||||||
prompts:
|
prompts:
|
||||||
generic.markdown_summary:
|
- id: generic.markdown_summary
|
||||||
"1.0.0":
|
default_profile: local-fast
|
||||||
default_profile: local-fast
|
description: Generic markdown summary from transcript and glossary.
|
||||||
description: Generic markdown summary from transcript and glossary.
|
inputs:
|
||||||
inputs:
|
- name: transcript
|
||||||
- name: transcript
|
required: true
|
||||||
required: true
|
content_type: text/markdown
|
||||||
content_type: text/markdown
|
description: Source transcript content
|
||||||
description: Source transcript content
|
- name: glossary
|
||||||
- name: glossary
|
required: false
|
||||||
required: false
|
content_type: text/yaml
|
||||||
content_type: text/yaml
|
description: Optional glossary context
|
||||||
description: Optional glossary context
|
messages:
|
||||||
messages:
|
- role: system
|
||||||
- role: system
|
content_file: ./generic.markdown_summary.system.md
|
||||||
content_file: ./generic.markdown_summary.system.md
|
- role: user
|
||||||
- role: user
|
content_file: ./generic.markdown_summary.user.md
|
||||||
content_file: ./generic.markdown_summary.user.md
|
output:
|
||||||
output:
|
format: markdown
|
||||||
format: markdown
|
validation_mode: basic
|
||||||
validation_mode: basic
|
repair_attempts: 0
|
||||||
repair_attempts: 0
|
|
||||||
|
|||||||
@@ -1,24 +1,23 @@
|
|||||||
prompts:
|
prompts:
|
||||||
generic.structured_events:
|
- id: generic.structured_events
|
||||||
"1.0.0":
|
default_profile: local-quality
|
||||||
default_profile: local-quality
|
description: Produce structured event JSON from a transcript.
|
||||||
description: Produce structured event JSON from a transcript.
|
inputs:
|
||||||
inputs:
|
- name: transcript
|
||||||
- name: transcript
|
required: true
|
||||||
required: true
|
content_type: text/markdown
|
||||||
content_type: text/markdown
|
description: Source transcript content
|
||||||
description: Source transcript content
|
- name: glossary
|
||||||
- name: glossary
|
required: false
|
||||||
required: false
|
content_type: text/yaml
|
||||||
content_type: text/yaml
|
description: Optional glossary context
|
||||||
description: Optional glossary context
|
messages:
|
||||||
messages:
|
- role: system
|
||||||
- role: system
|
content_file: ./generic.structured_events.system.md
|
||||||
content_file: ./generic.structured_events.system.md
|
- role: user
|
||||||
- role: user
|
content_file: ./generic.structured_events.user.md
|
||||||
content_file: ./generic.structured_events.user.md
|
output:
|
||||||
output:
|
format: json
|
||||||
format: json
|
validation_mode: json_schema
|
||||||
validation_mode: json_schema
|
schema_path: structured_events.schema.json
|
||||||
schema_path: structured_events.schema.json
|
repair_attempts: 0
|
||||||
repair_attempts: 0
|
|
||||||
|
|||||||
@@ -7,7 +7,6 @@ cd "$repo_root"
|
|||||||
go run ./cmd/scriptorium render \
|
go run ./cmd/scriptorium render \
|
||||||
--config ./examples/config.full.yml \
|
--config ./examples/config.full.yml \
|
||||||
--prompt generic.markdown_summary \
|
--prompt generic.markdown_summary \
|
||||||
--prompt-version 1.0.0 \
|
|
||||||
--profile local-gpu \
|
--profile local-gpu \
|
||||||
--session-id example-session \
|
--session-id example-session \
|
||||||
--reasoning-effort= \
|
--reasoning-effort= \
|
||||||
|
|||||||
@@ -200,7 +200,7 @@ func TestFrameworkConfigurationExplainUsesLiteralJSONPointersAndNativeHistory(t
|
|||||||
provider.option: dotted-visible-value
|
provider.option: dotted-visible-value
|
||||||
profiles:
|
profiles:
|
||||||
- id: summary
|
- id: summary
|
||||||
backend: local
|
backend: support/local
|
||||||
model: lower-model
|
model: lower-model
|
||||||
api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE
|
api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE
|
||||||
extra_params:
|
extra_params:
|
||||||
@@ -232,7 +232,7 @@ func TestFrameworkConfigurationExplainUsesLiteralJSONPointersAndNativeHistory(t
|
|||||||
provider.option: dotted-visible-value
|
provider.option: dotted-visible-value
|
||||||
profiles:
|
profiles:
|
||||||
- id: summary
|
- id: summary
|
||||||
backend: local
|
backend: support/local
|
||||||
model: effective-model
|
model: effective-model
|
||||||
api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE
|
api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE
|
||||||
extra_params:
|
extra_params:
|
||||||
|
|||||||
@@ -18,11 +18,11 @@ import (
|
|||||||
func TestFrameworkSourceGroupsComposeAndValidateEffectiveDefinitions(t *testing.T) {
|
func TestFrameworkSourceGroupsComposeAndValidateEffectiveDefinitions(t *testing.T) {
|
||||||
conflicting := t.TempDir()
|
conflicting := t.TempDir()
|
||||||
writeFrameworkSourceFile(t, conflicting, "first.yml", `profiles:
|
writeFrameworkSourceFile(t, conflicting, "first.yml", `profiles:
|
||||||
selected:
|
- id: selected
|
||||||
model: first
|
model: first
|
||||||
`)
|
`)
|
||||||
writeFrameworkSourceFile(t, conflicting, "second.yml", `profiles:
|
writeFrameworkSourceFile(t, conflicting, "second.yml", `profiles:
|
||||||
selected:
|
- id: selected
|
||||||
model: second
|
model: second
|
||||||
`)
|
`)
|
||||||
if _, err := newEngine(engineSettings{promptkitSources: []appconfig.SourceSelection{{Directory: conflicting}}}); err == nil {
|
if _, err := newEngine(engineSettings{promptkitSources: []appconfig.SourceSelection{{Directory: conflicting}}}); err == nil {
|
||||||
@@ -33,13 +33,13 @@ func TestFrameworkSourceGroupsComposeAndValidateEffectiveDefinitions(t *testing.
|
|||||||
upper := t.TempDir()
|
upper := t.TempDir()
|
||||||
empty := t.TempDir()
|
empty := t.TempDir()
|
||||||
writeFrameworkSourceFile(t, lower, "lower.yml", `profiles:
|
writeFrameworkSourceFile(t, lower, "lower.yml", `profiles:
|
||||||
selected:
|
- id: selected
|
||||||
endpoint: not-a-url
|
endpoint: not-a-url
|
||||||
model: lower-model
|
model: lower-model
|
||||||
temperature: 10
|
temperature: 10
|
||||||
`)
|
`)
|
||||||
writeFrameworkSourceFile(t, upper, "upper.yml", `profiles:
|
writeFrameworkSourceFile(t, upper, "upper.yml", `profiles:
|
||||||
selected:
|
- id: selected
|
||||||
endpoint: http://127.0.0.1:1/v1
|
endpoint: http://127.0.0.1:1/v1
|
||||||
model: upper-model
|
model: upper-model
|
||||||
temperature: 0.5
|
temperature: 0.5
|
||||||
@@ -60,26 +60,25 @@ func TestFrameworkSourceConstructionUsesCombinedDocumentAndRejectsInvalidSources
|
|||||||
root := t.TempDir()
|
root := t.TempDir()
|
||||||
combined := filepath.Join(root, "combined.yml")
|
combined := filepath.Join(root, "combined.yml")
|
||||||
writeFrameworkSourceFile(t, root, "combined.yml", `backends:
|
writeFrameworkSourceFile(t, root, "combined.yml", `backends:
|
||||||
local:
|
- id: local
|
||||||
endpoint: http://127.0.0.1:1/v1
|
endpoint: http://127.0.0.1:1/v1
|
||||||
profiles:
|
profiles:
|
||||||
selected:
|
- id: selected
|
||||||
backend: local
|
backend: local
|
||||||
model: combined-model
|
model: combined-model
|
||||||
prompts:
|
prompts:
|
||||||
combined.prompt:
|
- id: combined.prompt
|
||||||
"1":
|
default_profile: selected
|
||||||
default_profile: selected
|
messages:
|
||||||
messages:
|
- role: user
|
||||||
- role: user
|
content: hello
|
||||||
content: hello
|
output:
|
||||||
output:
|
format: text
|
||||||
format: text
|
validation_mode: none
|
||||||
validation_mode: none
|
|
||||||
`)
|
`)
|
||||||
client := &frameworkSourceCaptureClient{}
|
client := &frameworkSourceCaptureClient{}
|
||||||
engine := newFrameworkSourceTestEngine(t, []appconfig.SourceSelection{{File: combined}}, "", promptkit.WithLLMClient(client))
|
engine := newFrameworkSourceTestEngine(t, []appconfig.SourceSelection{{File: combined}}, "", promptkit.WithLLMClient(client))
|
||||||
if _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "combined.prompt", PromptVersion: "1"}); err != nil {
|
if _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "combined.prompt"}); err != nil {
|
||||||
t.Fatalf("run definitions from one combined document: %v", err)
|
t.Fatalf("run definitions from one combined document: %v", err)
|
||||||
}
|
}
|
||||||
if len(client.requests) != 1 || client.requests[0].Target.BackendID != "local" || client.requests[0].Target.Model != "combined-model" {
|
if len(client.requests) != 1 || client.requests[0].Target.BackendID != "local" || client.requests[0].Target.Model != "combined-model" {
|
||||||
@@ -94,17 +93,33 @@ prompts:
|
|||||||
|
|
||||||
invalidUnused := filepath.Join(root, "invalid-unused.yml")
|
invalidUnused := filepath.Join(root, "invalid-unused.yml")
|
||||||
writeFrameworkSourceFile(t, root, "invalid-unused.yml", `prompts:
|
writeFrameworkSourceFile(t, root, "invalid-unused.yml", `prompts:
|
||||||
unused.prompt:
|
- id: unused.prompt
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: "{{"
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`)
|
||||||
|
if _, err := newEngine(engineSettings{promptkitSources: []appconfig.SourceSelection{{File: invalidUnused}}}); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||||
|
t.Fatalf("invalid unused definition did not fail as framework configuration: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFrameworkRejectsObsoleteVersionKeyedPrompt(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
path := writeFrameworkSourceFile(t, root, "obsolete.yml", `prompts:
|
||||||
|
old-prompt:
|
||||||
"1":
|
"1":
|
||||||
messages:
|
messages:
|
||||||
- role: user
|
- role: user
|
||||||
content: "{{"
|
content: hello
|
||||||
output:
|
output:
|
||||||
format: text
|
format: text
|
||||||
validation_mode: none
|
validation_mode: none
|
||||||
`)
|
`)
|
||||||
if _, err := newEngine(engineSettings{promptkitSources: []appconfig.SourceSelection{{File: invalidUnused}}}); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
if _, err := newEngine(engineSettings{promptkitSources: []appconfig.SourceSelection{{File: path}}}); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||||
t.Fatalf("invalid unused definition did not fail as framework configuration: %v", err)
|
t.Fatalf("obsolete version-keyed definition was not rejected by PromptKit: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -119,34 +134,30 @@ func TestFrameworkImportsRetainResourceOriginsAndCaptureAtConstruction(t *testin
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
mainFile := filepath.Join(root, "main.yml")
|
mainFile := filepath.Join(root, "main.yml")
|
||||||
writeFrameworkSourceFile(t, root, "main.yml", `configkit:
|
writeFrameworkSourceFile(t, root, "main.yml", `config:
|
||||||
imports:
|
imports:
|
||||||
- path: catalog/base.yml
|
- path: catalog/base.yml
|
||||||
prompts:
|
required: true
|
||||||
" team/prompt ":
|
|
||||||
" derived/v2 ":
|
|
||||||
extends: " base/v1 "
|
|
||||||
`)
|
`)
|
||||||
writeFrameworkSourceFile(t, catalogDir, "base.yml", `profiles:
|
writeFrameworkSourceFile(t, catalogDir, "base.yml", `profiles:
|
||||||
" profile/primary ":
|
- id: profile-primary
|
||||||
endpoint: http://127.0.0.1:1/v1
|
endpoint: http://127.0.0.1:1/v1
|
||||||
model: imported-model
|
model: imported-model
|
||||||
prompts:
|
prompts:
|
||||||
" team/prompt ":
|
- id: team-prompt
|
||||||
" base/v1 ":
|
default_profile: profile-primary
|
||||||
default_profile: " profile/primary "
|
inputs:
|
||||||
inputs:
|
- name: source
|
||||||
- name: source
|
required: true
|
||||||
required: true
|
messages:
|
||||||
messages:
|
- role: user
|
||||||
- role: user
|
content_file: ./body.md
|
||||||
content_file: ./body.md
|
- role: user
|
||||||
- role: user
|
content: 'Artifact: {{input "source"}}'
|
||||||
content: 'Artifact: {{input "source"}}'
|
output:
|
||||||
output:
|
format: json
|
||||||
format: json
|
validation_mode: json_schema
|
||||||
validation_mode: json_schema
|
schema_path: response.schema.json
|
||||||
schema_path: response.schema.json
|
|
||||||
`)
|
`)
|
||||||
writeFrameworkSourceFile(t, catalogDir, "body.md", "Captured message for {{.audience}}")
|
writeFrameworkSourceFile(t, catalogDir, "body.md", "Captured message for {{.audience}}")
|
||||||
artifactPath := filepath.Join(root, "request-artifact.txt")
|
artifactPath := filepath.Join(root, "request-artifact.txt")
|
||||||
@@ -159,9 +170,9 @@ prompts:
|
|||||||
writeFrameworkSourceFile(t, root, "request-artifact.txt", "prepared artifact")
|
writeFrameworkSourceFile(t, root, "request-artifact.txt", "prepared artifact")
|
||||||
writeFrameworkSourceFile(t, schemaDir, "response.schema.json", `{"type":"object","title":"prepared schema"}`)
|
writeFrameworkSourceFile(t, schemaDir, "response.schema.json", `{"type":"object","title":"prepared schema"}`)
|
||||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||||
PromptID: " team/prompt ", PromptVersion: " derived/v2 ",
|
PromptID: "team-prompt",
|
||||||
Inputs: map[string]promptkit.ArtifactRef{"source": promptkit.File(artifactPath)},
|
Inputs: map[string]promptkit.ArtifactRef{"source": promptkit.File(artifactPath)},
|
||||||
Vars: map[string]string{"audience": "reader"},
|
Vars: map[string]string{"audience": "reader"},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("prepare inherited imported prompt: %v", err)
|
t.Fatalf("prepare inherited imported prompt: %v", err)
|
||||||
@@ -169,8 +180,8 @@ prompts:
|
|||||||
if len(prepared.Messages) != 2 || prepared.Messages[0].Content != "Captured message for reader" || prepared.Messages[1].Content != "Artifact: prepared artifact" {
|
if len(prepared.Messages) != 2 || prepared.Messages[0].Content != "Captured message for reader" || prepared.Messages[1].Content != "Artifact: prepared artifact" {
|
||||||
t.Fatalf("resource origin or capture/preparation behavior changed: %+v", prepared.Messages)
|
t.Fatalf("resource origin or capture/preparation behavior changed: %+v", prepared.Messages)
|
||||||
}
|
}
|
||||||
if prepared.SelectedProfileID != " profile/primary " || prepared.PromptVersion != " derived/v2 " {
|
if prepared.SelectedProfileID != "profile-primary" {
|
||||||
t.Fatalf("literal profile/version keys were not preserved: %+v", prepared)
|
t.Fatalf("profile ID was not preserved: %+v", prepared)
|
||||||
}
|
}
|
||||||
if prepared.StructuredOutput == nil || prepared.StructuredOutput.JSONSchema == nil || prepared.StructuredOutput.JSONSchema.Schema.(map[string]any)["title"] != "prepared schema" {
|
if prepared.StructuredOutput == nil || prepared.StructuredOutput.JSONSchema == nil || prepared.StructuredOutput.JSONSchema.Schema.(map[string]any)["title"] != "prepared schema" {
|
||||||
t.Fatalf("schema was not loaded during preparation: %+v", prepared.StructuredOutput)
|
t.Fatalf("schema was not loaded during preparation: %+v", prepared.StructuredOutput)
|
||||||
@@ -178,9 +189,9 @@ prompts:
|
|||||||
|
|
||||||
newEngine := newFrameworkSourceTestEngine(t, selections, schemaDir)
|
newEngine := newFrameworkSourceTestEngine(t, selections, schemaDir)
|
||||||
newPrepared, err := newEngine.Prepare(context.Background(), promptkit.RunRequest{
|
newPrepared, err := newEngine.Prepare(context.Background(), promptkit.RunRequest{
|
||||||
PromptID: " team/prompt ", PromptVersion: " derived/v2 ",
|
PromptID: "team-prompt",
|
||||||
Inputs: map[string]promptkit.ArtifactRef{"source": promptkit.File(artifactPath)},
|
Inputs: map[string]promptkit.ArtifactRef{"source": promptkit.File(artifactPath)},
|
||||||
Vars: map[string]string{"audience": "reader"},
|
Vars: map[string]string{"audience": "reader"},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("prepare with newly captured message: %v", err)
|
t.Fatalf("prepare with newly captured message: %v", err)
|
||||||
@@ -192,8 +203,7 @@ prompts:
|
|||||||
configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\nschema_dir: %q\n", mainFile, schemaDir))
|
configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\nschema_dir: %q\n", mainFile, schemaDir))
|
||||||
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
"--config", configPath,
|
"--config", configPath,
|
||||||
"--prompt", " team/prompt ",
|
"--prompt", "team-prompt",
|
||||||
"--prompt-version", " derived/v2 ",
|
|
||||||
"--input", "source=" + artifactPath,
|
"--input", "source=" + artifactPath,
|
||||||
"--var", "audience=reader",
|
"--var", "audience=reader",
|
||||||
"--format", "json",
|
"--format", "json",
|
||||||
@@ -205,8 +215,8 @@ prompts:
|
|||||||
if err := json.Unmarshal([]byte(stdout), &rendered); err != nil {
|
if err := json.Unmarshal([]byte(stdout), &rendered); err != nil {
|
||||||
t.Fatalf("decode rendered literal-ID fixture: %v", err)
|
t.Fatalf("decode rendered literal-ID fixture: %v", err)
|
||||||
}
|
}
|
||||||
if rendered.PromptID != " team/prompt " || rendered.PromptVersion != " derived/v2 " || rendered.SelectedProfileID != " profile/primary " {
|
if rendered.PromptID != "team-prompt" || rendered.SelectedProfileID != "profile-primary" {
|
||||||
t.Fatalf("CLI changed exact prompt/profile/version keys: %+v", rendered)
|
t.Fatalf("CLI changed exact prompt/profile IDs: %+v", rendered)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -214,14 +224,14 @@ func TestNativeProfileInheritanceAndPresenceReachTheClientBoundary(t *testing.T)
|
|||||||
root := t.TempDir()
|
root := t.TempDir()
|
||||||
frameworkFile := filepath.Join(root, "framework.yml")
|
frameworkFile := filepath.Join(root, "framework.yml")
|
||||||
writeFrameworkSourceFile(t, root, "framework.yml", `backends:
|
writeFrameworkSourceFile(t, root, "framework.yml", `backends:
|
||||||
first:
|
- id: first
|
||||||
endpoint: https://first.example/v1
|
endpoint: https://first.example/v1
|
||||||
api_key_env: FIRST_BACKEND_KEY
|
api_key_env: FIRST_BACKEND_KEY
|
||||||
extra_params:
|
extra_params:
|
||||||
source: first
|
source: first
|
||||||
nested:
|
nested:
|
||||||
first_backend: true
|
first_backend: true
|
||||||
second:
|
- id: second
|
||||||
endpoint: https://second.example/v1
|
endpoint: https://second.example/v1
|
||||||
api_key_env: SECOND_BACKEND_KEY
|
api_key_env: SECOND_BACKEND_KEY
|
||||||
extra_params:
|
extra_params:
|
||||||
@@ -229,7 +239,7 @@ func TestNativeProfileInheritanceAndPresenceReachTheClientBoundary(t *testing.T)
|
|||||||
nested:
|
nested:
|
||||||
second_backend: true
|
second_backend: true
|
||||||
profiles:
|
profiles:
|
||||||
parent:
|
- id: parent
|
||||||
backend: first
|
backend: first
|
||||||
model: parent-model
|
model: parent-model
|
||||||
temperature: 0.7
|
temperature: 0.7
|
||||||
@@ -241,12 +251,12 @@ profiles:
|
|||||||
nested:
|
nested:
|
||||||
parent_profile: true
|
parent_profile: true
|
||||||
retained: "yes"
|
retained: "yes"
|
||||||
backend-child:
|
- id: backend-child
|
||||||
extends: parent
|
extends: {id: parent}
|
||||||
backend: second
|
backend: second
|
||||||
model: child-model
|
model: child-model
|
||||||
presence-child:
|
- id: presence-child
|
||||||
extends: parent
|
extends: {id: parent}
|
||||||
backend: second
|
backend: second
|
||||||
api_key_env: null
|
api_key_env: null
|
||||||
temperature: null
|
temperature: null
|
||||||
@@ -255,29 +265,28 @@ profiles:
|
|||||||
extra_params:
|
extra_params:
|
||||||
nested:
|
nested:
|
||||||
explicit_false: false
|
explicit_false: false
|
||||||
map-null:
|
- id: map-null
|
||||||
backend: second
|
backend: second
|
||||||
model: map-model
|
model: map-model
|
||||||
extra_params: null
|
extra_params: null
|
||||||
map-empty:
|
- id: map-empty
|
||||||
backend: second
|
backend: second
|
||||||
model: map-model
|
model: map-model
|
||||||
extra_params: {}
|
extra_params: {}
|
||||||
prompts:
|
prompts:
|
||||||
execution.prompt:
|
- id: execution.prompt
|
||||||
"1":
|
messages:
|
||||||
messages:
|
- role: user
|
||||||
- role: user
|
content: hello
|
||||||
content: hello
|
output:
|
||||||
output:
|
format: text
|
||||||
format: text
|
validation_mode: none
|
||||||
validation_mode: none
|
|
||||||
`)
|
`)
|
||||||
client := &frameworkSourceCaptureClient{}
|
client := &frameworkSourceCaptureClient{}
|
||||||
engine := newFrameworkSourceTestEngine(t, []appconfig.SourceSelection{{File: frameworkFile}}, "", promptkit.WithLLMClient(client))
|
engine := newFrameworkSourceTestEngine(t, []appconfig.SourceSelection{{File: frameworkFile}}, "", promptkit.WithLLMClient(client))
|
||||||
|
|
||||||
for _, profileID := range []string{"backend-child", "presence-child", "map-null", "map-empty"} {
|
for _, profileID := range []string{"backend-child", "presence-child", "map-null", "map-empty"} {
|
||||||
if _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "execution.prompt", PromptVersion: "1", ProfileID: profileID}); err != nil {
|
if _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "execution.prompt", ProfileID: profileID}); err != nil {
|
||||||
t.Fatalf("run profile %q: %v", profileID, err)
|
t.Fatalf("run profile %q: %v", profileID, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -296,7 +305,7 @@ prompts:
|
|||||||
|
|
||||||
presenceRequest := client.requests[1]
|
presenceRequest := client.requests[1]
|
||||||
presenceTarget := presenceRequest.Target
|
presenceTarget := presenceRequest.Target
|
||||||
if presenceTarget.APIKeyEnv != "" || presenceTarget.Temperature != 0 || presenceTarget.MaxTokens != 0 || presenceTarget.TimeoutSeconds != 0 ||
|
if presenceTarget.APIKeyEnv != "SECOND_BACKEND_KEY" || presenceTarget.Temperature != 0 || presenceTarget.MaxTokens != 0 || presenceTarget.TimeoutSeconds != 0 ||
|
||||||
!presenceRequest.TargetPresence.MaxTokens || !presenceRequest.TargetPresence.TimeoutSeconds || presenceRequest.TargetPresence.Temperature || !presenceRequest.TargetPresence.TopP {
|
!presenceRequest.TargetPresence.MaxTokens || !presenceRequest.TargetPresence.TimeoutSeconds || presenceRequest.TargetPresence.Temperature || !presenceRequest.TargetPresence.TopP {
|
||||||
t.Fatalf("explicit null/zero presence did not reach the injected client: target=%+v presence=%+v", presenceTarget, presenceRequest.TargetPresence)
|
t.Fatalf("explicit null/zero presence did not reach the injected client: target=%+v presence=%+v", presenceTarget, presenceRequest.TargetPresence)
|
||||||
}
|
}
|
||||||
@@ -306,8 +315,8 @@ prompts:
|
|||||||
if !reflect.DeepEqual(presenceTarget.ExtraParams, wantPresenceParams) {
|
if !reflect.DeepEqual(presenceTarget.ExtraParams, wantPresenceParams) {
|
||||||
t.Fatalf("recursive provider-map retention changed: %#v", presenceTarget.ExtraParams)
|
t.Fatalf("recursive provider-map retention changed: %#v", presenceTarget.ExtraParams)
|
||||||
}
|
}
|
||||||
if client.requests[2].Target.ExtraParams != nil {
|
if !reflect.DeepEqual(client.requests[2].Target.ExtraParams, map[string]any{"source": "second", "nested": map[string]any{"second_backend": true}}) {
|
||||||
t.Fatalf("whole-map null did not suppress backend defaults: %#v", client.requests[2].Target.ExtraParams)
|
t.Fatalf("whole-map null did not fall back to backend defaults: %#v", client.requests[2].Target.ExtraParams)
|
||||||
}
|
}
|
||||||
if !reflect.DeepEqual(client.requests[3].Target.ExtraParams, map[string]any{"source": "second", "nested": map[string]any{"second_backend": true}}) {
|
if !reflect.DeepEqual(client.requests[3].Target.ExtraParams, map[string]any{"source": "second", "nested": map[string]any{"second_backend": true}}) {
|
||||||
t.Fatalf("empty provider map unexpectedly cleared backend defaults: %#v", client.requests[3].Target.ExtraParams)
|
t.Fatalf("empty provider map unexpectedly cleared backend defaults: %#v", client.requests[3].Target.ExtraParams)
|
||||||
@@ -317,17 +326,16 @@ prompts:
|
|||||||
func TestCLICompletesPartialFrameworkProfileWithRequestOverrides(t *testing.T) {
|
func TestCLICompletesPartialFrameworkProfileWithRequestOverrides(t *testing.T) {
|
||||||
root := t.TempDir()
|
root := t.TempDir()
|
||||||
frameworkFile := writeFrameworkSourceFile(t, root, "partial.yml", `profiles:
|
frameworkFile := writeFrameworkSourceFile(t, root, "partial.yml", `profiles:
|
||||||
partial: {}
|
- id: partial
|
||||||
prompts:
|
prompts:
|
||||||
partial.prompt:
|
- id: partial.prompt
|
||||||
"1":
|
default_profile: partial
|
||||||
default_profile: partial
|
messages:
|
||||||
messages:
|
- role: user
|
||||||
- role: user
|
content: hello
|
||||||
content: hello
|
output:
|
||||||
output:
|
format: text
|
||||||
format: text
|
validation_mode: none
|
||||||
validation_mode: none
|
|
||||||
`)
|
`)
|
||||||
provider := newTestLLMServer("completed partial profile", nil)
|
provider := newTestLLMServer("completed partial profile", nil)
|
||||||
defer provider.Close()
|
defer provider.Close()
|
||||||
|
|||||||
@@ -1089,7 +1089,7 @@ func TestRenderCommandExplicitZeroTemperatureReachesEffectiveSettings(t *testing
|
|||||||
|
|
||||||
writePromptFile(t, lib.promptDir, "prompt.render", "local-default")
|
writePromptFile(t, lib.promptDir, "prompt.render", "local-default")
|
||||||
profile := `profiles:
|
profile := `profiles:
|
||||||
local-default:
|
- id: local-default
|
||||||
endpoint: http://127.0.0.1:1/v1
|
endpoint: http://127.0.0.1:1/v1
|
||||||
model: profile-model
|
model: profile-model
|
||||||
temperature: 0.7
|
temperature: 0.7
|
||||||
@@ -1147,7 +1147,7 @@ func TestRenderCommandLoadsImportsAndOverlayWithoutChangingExecutionProfile(t *t
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
base := filepath.Join(lib.rootDir, "config.yml")
|
base := filepath.Join(lib.rootDir, "config.yml")
|
||||||
framework := writeFrameworkSourceFile(t, lib.rootDir, "framework.yml", "configkit:\n imports:\n - path: prompts/prompt.render.yaml\n")
|
framework := writeFrameworkSourceFile(t, lib.rootDir, "framework.yml", "config:\n imports:\n - path: prompts/prompt.render.yaml\n required: true\n")
|
||||||
for path, body := range map[string]string{
|
for path, body := range map[string]string{
|
||||||
base: "config:\n imports:\n - path: fragments/sources.yml\n required: true\n",
|
base: "config:\n imports:\n - path: fragments/sources.yml\n required: true\n",
|
||||||
filepath.Join(lib.rootDir, "fragments", "sources.yml"): fmt.Sprintf("promptkit_sources:\n - file: %q\n - directory: %q\n", framework, lib.profileDir),
|
filepath.Join(lib.rootDir, "fragments", "sources.yml"): fmt.Sprintf("promptkit_sources:\n - file: %q\n - directory: %q\n", framework, lib.profileDir),
|
||||||
@@ -1174,25 +1174,24 @@ func TestRenderCommandLoadsImportsAndOverlayWithoutChangingExecutionProfile(t *t
|
|||||||
func TestRenderCommandUsesConfiguredCustomBackend(t *testing.T) {
|
func TestRenderCommandUsesConfiguredCustomBackend(t *testing.T) {
|
||||||
lib := newCLITestLibrary(t)
|
lib := newCLITestLibrary(t)
|
||||||
writePromptDefinition(t, lib.promptDir, "custom.yaml", `prompts:
|
writePromptDefinition(t, lib.promptDir, "custom.yaml", `prompts:
|
||||||
custom:
|
- id: custom
|
||||||
"1":
|
default_profile: local-gpu
|
||||||
default_profile: local-gpu
|
messages:
|
||||||
messages:
|
- role: user
|
||||||
- role: user
|
content: "hello"
|
||||||
content: "hello"
|
output:
|
||||||
output:
|
format: text
|
||||||
format: text
|
validation_mode: none
|
||||||
validation_mode: none
|
|
||||||
`)
|
`)
|
||||||
if err := os.WriteFile(filepath.Join(lib.profileDir, "local-gpu.yaml"), []byte(`profiles:
|
if err := os.WriteFile(filepath.Join(lib.profileDir, "local-gpu.yaml"), []byte(`profiles:
|
||||||
local-gpu:
|
- id: local-gpu
|
||||||
backend: local-gpu
|
backend: local-gpu
|
||||||
model: local-model
|
model: local-model
|
||||||
`), 0o644); err != nil {
|
`), 0o644); err != nil {
|
||||||
t.Fatalf("write profile fixture: %v", err)
|
t.Fatalf("write profile fixture: %v", err)
|
||||||
}
|
}
|
||||||
configPath := writeCLIFrameworkConfig(t, lib, `backends:
|
configPath := writeCLIFrameworkConfig(t, lib, `backends:
|
||||||
local-gpu:
|
- id: local-gpu
|
||||||
endpoint: http://localhost:11434/v1
|
endpoint: http://localhost:11434/v1
|
||||||
extra_params:
|
extra_params:
|
||||||
provider_option: enabled
|
provider_option: enabled
|
||||||
@@ -1215,30 +1214,29 @@ func TestRenderCommandUsesConfiguredCustomBackend(t *testing.T) {
|
|||||||
func TestRenderCommandUsesInheritedCustomBackend(t *testing.T) {
|
func TestRenderCommandUsesInheritedCustomBackend(t *testing.T) {
|
||||||
lib := newCLITestLibrary(t)
|
lib := newCLITestLibrary(t)
|
||||||
writePromptDefinition(t, lib.promptDir, "custom.yaml", `prompts:
|
writePromptDefinition(t, lib.promptDir, "custom.yaml", `prompts:
|
||||||
custom:
|
- id: custom
|
||||||
"1":
|
default_profile: local-gpu
|
||||||
default_profile: local-gpu
|
messages:
|
||||||
messages:
|
- role: user
|
||||||
- role: user
|
content: "hello"
|
||||||
content: "hello"
|
output:
|
||||||
output:
|
format: text
|
||||||
format: text
|
validation_mode: none
|
||||||
validation_mode: none
|
|
||||||
`)
|
`)
|
||||||
if err := os.WriteFile(filepath.Join(lib.profileDir, "local-gpu.yaml"), []byte(`profiles:
|
if err := os.WriteFile(filepath.Join(lib.profileDir, "local-gpu.yaml"), []byte(`profiles:
|
||||||
local-gpu:
|
- id: local-gpu
|
||||||
backend: local-gpu
|
backend: local-gpu
|
||||||
model: local-model
|
model: local-model
|
||||||
`), 0o644); err != nil {
|
`), 0o644); err != nil {
|
||||||
t.Fatalf("write profile fixture: %v", err)
|
t.Fatalf("write profile fixture: %v", err)
|
||||||
}
|
}
|
||||||
configPath := writeCLIFrameworkConfig(t, lib, `backends:
|
configPath := writeCLIFrameworkConfig(t, lib, `backends:
|
||||||
local-base:
|
- id: local-base
|
||||||
endpoint: http://localhost:11434/v1
|
endpoint: http://localhost:11434/v1
|
||||||
api_key_env: LOCAL_GPU_API_KEY
|
api_key_env: LOCAL_GPU_API_KEY
|
||||||
concurrency_limit: 2
|
concurrency_limit: 2
|
||||||
local-gpu:
|
- id: local-gpu
|
||||||
extends: local-base
|
extends: {id: local-base}
|
||||||
queue_capacity: 0
|
queue_capacity: 0
|
||||||
extra_params:
|
extra_params:
|
||||||
provider_option: false
|
provider_option: false
|
||||||
@@ -1267,17 +1265,16 @@ func TestRenderCommandUsesInheritedCustomBackend(t *testing.T) {
|
|||||||
func TestConfiguredBackendPreservesIntegerParameterAndOptionalCredential(t *testing.T) {
|
func TestConfiguredBackendPreservesIntegerParameterAndOptionalCredential(t *testing.T) {
|
||||||
lib := newCLITestLibrary(t)
|
lib := newCLITestLibrary(t)
|
||||||
writePromptDefinition(t, lib.promptDir, "custom.yaml", `prompts:
|
writePromptDefinition(t, lib.promptDir, "custom.yaml", `prompts:
|
||||||
custom:
|
- id: custom
|
||||||
"1":
|
default_profile: local-gpu
|
||||||
default_profile: local-gpu
|
messages:
|
||||||
messages:
|
- role: user
|
||||||
- role: user
|
content: "hello"
|
||||||
content: "hello"
|
output:
|
||||||
output:
|
format: text
|
||||||
format: text
|
validation_mode: none
|
||||||
validation_mode: none
|
|
||||||
`)
|
`)
|
||||||
if err := os.WriteFile(filepath.Join(lib.profileDir, "local-gpu.yaml"), []byte("profiles:\n local-gpu:\n backend: local-gpu\n model: local-model\n"), 0o644); err != nil {
|
if err := os.WriteFile(filepath.Join(lib.profileDir, "local-gpu.yaml"), []byte("profiles:\n - id: local-gpu\n backend: local-gpu\n model: local-model\n"), 0o644); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
const credentialName = "SCRIPTORIUM_OPTIONAL_BACKEND_KEY"
|
const credentialName = "SCRIPTORIUM_OPTIONAL_BACKEND_KEY"
|
||||||
@@ -1301,7 +1298,7 @@ func TestConfiguredBackendPreservesIntegerParameterAndOptionalCredential(t *test
|
|||||||
}))
|
}))
|
||||||
defer provider.Close()
|
defer provider.Close()
|
||||||
configPath := writeCLIFrameworkConfig(t, lib, fmt.Sprintf(`backends:
|
configPath := writeCLIFrameworkConfig(t, lib, fmt.Sprintf(`backends:
|
||||||
local-gpu:
|
- id: local-gpu
|
||||||
endpoint: %q
|
endpoint: %q
|
||||||
api_key_env: %s
|
api_key_env: %s
|
||||||
extra_params:
|
extra_params:
|
||||||
@@ -1324,19 +1321,18 @@ func TestConfiguredBackendPreservesIntegerParameterAndOptionalCredential(t *test
|
|||||||
func TestRenderCommandMapsReasoningEffortAndSessionID(t *testing.T) {
|
func TestRenderCommandMapsReasoningEffortAndSessionID(t *testing.T) {
|
||||||
lib := newCLITestLibrary(t)
|
lib := newCLITestLibrary(t)
|
||||||
writePromptDefinition(t, lib.promptDir, "session.yaml", `prompts:
|
writePromptDefinition(t, lib.promptDir, "session.yaml", `prompts:
|
||||||
session:
|
- id: session
|
||||||
"1":
|
default_profile: local
|
||||||
default_profile: local
|
session_id: definition-session
|
||||||
session_id: definition-session
|
messages:
|
||||||
messages:
|
- role: user
|
||||||
- role: user
|
content: "hello"
|
||||||
content: "hello"
|
output:
|
||||||
output:
|
format: text
|
||||||
format: text
|
validation_mode: none
|
||||||
validation_mode: none
|
|
||||||
`)
|
`)
|
||||||
if err := os.WriteFile(filepath.Join(lib.profileDir, "local.yaml"), []byte(`profiles:
|
if err := os.WriteFile(filepath.Join(lib.profileDir, "local.yaml"), []byte(`profiles:
|
||||||
local:
|
- id: local
|
||||||
endpoint: http://127.0.0.1:1/v1
|
endpoint: http://127.0.0.1:1/v1
|
||||||
model: local-model
|
model: local-model
|
||||||
reasoning_effort: low
|
reasoning_effort: low
|
||||||
@@ -1383,15 +1379,14 @@ func TestRenderCommandMapsReasoningEffortAndSessionID(t *testing.T) {
|
|||||||
func TestRenderCommandOmitsEmptyEffectiveSessionID(t *testing.T) {
|
func TestRenderCommandOmitsEmptyEffectiveSessionID(t *testing.T) {
|
||||||
lib := newCLITestLibrary(t)
|
lib := newCLITestLibrary(t)
|
||||||
writePromptDefinition(t, lib.promptDir, "plain.yaml", `prompts:
|
writePromptDefinition(t, lib.promptDir, "plain.yaml", `prompts:
|
||||||
plain:
|
- id: plain
|
||||||
"1":
|
default_profile: local
|
||||||
default_profile: local
|
messages:
|
||||||
messages:
|
- role: user
|
||||||
- role: user
|
content: "hello"
|
||||||
content: "hello"
|
output:
|
||||||
output:
|
format: text
|
||||||
format: text
|
validation_mode: none
|
||||||
validation_mode: none
|
|
||||||
`)
|
`)
|
||||||
writeProfileFile(t, lib.profileDir, "local", "http://127.0.0.1:1/v1", "local-model")
|
writeProfileFile(t, lib.profileDir, "local", "http://127.0.0.1:1/v1", "local-model")
|
||||||
|
|
||||||
@@ -1412,15 +1407,14 @@ func TestRenderCommandOmitsEmptyEffectiveSessionID(t *testing.T) {
|
|||||||
func TestRenderCommandRejectsOverlongSessionID(t *testing.T) {
|
func TestRenderCommandRejectsOverlongSessionID(t *testing.T) {
|
||||||
lib := newCLITestLibrary(t)
|
lib := newCLITestLibrary(t)
|
||||||
writePromptDefinition(t, lib.promptDir, "session.yaml", `prompts:
|
writePromptDefinition(t, lib.promptDir, "session.yaml", `prompts:
|
||||||
session:
|
- id: session
|
||||||
"1":
|
default_profile: local
|
||||||
default_profile: local
|
messages:
|
||||||
messages:
|
- role: user
|
||||||
- role: user
|
content: "hello"
|
||||||
content: "hello"
|
output:
|
||||||
output:
|
format: text
|
||||||
format: text
|
validation_mode: none
|
||||||
validation_mode: none
|
|
||||||
`)
|
`)
|
||||||
writeProfileFile(t, lib.profileDir, "local", "http://127.0.0.1:1/v1", "local-model")
|
writeProfileFile(t, lib.profileDir, "local", "http://127.0.0.1:1/v1", "local-model")
|
||||||
|
|
||||||
@@ -1883,7 +1877,7 @@ func TestPromptkitProfileInspectionResolvesSupportedTargets(t *testing.T) {
|
|||||||
}{
|
}{
|
||||||
{name: "inherited custom backend", profileID: "custom-derived", wantBackend: "fixture-custom", wantModel: "fixture-derived-model", wantAPIKeyEnv: "FIXTURE_PROFILE_API_KEY"},
|
{name: "inherited custom backend", profileID: "custom-derived", wantBackend: "fixture-custom", wantModel: "fixture-derived-model", wantAPIKeyEnv: "FIXTURE_PROFILE_API_KEY"},
|
||||||
{name: "endpoint only", profileID: "endpoint-only", wantModel: "fixture-endpoint-model"},
|
{name: "endpoint only", profileID: "endpoint-only", wantModel: "fixture-endpoint-model"},
|
||||||
{name: "built in", profileID: "promptkit-openrouter-auto", wantBackend: "openrouter", wantModel: "openrouter/auto", wantAPIKeyEnv: "OPENROUTER_API_KEY"},
|
{name: "built in", profileID: "auto", wantBackend: "openrouter", wantModel: "openrouter/auto"},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
@@ -1929,21 +1923,20 @@ func TestPromptkitProfileInspectionResolvesSupportedTargets(t *testing.T) {
|
|||||||
func TestProfileInspectionUsesCombinedSourcesAndValidatesUnusedPrompts(t *testing.T) {
|
func TestProfileInspectionUsesCombinedSourcesAndValidatesUnusedPrompts(t *testing.T) {
|
||||||
lib := newCLITestLibrary(t)
|
lib := newCLITestLibrary(t)
|
||||||
combined := `backends:
|
combined := `backends:
|
||||||
fixture:
|
- id: fixture
|
||||||
endpoint: http://127.0.0.1:1/v1
|
endpoint: http://127.0.0.1:1/v1
|
||||||
profiles:
|
profiles:
|
||||||
inspectable:
|
- id: inspectable
|
||||||
backend: fixture
|
backend: fixture
|
||||||
model: fixture-model
|
model: fixture-model
|
||||||
prompts:
|
prompts:
|
||||||
unused:
|
- id: unused
|
||||||
"1.0.0":
|
messages:
|
||||||
messages:
|
- role: user
|
||||||
- role: user
|
content: "valid unused prompt"
|
||||||
content: "valid unused prompt"
|
output:
|
||||||
output:
|
format: text
|
||||||
format: text
|
validation_mode: none
|
||||||
validation_mode: none
|
|
||||||
`
|
`
|
||||||
frameworkPath := writeFrameworkFile(t, lib.rootDir, "combined.yml", combined)
|
frameworkPath := writeFrameworkFile(t, lib.rootDir, "combined.yml", combined)
|
||||||
configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n", frameworkPath))
|
configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n", frameworkPath))
|
||||||
@@ -1976,19 +1969,18 @@ func TestMaintainedProfilesResolveAcrossSelectionAndSourcePrecedence(t *testing.
|
|||||||
promptDir := t.TempDir()
|
promptDir := t.TempDir()
|
||||||
profileDir := t.TempDir()
|
profileDir := t.TempDir()
|
||||||
writePromptDefinition(t, promptDir, "maintained.yaml", `prompts:
|
writePromptDefinition(t, promptDir, "maintained.yaml", `prompts:
|
||||||
maintained:
|
- id: maintained
|
||||||
"1.0.0":
|
default_profile: gpt-sol-latest
|
||||||
default_profile: promptkit-openrouter-standard
|
messages:
|
||||||
messages:
|
- role: user
|
||||||
- role: user
|
content: Hello
|
||||||
content: Hello
|
output:
|
||||||
output:
|
format: text
|
||||||
format: text
|
validation_mode: none
|
||||||
validation_mode: none
|
|
||||||
`)
|
`)
|
||||||
configPath := writeAppConfigFile(t, sourceSelectionsYAML(promptDir, profileDir, ""))
|
configPath := writeAppConfigFile(t, sourceSelectionsYAML(promptDir, profileDir, ""))
|
||||||
|
|
||||||
for _, profileID := range []string{"promptkit-openrouter-standard", "promptkit-openrouter-auto"} {
|
for _, profileID := range []string{"gpt-sol-latest", "auto"} {
|
||||||
t.Run(profileID+" inspection", func(t *testing.T) {
|
t.Run(profileID+" inspection", func(t *testing.T) {
|
||||||
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{"profile", "--config", configPath, "--profile", profileID, "--format", "json"})
|
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{"profile", "--config", configPath, "--profile", profileID, "--format", "json"})
|
||||||
if code != ExitOK {
|
if code != ExitOK {
|
||||||
@@ -2001,7 +1993,7 @@ func TestMaintainedProfilesResolveAcrossSelectionAndSourcePrecedence(t *testing.
|
|||||||
if inspection.ProfileID != profileID || pointerValue(inspection.ProfileModelParams.BackendID) != "openrouter" || pointerValue(inspection.ProfileModelParams.Model) == "" {
|
if inspection.ProfileID != profileID || pointerValue(inspection.ProfileModelParams.BackendID) != "openrouter" || pointerValue(inspection.ProfileModelParams.Model) == "" {
|
||||||
t.Fatalf("maintained profile did not resolve: %+v", inspection)
|
t.Fatalf("maintained profile did not resolve: %+v", inspection)
|
||||||
}
|
}
|
||||||
if profileID == "promptkit-openrouter-auto" && pointerValue(inspection.ProfileModelParams.Model) != "openrouter/auto" {
|
if profileID == "auto" && pointerValue(inspection.ProfileModelParams.Model) != "openrouter/auto" {
|
||||||
t.Fatalf("unexpected auto target: %+v", inspection.ProfileModelParams)
|
t.Fatalf("unexpected auto target: %+v", inspection.ProfileModelParams)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
@@ -2012,8 +2004,8 @@ func TestMaintainedProfilesResolveAcrossSelectionAndSourcePrecedence(t *testing.
|
|||||||
selection []string
|
selection []string
|
||||||
wantID string
|
wantID string
|
||||||
}{
|
}{
|
||||||
{name: "prompt default", wantID: "promptkit-openrouter-standard"},
|
{name: "prompt default", wantID: "gpt-sol-latest"},
|
||||||
{name: "explicit selection", selection: []string{"--profile", "promptkit-openrouter-auto"}, wantID: "promptkit-openrouter-auto"},
|
{name: "explicit selection", selection: []string{"--profile", "auto"}, wantID: "auto"},
|
||||||
} {
|
} {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
args := append([]string{"--config", configPath, "--prompt", "maintained", "--format", "json"}, tc.selection...)
|
args := append([]string{"--config", configPath, "--prompt", "maintained", "--format", "json"}, tc.selection...)
|
||||||
@@ -2034,11 +2026,11 @@ func TestMaintainedProfilesResolveAcrossSelectionAndSourcePrecedence(t *testing.
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
writePromptDefinition(t, profileDir, "derived.yaml", "profiles:\n derived-maintained:\n extends: promptkit-openrouter-standard\n model: fixture-inherited-model\n")
|
writePromptDefinition(t, profileDir, "derived.yaml", "profiles:\n - id: derived-maintained\n extends: {id: gpt-sol-latest}\n model: fixture-inherited-model\n")
|
||||||
writePromptDefinition(t, profileDir, "replacement.yaml", "profiles:\n promptkit-openrouter-auto:\n backend: openrouter\n model: fixture-override-model\n")
|
writePromptDefinition(t, profileDir, "replacement.yaml", "profiles:\n - id: auto\n backend: openrouter\n model: fixture-override-model\n")
|
||||||
for _, tc := range []struct{ id, model string }{
|
for _, tc := range []struct{ id, model string }{
|
||||||
{id: "derived-maintained", model: "fixture-inherited-model"},
|
{id: "derived-maintained", model: "fixture-inherited-model"},
|
||||||
{id: "promptkit-openrouter-auto", model: "fixture-override-model"},
|
{id: "auto", model: "fixture-override-model"},
|
||||||
} {
|
} {
|
||||||
t.Run(tc.id+" source", func(t *testing.T) {
|
t.Run(tc.id+" source", func(t *testing.T) {
|
||||||
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{"profile", "--config", configPath, "--profile", tc.id, "--format", "json"})
|
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{"profile", "--config", configPath, "--profile", tc.id, "--format", "json"})
|
||||||
@@ -2089,7 +2081,7 @@ func TestProfileInspectionHonorsDirectoryPrecedenceOutputAndFailures(t *testing.
|
|||||||
args := []string{"profile", "--config", configPath, "--profile", profileID}
|
args := []string{"profile", "--config", configPath, "--profile", profileID}
|
||||||
if profileID == "invalid" {
|
if profileID == "invalid" {
|
||||||
profileDir := t.TempDir()
|
profileDir := t.TempDir()
|
||||||
writePromptDefinition(t, profileDir, "invalid.yaml", "profiles:\n invalid:\n endpoint: not-a-url\n model: fixture\n")
|
writePromptDefinition(t, profileDir, "invalid.yaml", "profiles:\n - id: invalid\n endpoint: not-a-url\n model: fixture\n")
|
||||||
args = append(args,
|
args = append(args,
|
||||||
"--promptkit-source", filepath.Join(fixtureRoot, "prompts"),
|
"--promptkit-source", filepath.Join(fixtureRoot, "prompts"),
|
||||||
"--promptkit-source", profileDir,
|
"--promptkit-source", profileDir,
|
||||||
@@ -2114,14 +2106,14 @@ func TestConfiguredBackendValidationComesFromPromptkit(t *testing.T) {
|
|||||||
{
|
{
|
||||||
name: "invalid endpoint",
|
name: "invalid endpoint",
|
||||||
backend: `backends:
|
backend: `backends:
|
||||||
local-gpu:
|
- id: local-gpu
|
||||||
endpoint: not-a-url
|
endpoint: not-a-url
|
||||||
`,
|
`,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "invalid capacity relationship",
|
name: "invalid capacity relationship",
|
||||||
backend: `backends:
|
backend: `backends:
|
||||||
local-gpu:
|
- id: local-gpu
|
||||||
endpoint: http://localhost:11434/v1
|
endpoint: http://localhost:11434/v1
|
||||||
queue_capacity: 0
|
queue_capacity: 0
|
||||||
`,
|
`,
|
||||||
@@ -2129,7 +2121,7 @@ func TestConfiguredBackendValidationComesFromPromptkit(t *testing.T) {
|
|||||||
{
|
{
|
||||||
name: "reserved extra parameter",
|
name: "reserved extra parameter",
|
||||||
backend: `backends:
|
backend: `backends:
|
||||||
local-gpu:
|
- id: local-gpu
|
||||||
endpoint: http://localhost:11434/v1
|
endpoint: http://localhost:11434/v1
|
||||||
extra_params:
|
extra_params:
|
||||||
model: forbidden
|
model: forbidden
|
||||||
@@ -2154,12 +2146,12 @@ func TestConfiguredBackendValidationComesFromPromptkit(t *testing.T) {
|
|||||||
|
|
||||||
t.Run("built-in backend overlay", func(t *testing.T) {
|
t.Run("built-in backend overlay", func(t *testing.T) {
|
||||||
configPath := writeCLIFrameworkConfig(t, lib, `backends:
|
configPath := writeCLIFrameworkConfig(t, lib, `backends:
|
||||||
openrouter:
|
- id: openrouter
|
||||||
endpoint: http://localhost:11434/v1
|
endpoint: http://localhost:11434/v1
|
||||||
`, "")
|
`, "")
|
||||||
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
|
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
|
||||||
"profile", "--config", configPath,
|
"profile", "--config", configPath,
|
||||||
"--profile", "promptkit-openrouter-auto", "--format", "json",
|
"--profile", "auto", "--format", "json",
|
||||||
})
|
})
|
||||||
if code != ExitOK {
|
if code != ExitOK {
|
||||||
t.Fatalf("built-in backend overlay failed: code=%d out=%q err=%q", code, stdout, stderr)
|
t.Fatalf("built-in backend overlay failed: code=%d out=%q err=%q", code, stdout, stderr)
|
||||||
@@ -2177,10 +2169,10 @@ func TestConfiguredBackendValidationComesFromPromptkit(t *testing.T) {
|
|||||||
func TestInheritedBackendParentStillReceivesPromptkitValidation(t *testing.T) {
|
func TestInheritedBackendParentStillReceivesPromptkitValidation(t *testing.T) {
|
||||||
lib := newCLITestLibrary(t)
|
lib := newCLITestLibrary(t)
|
||||||
configPath := writeCLIFrameworkConfig(t, lib, `backends:
|
configPath := writeCLIFrameworkConfig(t, lib, `backends:
|
||||||
base:
|
- id: base
|
||||||
endpoint: not-a-url
|
endpoint: not-a-url
|
||||||
child:
|
- id: child
|
||||||
extends: base
|
extends: {id: base}
|
||||||
endpoint: http://localhost:11434/v1
|
endpoint: http://localhost:11434/v1
|
||||||
`, "")
|
`, "")
|
||||||
cfg, err := parseRenderArgs([]string{"--config", configPath, "--prompt", "custom"})
|
cfg, err := parseRenderArgs([]string{"--config", configPath, "--prompt", "custom"})
|
||||||
@@ -2332,7 +2324,7 @@ func TestRenderCommandUsesBuiltInProfileWithoutCustomProfileSource(t *testing.T)
|
|||||||
lib := newCLITestLibrary(t)
|
lib := newCLITestLibrary(t)
|
||||||
inputPath := lib.writeInputFile(t, "transcript.md", "hello")
|
inputPath := lib.writeInputFile(t, "transcript.md", "hello")
|
||||||
|
|
||||||
writePromptFile(t, lib.promptDir, "prompt.builtin", "mistral-small-3")
|
writePromptFile(t, lib.promptDir, "prompt.builtin", "auto")
|
||||||
|
|
||||||
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
"--config", lib.configPath,
|
"--config", lib.configPath,
|
||||||
@@ -2343,10 +2335,10 @@ func TestRenderCommandUsesBuiltInProfileWithoutCustomProfileSource(t *testing.T)
|
|||||||
if code != ExitOK {
|
if code != ExitOK {
|
||||||
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
||||||
}
|
}
|
||||||
if !strings.Contains(stdout, "selected_profile_id: mistral-small-3") {
|
if !strings.Contains(stdout, "selected_profile_id: auto") {
|
||||||
t.Fatalf("expected built-in selected profile, got %q", stdout)
|
t.Fatalf("expected built-in selected profile, got %q", stdout)
|
||||||
}
|
}
|
||||||
if !strings.Contains(stdout, "model: mistralai/mistral-small-3.2-24b-instruct") {
|
if !strings.Contains(stdout, "model: openrouter/auto") {
|
||||||
t.Fatalf("expected built-in model, got %q", stdout)
|
t.Fatalf("expected built-in model, got %q", stdout)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -2385,54 +2377,50 @@ func TestRenderCommandUsesDefinitionInputRules(t *testing.T) {
|
|||||||
writeProfileFile(t, lib.profileDir, "local", "http://127.0.0.1:1/v1", "model")
|
writeProfileFile(t, lib.profileDir, "local", "http://127.0.0.1:1/v1", "model")
|
||||||
|
|
||||||
writePromptDefinition(t, lib.promptDir, "sole.yaml", `prompts:
|
writePromptDefinition(t, lib.promptDir, "sole.yaml", `prompts:
|
||||||
sole:
|
- id: sole
|
||||||
"1":
|
default_profile: local
|
||||||
default_profile: local
|
messages:
|
||||||
messages:
|
- role: user
|
||||||
- role: user
|
content: "hello"
|
||||||
content: "hello"
|
output:
|
||||||
output:
|
format: text
|
||||||
format: text
|
validation_mode: none
|
||||||
validation_mode: none
|
|
||||||
`)
|
`)
|
||||||
writePromptDefinition(t, lib.promptDir, "optional.yaml", `prompts:
|
writePromptDefinition(t, lib.promptDir, "optional.yaml", `prompts:
|
||||||
optional:
|
- id: optional
|
||||||
"1":
|
default_profile: local
|
||||||
default_profile: local
|
inputs:
|
||||||
inputs:
|
- name: note
|
||||||
- name: note
|
required: false
|
||||||
required: false
|
messages:
|
||||||
messages:
|
- role: user
|
||||||
- role: user
|
content: "hello"
|
||||||
content: "hello"
|
output:
|
||||||
output:
|
format: text
|
||||||
format: text
|
validation_mode: none
|
||||||
validation_mode: none
|
|
||||||
`)
|
`)
|
||||||
writePromptDefinition(t, lib.promptDir, "required.yaml", `prompts:
|
writePromptDefinition(t, lib.promptDir, "required.yaml", `prompts:
|
||||||
required:
|
- id: required
|
||||||
"1":
|
default_profile: local
|
||||||
default_profile: local
|
inputs:
|
||||||
inputs:
|
- name: note
|
||||||
- name: note
|
required: true
|
||||||
required: true
|
messages:
|
||||||
messages:
|
- role: user
|
||||||
- role: user
|
content: "hello"
|
||||||
content: "hello"
|
output:
|
||||||
output:
|
format: text
|
||||||
format: text
|
validation_mode: none
|
||||||
validation_mode: none
|
|
||||||
`)
|
`)
|
||||||
writePromptDefinition(t, lib.promptDir, "template.yaml", `prompts:
|
writePromptDefinition(t, lib.promptDir, "template.yaml", `prompts:
|
||||||
template:
|
- id: template
|
||||||
"1":
|
default_profile: local
|
||||||
default_profile: local
|
messages:
|
||||||
messages:
|
- role: user
|
||||||
- role: user
|
content: '{{input "note"}}'
|
||||||
content: '{{input "note"}}'
|
output:
|
||||||
output:
|
format: text
|
||||||
format: text
|
validation_mode: none
|
||||||
validation_mode: none
|
|
||||||
`)
|
`)
|
||||||
|
|
||||||
baseArgs := []string{"--config", lib.configPath, "--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir}
|
baseArgs := []string{"--config", lib.configPath, "--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir}
|
||||||
@@ -2746,19 +2734,18 @@ func writePromptDefinition(t *testing.T, dir, name, definition string) {
|
|||||||
func writePromptFileWithTemplate(t *testing.T, dir, id, defaultProfile, templateContent string) {
|
func writePromptFileWithTemplate(t *testing.T, dir, id, defaultProfile, templateContent string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
data := fmt.Sprintf(`prompts:
|
data := fmt.Sprintf(`prompts:
|
||||||
%q:
|
- id: %q
|
||||||
"1.0.0":
|
default_profile: %q
|
||||||
default_profile: %q
|
inputs:
|
||||||
inputs:
|
- name: transcript
|
||||||
- name: transcript
|
required: true
|
||||||
required: true
|
messages:
|
||||||
messages:
|
- role: user
|
||||||
- role: user
|
content: %q
|
||||||
content: %q
|
output:
|
||||||
output:
|
format: text
|
||||||
format: text
|
validation_mode: none
|
||||||
validation_mode: none
|
repair_attempts: 0
|
||||||
repair_attempts: 0
|
|
||||||
`, id, defaultProfile, templateContent)
|
`, id, defaultProfile, templateContent)
|
||||||
if err := os.WriteFile(filepath.Join(dir, id+".yaml"), []byte(data), 0o644); err != nil {
|
if err := os.WriteFile(filepath.Join(dir, id+".yaml"), []byte(data), 0o644); err != nil {
|
||||||
t.Fatalf("failed to write prompt fixture: %v", err)
|
t.Fatalf("failed to write prompt fixture: %v", err)
|
||||||
@@ -2767,7 +2754,7 @@ func writePromptFileWithTemplate(t *testing.T, dir, id, defaultProfile, template
|
|||||||
|
|
||||||
func writeProfileFile(t *testing.T, dir, id, endpoint, model string) {
|
func writeProfileFile(t *testing.T, dir, id, endpoint, model string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
data := fmt.Sprintf("profiles:\n %q:\n endpoint: %q\n model: %q\n", id, endpoint, model)
|
data := fmt.Sprintf("profiles:\n - id: %q\n endpoint: %q\n model: %q\n", id, endpoint, model)
|
||||||
if err := os.WriteFile(filepath.Join(dir, id+".yaml"), []byte(data), 0o644); err != nil {
|
if err := os.WriteFile(filepath.Join(dir, id+".yaml"), []byte(data), 0o644); err != nil {
|
||||||
t.Fatalf("failed to write profile fixture: %v", err)
|
t.Fatalf("failed to write profile fixture: %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -629,51 +629,48 @@ func TestHandlerDelegatesDefinitionInputRequirements(t *testing.T) {
|
|||||||
{
|
{
|
||||||
name: "no declared inputs",
|
name: "no declared inputs",
|
||||||
definition: `prompts:
|
definition: `prompts:
|
||||||
p:
|
- id: p
|
||||||
"1":
|
default_profile: exec
|
||||||
default_profile: exec
|
messages:
|
||||||
messages:
|
- role: user
|
||||||
- role: user
|
content: "hello"
|
||||||
content: "hello"
|
output:
|
||||||
output:
|
format: text
|
||||||
format: text
|
validation_mode: none
|
||||||
validation_mode: none
|
|
||||||
`,
|
`,
|
||||||
wantStatus: http.StatusOK,
|
wantStatus: http.StatusOK,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "optional input omitted",
|
name: "optional input omitted",
|
||||||
definition: `prompts:
|
definition: `prompts:
|
||||||
p:
|
- id: p
|
||||||
"1":
|
default_profile: exec
|
||||||
default_profile: exec
|
inputs:
|
||||||
inputs:
|
- name: note
|
||||||
- name: note
|
required: false
|
||||||
required: false
|
messages:
|
||||||
messages:
|
- role: user
|
||||||
- role: user
|
content: "hello"
|
||||||
content: "hello"
|
output:
|
||||||
output:
|
format: text
|
||||||
format: text
|
validation_mode: none
|
||||||
validation_mode: none
|
|
||||||
`,
|
`,
|
||||||
wantStatus: http.StatusOK,
|
wantStatus: http.StatusOK,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "required input omitted",
|
name: "required input omitted",
|
||||||
definition: `prompts:
|
definition: `prompts:
|
||||||
p:
|
- id: p
|
||||||
"1":
|
default_profile: exec
|
||||||
default_profile: exec
|
inputs:
|
||||||
inputs:
|
- name: note
|
||||||
- name: note
|
required: true
|
||||||
required: true
|
messages:
|
||||||
messages:
|
- role: user
|
||||||
- role: user
|
content: "hello"
|
||||||
content: "hello"
|
output:
|
||||||
output:
|
format: text
|
||||||
format: text
|
validation_mode: none
|
||||||
validation_mode: none
|
|
||||||
`,
|
`,
|
||||||
wantStatus: http.StatusBadRequest,
|
wantStatus: http.StatusBadRequest,
|
||||||
wantCode: "prompt_render_failed",
|
wantCode: "prompt_render_failed",
|
||||||
@@ -681,15 +678,14 @@ func TestHandlerDelegatesDefinitionInputRequirements(t *testing.T) {
|
|||||||
{
|
{
|
||||||
name: "template input omitted",
|
name: "template input omitted",
|
||||||
definition: `prompts:
|
definition: `prompts:
|
||||||
p:
|
- id: p
|
||||||
"1":
|
default_profile: exec
|
||||||
default_profile: exec
|
messages:
|
||||||
messages:
|
- role: user
|
||||||
- role: user
|
content: '{{input "note"}}'
|
||||||
content: '{{input "note"}}'
|
output:
|
||||||
output:
|
format: text
|
||||||
format: text
|
validation_mode: none
|
||||||
validation_mode: none
|
|
||||||
`,
|
`,
|
||||||
wantStatus: http.StatusBadRequest,
|
wantStatus: http.StatusBadRequest,
|
||||||
wantCode: "prompt_render_failed",
|
wantCode: "prompt_render_failed",
|
||||||
@@ -1020,18 +1016,18 @@ func TestHandlerSessionIDUsesPromptkitResolution(t *testing.T) {
|
|||||||
profileDir := t.TempDir()
|
profileDir := t.TempDir()
|
||||||
sessionLine := ""
|
sessionLine := ""
|
||||||
if tc.definitionSession != "" {
|
if tc.definitionSession != "" {
|
||||||
sessionLine = " session_id: " + tc.definitionSession + "\n"
|
sessionLine = " session_id: " + tc.definitionSession + "\n"
|
||||||
}
|
}
|
||||||
if err := os.WriteFile(filepath.Join(promptDir, "prompt.yaml"), []byte("prompts:\n p:\n \"1\":\n default_profile: exec\n"+sessionLine+` messages:
|
if err := os.WriteFile(filepath.Join(promptDir, "prompt.yaml"), []byte("prompts:\n - id: p\n default_profile: exec\n"+sessionLine+` messages:
|
||||||
- role: user
|
- role: user
|
||||||
content: "hi"
|
content: "hi"
|
||||||
output:
|
output:
|
||||||
format: text
|
format: text
|
||||||
validation_mode: none
|
validation_mode: none
|
||||||
`), 0o644); err != nil {
|
`), 0o644); err != nil {
|
||||||
t.Fatalf("write prompt fixture: %v", err)
|
t.Fatalf("write prompt fixture: %v", err)
|
||||||
}
|
}
|
||||||
if err := os.WriteFile(filepath.Join(profileDir, "profile.yaml"), []byte("profiles:\n exec:\n endpoint: http://127.0.0.1:1/v1\n model: test\n"), 0o644); err != nil {
|
if err := os.WriteFile(filepath.Join(profileDir, "profile.yaml"), []byte("profiles:\n - id: exec\n endpoint: http://127.0.0.1:1/v1\n model: test\n"), 0o644); err != nil {
|
||||||
t.Fatalf("write profile fixture: %v", err)
|
t.Fatalf("write profile fixture: %v", err)
|
||||||
}
|
}
|
||||||
engine := newHandlerEngineFromDirectories(t, promptDir, profileDir, nil, nil, promptkit.WithLLMClient(handlerLLMClient{}))
|
engine := newHandlerEngineFromDirectories(t, promptDir, profileDir, nil, nil, promptkit.WithLLMClient(handlerLLMClient{}))
|
||||||
@@ -1067,20 +1063,19 @@ func TestHandlerSharesBackendCapacityAcrossConcurrentRequests(t *testing.T) {
|
|||||||
promptDir := t.TempDir()
|
promptDir := t.TempDir()
|
||||||
profileDir := t.TempDir()
|
profileDir := t.TempDir()
|
||||||
if err := os.WriteFile(filepath.Join(promptDir, "prompt.yaml"), []byte(`prompts:
|
if err := os.WriteFile(filepath.Join(promptDir, "prompt.yaml"), []byte(`prompts:
|
||||||
p:
|
- id: p
|
||||||
"1":
|
default_profile: limited
|
||||||
default_profile: limited
|
messages:
|
||||||
messages:
|
- role: user
|
||||||
- role: user
|
content: "hi"
|
||||||
content: "hi"
|
output:
|
||||||
output:
|
format: text
|
||||||
format: text
|
validation_mode: none
|
||||||
validation_mode: none
|
|
||||||
`), 0o644); err != nil {
|
`), 0o644); err != nil {
|
||||||
t.Fatalf("write prompt fixture: %v", err)
|
t.Fatalf("write prompt fixture: %v", err)
|
||||||
}
|
}
|
||||||
if err := os.WriteFile(filepath.Join(profileDir, "profile.yaml"), []byte(`profiles:
|
if err := os.WriteFile(filepath.Join(profileDir, "profile.yaml"), []byte(`profiles:
|
||||||
limited:
|
- id: limited
|
||||||
backend: limited
|
backend: limited
|
||||||
model: test
|
model: test
|
||||||
`), 0o644); err != nil {
|
`), 0o644); err != nil {
|
||||||
@@ -1088,7 +1083,7 @@ func TestHandlerSharesBackendCapacityAcrossConcurrentRequests(t *testing.T) {
|
|||||||
}
|
}
|
||||||
queueCapacity := 0
|
queueCapacity := 0
|
||||||
client := &blockingLLMClient{started: make(chan struct{}, 1), release: make(chan struct{})}
|
client := &blockingLLMClient{started: make(chan struct{}, 1), release: make(chan struct{})}
|
||||||
backend := promptkit.BackendConfig{Endpoint: stringPointer("http://127.0.0.1:1/v1"), ConcurrencyLimit: intPointer(1), QueueCapacity: &queueCapacity}
|
backend := promptkit.BackendConfig{ID: "limited", Endpoint: stringPointer("http://127.0.0.1:1/v1"), ConcurrencyLimit: intPointer(1), QueueCapacity: &queueCapacity}
|
||||||
engine := newHandlerEngineFromDirectories(t, promptDir, profileDir, nil, &backend, promptkit.WithLLMClient(client))
|
engine := newHandlerEngineFromDirectories(t, promptDir, profileDir, nil, &backend, promptkit.WithLLMClient(client))
|
||||||
h := NewHandler(engine)
|
h := NewHandler(engine)
|
||||||
first := httptest.NewRecorder()
|
first := httptest.NewRecorder()
|
||||||
@@ -1822,21 +1817,20 @@ func newHandlerEngineWithOptions(t *testing.T, options ...promptkit.Option) *pro
|
|||||||
promptDir := t.TempDir()
|
promptDir := t.TempDir()
|
||||||
profileDir := t.TempDir()
|
profileDir := t.TempDir()
|
||||||
if err := os.WriteFile(filepath.Join(promptDir, "prompt.yaml"), []byte(`prompts:
|
if err := os.WriteFile(filepath.Join(promptDir, "prompt.yaml"), []byte(`prompts:
|
||||||
p:
|
- id: p
|
||||||
"1":
|
default_profile: exec
|
||||||
default_profile: exec
|
messages:
|
||||||
messages:
|
- role: user
|
||||||
- role: user
|
content: "hi"
|
||||||
content: "hi"
|
output:
|
||||||
output:
|
format: text
|
||||||
format: text
|
validation_mode: none
|
||||||
validation_mode: none
|
repair_attempts: 0
|
||||||
repair_attempts: 0
|
|
||||||
`), 0o644); err != nil {
|
`), 0o644); err != nil {
|
||||||
t.Fatalf("write prompt fixture: %v", err)
|
t.Fatalf("write prompt fixture: %v", err)
|
||||||
}
|
}
|
||||||
if err := os.WriteFile(filepath.Join(profileDir, "profile.yaml"), []byte(`profiles:
|
if err := os.WriteFile(filepath.Join(profileDir, "profile.yaml"), []byte(`profiles:
|
||||||
exec:
|
- id: exec
|
||||||
endpoint: http://example.invalid/v1
|
endpoint: http://example.invalid/v1
|
||||||
model: model
|
model: model
|
||||||
`), 0o644); err != nil {
|
`), 0o644); err != nil {
|
||||||
@@ -1855,7 +1849,7 @@ func newDefinitionHandler(t *testing.T, definition string) *Handler {
|
|||||||
t.Fatalf("write prompt fixture: %v", err)
|
t.Fatalf("write prompt fixture: %v", err)
|
||||||
}
|
}
|
||||||
if err := os.WriteFile(filepath.Join(profileDir, "profile.yaml"), []byte(`profiles:
|
if err := os.WriteFile(filepath.Join(profileDir, "profile.yaml"), []byte(`profiles:
|
||||||
exec:
|
- id: exec
|
||||||
endpoint: http://example.invalid/v1
|
endpoint: http://example.invalid/v1
|
||||||
model: model
|
model: model
|
||||||
`), 0o644); err != nil {
|
`), 0o644); err != nil {
|
||||||
|
|||||||
@@ -107,19 +107,18 @@ func TestCollectorReceivesRealEngineTerminalEvent(t *testing.T) {
|
|||||||
promptDir := t.TempDir()
|
promptDir := t.TempDir()
|
||||||
profileDir := t.TempDir()
|
profileDir := t.TempDir()
|
||||||
if err := os.WriteFile(filepath.Join(promptDir, "prompt.yaml"), []byte(`prompts:
|
if err := os.WriteFile(filepath.Join(promptDir, "prompt.yaml"), []byte(`prompts:
|
||||||
p:
|
- id: p
|
||||||
"1":
|
default_profile: local
|
||||||
default_profile: local
|
messages:
|
||||||
messages:
|
- role: user
|
||||||
- role: user
|
content: hi
|
||||||
content: hi
|
output:
|
||||||
output:
|
format: text
|
||||||
format: text
|
validation_mode: none
|
||||||
validation_mode: none
|
|
||||||
`), 0o644); err != nil {
|
`), 0o644); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if err := os.WriteFile(filepath.Join(profileDir, "profile.yaml"), []byte("profiles:\n local:\n endpoint: http://127.0.0.1:1/v1\n model: local-model\n"), 0o644); err != nil {
|
if err := os.WriteFile(filepath.Join(profileDir, "profile.yaml"), []byte("profiles:\n - id: local\n endpoint: http://127.0.0.1:1/v1\n model: local-model\n"), 0o644); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
promptInputs, err := promptkit.DirectoryInputs(promptDir)
|
promptInputs, err := promptkit.DirectoryInputs(promptDir)
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
backends:
|
backends:
|
||||||
fixture-custom:
|
- id: fixture-custom
|
||||||
endpoint: http://127.0.0.1:11434/v1
|
endpoint: http://127.0.0.1:11434/v1
|
||||||
api_key_env: FIXTURE_BACKEND_API_KEY
|
api_key_env: FIXTURE_BACKEND_API_KEY
|
||||||
extra_params:
|
extra_params:
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
profiles:
|
profiles:
|
||||||
custom-base:
|
- id: custom-base
|
||||||
backend: fixture-custom
|
backend: fixture-custom
|
||||||
model: fixture-base-model
|
model: fixture-base-model
|
||||||
temperature: 0.25
|
temperature: 0.25
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
profiles:
|
profiles:
|
||||||
custom-derived:
|
- id: custom-derived
|
||||||
extends: custom-base
|
extends: {id: custom-base}
|
||||||
model: fixture-derived-model
|
model: fixture-derived-model
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
profiles:
|
profiles:
|
||||||
endpoint-only:
|
- id: endpoint-only
|
||||||
endpoint: http://127.0.0.1:8000/v1
|
endpoint: http://127.0.0.1:8000/v1
|
||||||
model: fixture-endpoint-model
|
model: fixture-endpoint-model
|
||||||
temperature: 0.1
|
temperature: 0.1
|
||||||
|
|||||||
79
testdata/promptkit-v0.9/prompts/complete-v1.yaml
vendored
79
testdata/promptkit-v0.9/prompts/complete-v1.yaml
vendored
@@ -1,41 +1,40 @@
|
|||||||
prompts:
|
prompts:
|
||||||
compat.complete:
|
- id: compat.complete
|
||||||
"1.0.0":
|
description: Exercise the complete Promptkit v0.9 prompt-definition surface.
|
||||||
description: Exercise the complete Promptkit v0.9 prompt-definition surface.
|
default_profile: custom-derived
|
||||||
default_profile: custom-derived
|
session_id: 'fixture-{{.topic}}'
|
||||||
session_id: 'fixture-{{.topic}}'
|
inputs:
|
||||||
inputs:
|
- name: source
|
||||||
- name: source
|
required: true
|
||||||
required: true
|
content_type: text/markdown
|
||||||
content_type: text/markdown
|
description: Source material rendered by a file-backed message.
|
||||||
description: Source material rendered by a file-backed message.
|
- name: context
|
||||||
- name: context
|
required: false
|
||||||
required: false
|
content_type: text/plain
|
||||||
content_type: text/plain
|
description: Optional caller context.
|
||||||
description: Optional caller context.
|
messages:
|
||||||
messages:
|
- role: developer
|
||||||
- role: developer
|
content: 'Use the topic {{.topic}}.'
|
||||||
content: 'Use the topic {{.topic}}.'
|
cache_control:
|
||||||
cache_control:
|
type: ephemeral
|
||||||
type: ephemeral
|
ttl: 1h
|
||||||
ttl: 1h
|
- role: system
|
||||||
- role: system
|
content_file: ./messages/complete.system.md
|
||||||
content_file: ./messages/complete.system.md
|
- role: user
|
||||||
- role: user
|
content_file: ./messages/complete.user.md
|
||||||
content_file: ./messages/complete.user.md
|
- role: assistant
|
||||||
- role: assistant
|
content: Return one JSON object.
|
||||||
content: Return one JSON object.
|
output:
|
||||||
output:
|
format: json
|
||||||
format: json
|
validation_mode: json_schema
|
||||||
validation_mode: json_schema
|
schema_path: complete.schema.json
|
||||||
schema_path: complete.schema.json
|
repair_attempts: 2
|
||||||
repair_attempts: 2
|
- id: compat.complete-text
|
||||||
"2.0.0":
|
default_profile: endpoint-only
|
||||||
default_profile: endpoint-only
|
messages:
|
||||||
messages:
|
- role: user
|
||||||
- role: user
|
content: Version two has no declared inputs.
|
||||||
content: Version two has no declared inputs.
|
output:
|
||||||
output:
|
format: text
|
||||||
format: text
|
validation_mode: basic
|
||||||
validation_mode: basic
|
repair_attempts: 1
|
||||||
repair_attempts: 1
|
|
||||||
|
|||||||
19
testdata/promptkit-v0.9/prompts/json.yaml
vendored
19
testdata/promptkit-v0.9/prompts/json.yaml
vendored
@@ -1,11 +1,10 @@
|
|||||||
prompts:
|
prompts:
|
||||||
compat.json:
|
- id: compat.json
|
||||||
"1.0.0":
|
default_profile: endpoint-only
|
||||||
default_profile: endpoint-only
|
messages:
|
||||||
messages:
|
- role: user
|
||||||
- role: user
|
content: Return JSON.
|
||||||
content: Return JSON.
|
output:
|
||||||
output:
|
format: json
|
||||||
format: json
|
validation_mode: json
|
||||||
validation_mode: json
|
repair_attempts: 0
|
||||||
repair_attempts: 0
|
|
||||||
|
|||||||
19
testdata/promptkit-v0.9/prompts/none.yaml
vendored
19
testdata/promptkit-v0.9/prompts/none.yaml
vendored
@@ -1,11 +1,10 @@
|
|||||||
prompts:
|
prompts:
|
||||||
compat.none:
|
- id: compat.none
|
||||||
"1.0.0":
|
default_profile: endpoint-only
|
||||||
default_profile: endpoint-only
|
messages:
|
||||||
messages:
|
- role: user
|
||||||
- role: user
|
content: Return text.
|
||||||
content: Return text.
|
output:
|
||||||
output:
|
format: text
|
||||||
format: text
|
validation_mode: none
|
||||||
validation_mode: none
|
repair_attempts: 0
|
||||||
repair_attempts: 0
|
|
||||||
|
|||||||
Reference in New Issue
Block a user