Adapt inspection to native PromptKit records

This commit is contained in:
2026-11-18 19:08:41 +00:00
parent 77f168c660
commit de7c866662
8 changed files with 306 additions and 270 deletions

View File

@@ -27,11 +27,11 @@ func TestFrameworkConfigurationCommandsNeedNoPromptOrCredential(t *testing.T) {
root := t.TempDir()
frameworkFile := filepath.Join(root, "framework.yml")
writeFrameworkSourceFile(t, root, "framework.yml", `backends:
local:
- id: local
endpoint: http://127.0.0.1:1/v1
api_key_env: SCRIPTORIUM_FRAMEWORK_KEY
profiles:
protected:
- id: protected
backend: local
model: offline-model
api_key_required: true
@@ -59,24 +59,21 @@ func TestFrameworkConfigurationSourcesProjectsOrderedNativeRecords(t *testing.T)
if err := os.MkdirAll(importDir, 0o755); err != nil {
t.Fatal(err)
}
mainPath := writeFrameworkSourceFile(t, root, "main.yml", `configkit:
mainPath := writeFrameworkSourceFile(t, root, "main.yml", `config:
imports:
- path: catalog/fragment.yml
required: true
backends:
local:
- id: local
endpoint: http://127.0.0.1:1/v1
profiles:
selected:
backend: local
model: selected-model
`)
fragmentPath := writeFrameworkSourceFile(t, importDir, "fragment.yml", `profiles:
imported:
- id: imported
model: imported-model
`)
directory := t.TempDir()
directoryPath := writeFrameworkSourceFile(t, directory, "directory.yml", `profiles:
directory:
- id: directory
model: directory-model
`)
configuredMissing := filepath.Join(root, "configured-but-replaced.yml")
@@ -135,14 +132,13 @@ profiles:
func TestFrameworkConfigurationShowFailsBeforeOutputForInvalidFinalDefinition(t *testing.T) {
root := t.TempDir()
frameworkFile := writeFrameworkSourceFile(t, root, "invalid.yml", `prompts:
unused:
"1":
messages:
- role: user
content: "{{"
output:
format: text
validation_mode: none
- id: unused
messages:
- role: user
content: "{{"
output:
format: text
validation_mode: none
`)
configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n", frameworkFile))
code, stdout, stderr := runCLICommand(t, configCommand, []string{"validate", "--config", configPath})
@@ -191,57 +187,59 @@ func TestFrameworkConfigurationCommandsRejectExtraArgumentsAndReportWriterErrors
func TestFrameworkConfigurationExplainUsesLiteralJSONPointersAndNativeHistory(t *testing.T) {
root := t.TempDir()
lower := writeFrameworkSourceFile(t, root, "lower.yml", `backends:
local:
endpoint: http://127.0.0.1:1/v1
api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE
extra_params:
token: backend-token-displaced-canary
password: backend-password-canary
visible_option: backend-visible-value
provider.option: dotted-visible-value
profiles:
base:
backend: local
model: inherited-model
api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE
extra_params:
token: profile-token-displaced-canary
password: profile-password-displaced-canary
visible_option: profile-visible-value
provider.option: dotted-visible-value
"team/summary":
extends: base
model: lower-model
prompts:
explainable:
"1":
default_profile: "team/summary"
messages:
- role: user
content: hello
output:
format: text
validation_mode: none
lower := writeFrameworkSourceFile(t, root, "lower.yml", `collections:
support:
backends:
- id: local
endpoint: http://127.0.0.1:1/v1
api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE
extra_params:
token: backend-token-displaced-canary
password: backend-password-canary
visible_option: backend-visible-value
provider.option: dotted-visible-value
profiles:
- id: summary
backend: local
model: lower-model
api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE
extra_params:
token: profile-token-displaced-canary
password: profile-password-displaced-canary
visible_option: profile-visible-value
provider.option: dotted-visible-value
prompts:
- id: explainable
default_profile: support/summary
messages:
- role: user
content: hello
output:
format: text
validation_mode: none
`)
upperRoot := t.TempDir()
upper := writeFrameworkSourceFile(t, upperRoot, "upper.yml", `backends:
local:
endpoint: http://127.0.0.1:1/v1
api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE
extra_params:
token: backend-token-effective-canary
password: backend-password-effective-canary
visible_option: backend-visible-value
provider.option: dotted-visible-value
profiles:
"team/summary":
model: effective-model
extra_params:
token: profile-token-effective-canary
password: profile-password-effective-canary
visible_option: profile-visible-value
provider.option: dotted-visible-value
upper := writeFrameworkSourceFile(t, upperRoot, "upper.yml", `collections:
support:
backends:
- id: local
endpoint: http://127.0.0.1:1/v1
api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE
extra_params:
token: backend-token-effective-canary
password: backend-password-effective-canary
visible_option: backend-visible-value
provider.option: dotted-visible-value
profiles:
- id: summary
backend: local
model: effective-model
api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE
extra_params:
token: profile-token-effective-canary
password: profile-password-effective-canary
visible_option: profile-visible-value
provider.option: dotted-visible-value
`)
configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n - file: %q\n", lower, upper))
const secret = "environment-value-must-never-be-emitted"
@@ -253,33 +251,33 @@ profiles:
unwant []string
}{
{
path: "/backends/local/extra_params/token",
path: "/collections/support/backends/0/extra_params/token",
unwant: []string{"backend-token-displaced-canary", "backend-token-effective-canary"},
},
{
path: "/backends/local/extra_params/password",
path: "/collections/support/backends/0/extra_params/password",
unwant: []string{"backend-password-displaced-canary", "backend-password-effective-canary"},
},
{
path: "/profiles/team~1summary/model",
want: []string{"effective-model", "inherited-model", "lower-model"},
path: "/collections/support/profiles/0/model",
want: []string{"effective-model", "lower-model"},
unwant: []string{},
},
{
path: "/profiles/team~1summary/extra_params/token",
path: "/collections/support/profiles/0/extra_params/token",
unwant: []string{"profile-token-displaced-canary", "profile-token-effective-canary"},
},
{
path: "/profiles/team~1summary/extra_params/password",
path: "/collections/support/profiles/0/extra_params/password",
unwant: []string{"profile-password-displaced-canary", "profile-password-effective-canary"},
},
{
path: "/profiles/team~1summary/api_key_env",
path: "/collections/support/profiles/0/api_key_env",
want: []string{"SCRIPTORIUM_EXPLAIN_REFERENCE"},
unwant: []string{secret},
},
{
path: "/profiles/team~1summary/extra_params/provider.option",
path: "/collections/support/profiles/0/extra_params/provider.option",
want: []string{"dotted-visible-value"},
unwant: []string{"provider/option"},
},
@@ -333,7 +331,7 @@ profiles:
}
profileCode, profileOutput, profileError := runCLICommand(t, inspectCommand, []string{
"profile", "--config", configPath, "--profile", "team/summary", "--format", "json",
"profile", "--config", configPath, "--profile", "support/summary", "--format", "json",
})
if profileCode != ExitOK || profileError != "" {
t.Fatalf("existing profile inspection failed: code=%d stdout=%q stderr=%q", profileCode, profileOutput, profileError)
@@ -342,14 +340,14 @@ profiles:
if err := json.Unmarshal([]byte(profileOutput), &profile); err != nil {
t.Fatalf("decode profile inspection: %v\n%s", err, profileOutput)
}
if profile.EffectiveModelParams.Model != "effective-model" || profile.EffectiveModelParams.ExtraParams["visible_option"] != "profile-visible-value" {
t.Fatalf("existing profile inspection changed configured provider data: %+v", profile.EffectiveModelParams)
if pointerValue(profile.ProfileModelParams.Model) != "effective-model" || pointerValue(profile.ProfileModelParams.ExtraParams)["visible_option"] != "profile-visible-value" {
t.Fatalf("existing profile inspection changed configured provider data: %+v", profile.ProfileModelParams)
}
if strings.Contains(profileOutput, secret) {
t.Fatalf("profile inspection emitted resolved environment data: %s", profileOutput)
}
code, rendered, stderr := runCLICommand(t, renderCommand, []string{"--config", configPath, "--prompt", "explainable"})
code, rendered, stderr := runCLICommand(t, renderCommand, []string{"--config", configPath, "--prompt", "support/explainable"})
if code != ExitOK || stderr != "" {
t.Fatalf("prepared output failed: code=%d stdout=%q stderr=%q", code, rendered, stderr)
}