Adapt inspection to native PromptKit records

This commit is contained in:
2026-11-18 19:08:41 +00:00
parent 77f168c660
commit de7c866662
8 changed files with 306 additions and 270 deletions

View File

@@ -27,11 +27,11 @@ func TestFrameworkConfigurationCommandsNeedNoPromptOrCredential(t *testing.T) {
root := t.TempDir()
frameworkFile := filepath.Join(root, "framework.yml")
writeFrameworkSourceFile(t, root, "framework.yml", `backends:
local:
- id: local
endpoint: http://127.0.0.1:1/v1
api_key_env: SCRIPTORIUM_FRAMEWORK_KEY
profiles:
protected:
- id: protected
backend: local
model: offline-model
api_key_required: true
@@ -59,24 +59,21 @@ func TestFrameworkConfigurationSourcesProjectsOrderedNativeRecords(t *testing.T)
if err := os.MkdirAll(importDir, 0o755); err != nil {
t.Fatal(err)
}
mainPath := writeFrameworkSourceFile(t, root, "main.yml", `configkit:
mainPath := writeFrameworkSourceFile(t, root, "main.yml", `config:
imports:
- path: catalog/fragment.yml
required: true
backends:
local:
- id: local
endpoint: http://127.0.0.1:1/v1
profiles:
selected:
backend: local
model: selected-model
`)
fragmentPath := writeFrameworkSourceFile(t, importDir, "fragment.yml", `profiles:
imported:
- id: imported
model: imported-model
`)
directory := t.TempDir()
directoryPath := writeFrameworkSourceFile(t, directory, "directory.yml", `profiles:
directory:
- id: directory
model: directory-model
`)
configuredMissing := filepath.Join(root, "configured-but-replaced.yml")
@@ -135,14 +132,13 @@ profiles:
func TestFrameworkConfigurationShowFailsBeforeOutputForInvalidFinalDefinition(t *testing.T) {
root := t.TempDir()
frameworkFile := writeFrameworkSourceFile(t, root, "invalid.yml", `prompts:
unused:
"1":
messages:
- role: user
content: "{{"
output:
format: text
validation_mode: none
- id: unused
messages:
- role: user
content: "{{"
output:
format: text
validation_mode: none
`)
configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n", frameworkFile))
code, stdout, stderr := runCLICommand(t, configCommand, []string{"validate", "--config", configPath})
@@ -191,57 +187,59 @@ func TestFrameworkConfigurationCommandsRejectExtraArgumentsAndReportWriterErrors
func TestFrameworkConfigurationExplainUsesLiteralJSONPointersAndNativeHistory(t *testing.T) {
root := t.TempDir()
lower := writeFrameworkSourceFile(t, root, "lower.yml", `backends:
local:
endpoint: http://127.0.0.1:1/v1
api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE
extra_params:
token: backend-token-displaced-canary
password: backend-password-canary
visible_option: backend-visible-value
provider.option: dotted-visible-value
profiles:
base:
backend: local
model: inherited-model
api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE
extra_params:
token: profile-token-displaced-canary
password: profile-password-displaced-canary
visible_option: profile-visible-value
provider.option: dotted-visible-value
"team/summary":
extends: base
model: lower-model
prompts:
explainable:
"1":
default_profile: "team/summary"
messages:
- role: user
content: hello
output:
format: text
validation_mode: none
lower := writeFrameworkSourceFile(t, root, "lower.yml", `collections:
support:
backends:
- id: local
endpoint: http://127.0.0.1:1/v1
api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE
extra_params:
token: backend-token-displaced-canary
password: backend-password-canary
visible_option: backend-visible-value
provider.option: dotted-visible-value
profiles:
- id: summary
backend: local
model: lower-model
api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE
extra_params:
token: profile-token-displaced-canary
password: profile-password-displaced-canary
visible_option: profile-visible-value
provider.option: dotted-visible-value
prompts:
- id: explainable
default_profile: support/summary
messages:
- role: user
content: hello
output:
format: text
validation_mode: none
`)
upperRoot := t.TempDir()
upper := writeFrameworkSourceFile(t, upperRoot, "upper.yml", `backends:
local:
endpoint: http://127.0.0.1:1/v1
api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE
extra_params:
token: backend-token-effective-canary
password: backend-password-effective-canary
visible_option: backend-visible-value
provider.option: dotted-visible-value
profiles:
"team/summary":
model: effective-model
extra_params:
token: profile-token-effective-canary
password: profile-password-effective-canary
visible_option: profile-visible-value
provider.option: dotted-visible-value
upper := writeFrameworkSourceFile(t, upperRoot, "upper.yml", `collections:
support:
backends:
- id: local
endpoint: http://127.0.0.1:1/v1
api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE
extra_params:
token: backend-token-effective-canary
password: backend-password-effective-canary
visible_option: backend-visible-value
provider.option: dotted-visible-value
profiles:
- id: summary
backend: local
model: effective-model
api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE
extra_params:
token: profile-token-effective-canary
password: profile-password-effective-canary
visible_option: profile-visible-value
provider.option: dotted-visible-value
`)
configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n - file: %q\n", lower, upper))
const secret = "environment-value-must-never-be-emitted"
@@ -253,33 +251,33 @@ profiles:
unwant []string
}{
{
path: "/backends/local/extra_params/token",
path: "/collections/support/backends/0/extra_params/token",
unwant: []string{"backend-token-displaced-canary", "backend-token-effective-canary"},
},
{
path: "/backends/local/extra_params/password",
path: "/collections/support/backends/0/extra_params/password",
unwant: []string{"backend-password-displaced-canary", "backend-password-effective-canary"},
},
{
path: "/profiles/team~1summary/model",
want: []string{"effective-model", "inherited-model", "lower-model"},
path: "/collections/support/profiles/0/model",
want: []string{"effective-model", "lower-model"},
unwant: []string{},
},
{
path: "/profiles/team~1summary/extra_params/token",
path: "/collections/support/profiles/0/extra_params/token",
unwant: []string{"profile-token-displaced-canary", "profile-token-effective-canary"},
},
{
path: "/profiles/team~1summary/extra_params/password",
path: "/collections/support/profiles/0/extra_params/password",
unwant: []string{"profile-password-displaced-canary", "profile-password-effective-canary"},
},
{
path: "/profiles/team~1summary/api_key_env",
path: "/collections/support/profiles/0/api_key_env",
want: []string{"SCRIPTORIUM_EXPLAIN_REFERENCE"},
unwant: []string{secret},
},
{
path: "/profiles/team~1summary/extra_params/provider.option",
path: "/collections/support/profiles/0/extra_params/provider.option",
want: []string{"dotted-visible-value"},
unwant: []string{"provider/option"},
},
@@ -333,7 +331,7 @@ profiles:
}
profileCode, profileOutput, profileError := runCLICommand(t, inspectCommand, []string{
"profile", "--config", configPath, "--profile", "team/summary", "--format", "json",
"profile", "--config", configPath, "--profile", "support/summary", "--format", "json",
})
if profileCode != ExitOK || profileError != "" {
t.Fatalf("existing profile inspection failed: code=%d stdout=%q stderr=%q", profileCode, profileOutput, profileError)
@@ -342,14 +340,14 @@ profiles:
if err := json.Unmarshal([]byte(profileOutput), &profile); err != nil {
t.Fatalf("decode profile inspection: %v\n%s", err, profileOutput)
}
if profile.EffectiveModelParams.Model != "effective-model" || profile.EffectiveModelParams.ExtraParams["visible_option"] != "profile-visible-value" {
t.Fatalf("existing profile inspection changed configured provider data: %+v", profile.EffectiveModelParams)
if pointerValue(profile.ProfileModelParams.Model) != "effective-model" || pointerValue(profile.ProfileModelParams.ExtraParams)["visible_option"] != "profile-visible-value" {
t.Fatalf("existing profile inspection changed configured provider data: %+v", profile.ProfileModelParams)
}
if strings.Contains(profileOutput, secret) {
t.Fatalf("profile inspection emitted resolved environment data: %s", profileOutput)
}
code, rendered, stderr := runCLICommand(t, renderCommand, []string{"--config", configPath, "--prompt", "explainable"})
code, rendered, stderr := runCLICommand(t, renderCommand, []string{"--config", configPath, "--prompt", "support/explainable"})
if code != ExitOK || stderr != "" {
t.Fatalf("prepared output failed: code=%d stdout=%q stderr=%q", code, rendered, stderr)
}

View File

@@ -51,8 +51,8 @@ func TestFrameworkSourceGroupsComposeAndValidateEffectiveDefinitions(t *testing.
if err != nil {
t.Fatalf("inspect profile after later source replacement: %v", err)
}
if inspection.EffectiveModelParams.Endpoint != "http://127.0.0.1:1/v1" || inspection.EffectiveModelParams.Model != "upper-model" || inspection.EffectiveModelParams.Temperature != 0.5 {
t.Fatalf("later source did not replace lower values before validation: %+v", inspection.EffectiveModelParams)
if pointerValue(inspection.Endpoint) != "http://127.0.0.1:1/v1" || pointerValue(inspection.Model) != "upper-model" || pointerValue(inspection.Temperature) != 0.5 {
t.Fatalf("later source did not replace lower values before validation: %+v", inspection)
}
}

View File

@@ -14,12 +14,12 @@ import (
)
type promptInspectionConfig struct {
configPath, promptID, promptVersion, schemaDir, outputPath string
configOverlay string
configExplicit bool
outputFormat appformat.OutputFormat
overrides []configkit.Override
diagnostics []configkit.Diagnostic
configPath, promptID, schemaDir, outputPath string
configOverlay string
configExplicit bool
outputFormat appformat.OutputFormat
overrides []configkit.Override
diagnostics []configkit.Diagnostic
}
type profileInspectionConfig struct {
@@ -68,7 +68,7 @@ func inspectCommand(args []string, stdout, stderr io.Writer) int {
fmt.Fprintf(stderr, "engine diagnostic write error: %v\n", err)
return ExitRuntimeError
}
inspection, err := engine.InspectPrompt(context.Background(), cfg.promptID, cfg.promptVersion)
inspection, err := engine.InspectPrompt(context.Background(), cfg.promptID)
if err != nil {
fmt.Fprintf(stderr, "inspect error: %v\n", err)
return ExitRuntimeError
@@ -177,7 +177,6 @@ func parsePromptInspectionArgs(args []string) (*promptInspectionConfig, error) {
fs.Var(&promptkitSources, "promptkit-source", "selected PromptKit source path (repeatable)")
fs.StringVar(&cfg.schemaDir, "schema-dir", "", "base directory for validation schemas")
fs.StringVar(&cfg.promptID, "prompt", "", "prompt ID to inspect")
fs.StringVar(&cfg.promptVersion, "prompt-version", "", "optional prompt version")
rawFormat := ""
fs.StringVar(&rawFormat, "format", "", "output format: text or json")
fs.StringVar(&cfg.outputPath, "out", "", "optional output file path")

View File

@@ -734,7 +734,7 @@ func printUsage(w io.Writer) {
fmt.Fprintln(w, " run: scriptorium run [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--schema-dir DIR] --prompt ID [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--out path] [--execution-report path] [--timeout 10m] [--run-timeout 10m] [--transient-retries N]")
fmt.Fprintln(w, " render: scriptorium render [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--schema-dir DIR] --prompt ID [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--format text|json] [--out path] [--timeout 10m] [--transient-retries N]")
fmt.Fprintf(w, " serve: scriptorium serve [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--addr %s] [--schema-dir DIR] [--artifact-root DIR] [--max-request-bytes N] [--max-artifact-bytes N] [--max-response-bytes N]\n", defaults.HTTPAddrDefault)
fmt.Fprintln(w, " inspect prompt: scriptorium inspect prompt --prompt ID [--prompt-version VERSION] [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--schema-dir DIR] [--format text|json] [--out path]")
fmt.Fprintln(w, " inspect prompt: scriptorium inspect prompt --prompt ID [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--schema-dir DIR] [--format text|json] [--out path]")
fmt.Fprintln(w, " inspect profile: scriptorium inspect profile --profile ID [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--schema-dir DIR] [--format text|json] [--out path]")
fmt.Fprintln(w, " inspect configuration: scriptorium inspect configuration <validate|show|sources|explain> [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--schema-dir DIR] [--path JSON_POINTER (required for explain)]")
fmt.Fprintln(w, " config: scriptorium config <validate|show|sources|explain> [--config PATH] [--config-overlay NAME] [application overrides] [--path /pointer for explain]")

View File

@@ -1442,26 +1442,29 @@ func TestRenderCommandRejectsOverlongSessionID(t *testing.T) {
func TestInspectPromptCommandFormatsDefinitionWithoutProfileOrGeneration(t *testing.T) {
lib := newCLITestLibrary(t)
writePromptDefinition(t, lib.promptDir, "inspect.yaml", `prompts:
inspect:
"1":
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
- id: inspect
inputs: []
messages:
- role: user
content: "private message"
output:
format: text
validation_mode: none
`)
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{"prompt", "--config", lib.configPath, "--promptkit-source", lib.promptDir, "--prompt", "inspect", "--format", "json"})
if code != ExitOK {
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
}
if !strings.Contains(stdout, `"prompt_id": "inspect"`) || !strings.Contains(stdout, `"inputs": []`) {
if !strings.Contains(stdout, `"prompt_id": "inspect"`) || !strings.Contains(stdout, `"inputs": []`) || strings.Contains(stdout, "private message") || strings.Contains(stdout, "prompt_version") {
t.Fatalf("unexpected inspection output: %s", stdout)
}
code, stdout, stderr = runCLICommand(t, inspectCommand, []string{"prompt", "--config", lib.configPath, "--promptkit-source", lib.promptDir, "--prompt", "missing"})
if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "inspect error") {
t.Fatalf("expected failed inspection without output, got code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
if _, err := parsePromptInspectionArgs([]string{"--prompt", "inspect", "--prompt-version", "1"}); err == nil {
t.Fatal("inspection accepted removed prompt version flag")
}
}
func TestInspectionParsersPreserveExplicitEmptyConfigPath(t *testing.T) {
@@ -1821,28 +1824,34 @@ func TestExecutionReportPathValidationAndPreInvocationFailure(t *testing.T) {
func intPointer(value int) *int { return &value }
func TestPromptkitPromptInspectionSelectsVersionsAndContracts(t *testing.T) {
func pointerValue[T any](value *T) T {
if value == nil {
var zero T
return zero
}
return *value
}
func TestPromptkitPromptInspectionSelectsIDsAndContracts(t *testing.T) {
fixtureRoot := definitionFixtureRoot(t)
configPath := writeDefinitionFixtureConfig(t, fixtureRoot)
tests := []struct {
promptID string
version string
format string
mode string
}{
{promptID: "compat.complete", version: "1.0.0", format: "json", mode: "json_schema"},
{promptID: "compat.complete", version: "2.0.0", format: "text", mode: "basic"},
{promptID: "compat.json", version: "1.0.0", format: "json", mode: "json"},
{promptID: "compat.none", version: "1.0.0", format: "text", mode: "none"},
{promptID: "compat.complete", format: "json", mode: "json_schema"},
{promptID: "compat.complete-text", format: "text", mode: "basic"},
{promptID: "compat.json", format: "json", mode: "json"},
{promptID: "compat.none", format: "text", mode: "none"},
}
for _, tc := range tests {
t.Run(tc.promptID+"@"+tc.version, func(t *testing.T) {
t.Run(tc.promptID, func(t *testing.T) {
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
"prompt",
"--config", configPath,
"--prompt", tc.promptID,
"--prompt-version", tc.version,
"--format", "json",
})
if code != ExitOK {
@@ -1852,7 +1861,7 @@ func TestPromptkitPromptInspectionSelectsVersionsAndContracts(t *testing.T) {
if err := json.Unmarshal([]byte(stdout), &inspection); err != nil {
t.Fatalf("decode prompt inspection: %v\nbody=%s", err, stdout)
}
if inspection.PromptVersion != tc.version || inspection.OutputContract.Format != tc.format || inspection.OutputContract.ValidationMode != tc.mode {
if inspection.PromptID != tc.promptID || inspection.OutputContract.Format != tc.format || inspection.OutputContract.ValidationMode != tc.mode {
t.Fatalf("unexpected prompt inspection: %+v", inspection)
}
})
@@ -1891,12 +1900,12 @@ func TestPromptkitProfileInspectionResolvesSupportedTargets(t *testing.T) {
if err := json.Unmarshal([]byte(stdout), &inspection); err != nil {
t.Fatalf("decode profile inspection: %v\nbody=%s", err, stdout)
}
params := inspection.EffectiveModelParams
if inspection.ProfileID != tc.profileID || params.BackendID != tc.wantBackend || params.Model != tc.wantModel || params.APIKeyEnv != tc.wantAPIKeyEnv {
params := inspection.ProfileModelParams
if inspection.ProfileID != tc.profileID || pointerValue(params.BackendID) != tc.wantBackend || pointerValue(params.Model) != tc.wantModel || pointerValue(params.APIKeyEnv) != tc.wantAPIKeyEnv {
t.Fatalf("unexpected effective profile: %+v", inspection)
}
if tc.profileID == "custom-derived" {
if params.ServiceTier != "flex" || params.ReasoningEffort != "high" || params.TimeoutSeconds != 45 || len(params.ExtraParams) == 0 {
if pointerValue(params.ServiceTier) != "flex" || pointerValue(params.ReasoningEffort) != "high" || pointerValue(params.TimeoutSeconds) != 45 || len(pointerValue(params.ExtraParams)) == 0 {
t.Fatalf("inherited profile controls were not resolved: %+v", params)
}
}
@@ -1948,8 +1957,8 @@ prompts:
if err := json.Unmarshal([]byte(stdout), &inspection); err != nil {
t.Fatalf("decode combined-source profile inspection: %v", err)
}
if inspection.EffectiveModelParams.BackendID != "fixture" || inspection.EffectiveModelParams.Model != "fixture-model" {
t.Fatalf("combined source did not provide the selected profile: %+v", inspection.EffectiveModelParams)
if pointerValue(inspection.ProfileModelParams.BackendID) != "fixture" || pointerValue(inspection.ProfileModelParams.Model) != "fixture-model" {
t.Fatalf("combined source did not provide the selected profile: %+v", inspection.ProfileModelParams)
}
invalidCombined := strings.Replace(combined, `content: "valid unused prompt"`, `content: "{{"`, 1)
@@ -1989,11 +1998,11 @@ func TestMaintainedProfilesResolveAcrossSelectionAndSourcePrecedence(t *testing.
if err := json.Unmarshal([]byte(stdout), &inspection); err != nil {
t.Fatal(err)
}
if inspection.ProfileID != profileID || inspection.EffectiveModelParams.BackendID != "openrouter" || inspection.EffectiveModelParams.Model == "" {
if inspection.ProfileID != profileID || pointerValue(inspection.ProfileModelParams.BackendID) != "openrouter" || pointerValue(inspection.ProfileModelParams.Model) == "" {
t.Fatalf("maintained profile did not resolve: %+v", inspection)
}
if profileID == "promptkit-openrouter-auto" && inspection.EffectiveModelParams.Model != "openrouter/auto" {
t.Fatalf("unexpected auto target: %+v", inspection.EffectiveModelParams)
if profileID == "promptkit-openrouter-auto" && pointerValue(inspection.ProfileModelParams.Model) != "openrouter/auto" {
t.Fatalf("unexpected auto target: %+v", inspection.ProfileModelParams)
}
})
}
@@ -2040,7 +2049,7 @@ func TestMaintainedProfilesResolveAcrossSelectionAndSourcePrecedence(t *testing.
if err := json.Unmarshal([]byte(stdout), &inspection); err != nil {
t.Fatal(err)
}
if inspection.EffectiveModelParams.BackendID != "openrouter" || inspection.EffectiveModelParams.Model != tc.model {
if pointerValue(inspection.ProfileModelParams.BackendID) != "openrouter" || pointerValue(inspection.ProfileModelParams.Model) != tc.model {
t.Fatalf("unexpected effective target: %+v", inspection)
}
})
@@ -2071,7 +2080,7 @@ func TestProfileInspectionHonorsDirectoryPrecedenceOutputAndFailures(t *testing.
if err != nil {
t.Fatalf("read profile inspection output: %v", err)
}
if !strings.Contains(string(output), `"model": "override-model"`) || !strings.Contains(string(output), `"backend_id": ""`) {
if !strings.Contains(string(output), `"model": "override-model"`) || strings.Contains(string(output), `"backend_id"`) {
t.Fatalf("profile source override was not used: %s", output)
}
@@ -2159,8 +2168,8 @@ func TestConfiguredBackendValidationComesFromPromptkit(t *testing.T) {
if err := json.Unmarshal([]byte(stdout), &inspection); err != nil {
t.Fatalf("decode built-in profile inspection: %v", err)
}
if inspection.EffectiveModelParams.BackendID != "openrouter" || inspection.EffectiveModelParams.Endpoint != "http://localhost:11434/v1" {
t.Fatalf("caller overlay did not update the built-in backend: %+v", inspection.EffectiveModelParams)
if pointerValue(inspection.ProfileModelParams.BackendID) != "openrouter" || inspection.ProfileModelParams.Endpoint != nil {
t.Fatalf("profile inspection unexpectedly completed backend settings: %+v", inspection.ProfileModelParams)
}
})
}