Migrate CLI execution and terminal reports
This commit is contained in:
@@ -33,7 +33,6 @@ type runConfig struct {
|
||||
|
||||
promptkitSources []appconfig.SourceSelection
|
||||
promptID string
|
||||
promptVersion string
|
||||
profileID string
|
||||
sessionID string
|
||||
inputRaw listFlag
|
||||
@@ -170,16 +169,10 @@ func executeRun(cfg *runConfig, req promptkit.RunRequest, run func(context.Conte
|
||||
collector = executionreport.NewCollector()
|
||||
req.Observer = collector.Observe
|
||||
}
|
||||
runContext := context.Background()
|
||||
cancel := func() {}
|
||||
if cfg.runTimeout > 0 {
|
||||
runContext, cancel = context.WithTimeout(runContext, cfg.runTimeout)
|
||||
}
|
||||
res, runErr := run(runContext, req)
|
||||
contextErr := runContext.Err()
|
||||
cancel()
|
||||
req.OperationTimeout = cfg.runTimeout
|
||||
res, runErr := run(context.Background(), req)
|
||||
if runErr != nil {
|
||||
if errors.Is(contextErr, context.DeadlineExceeded) {
|
||||
if errors.Is(runErr, promptkit.ErrOperationTimeout) {
|
||||
fmt.Fprintln(stderr, "run error: execution deadline exceeded")
|
||||
} else {
|
||||
fmt.Fprintln(stderr, runErrorMessage(runErr))
|
||||
@@ -439,7 +432,6 @@ func registerExecutionRequestFlags(fs *flag.FlagSet, cfg *runConfig) {
|
||||
var promptkitSources promptKitSourceFlagValue
|
||||
fs.Var(&promptkitSources, "promptkit-source", "selected PromptKit source path (repeatable)")
|
||||
fs.StringVar(&cfg.promptID, "prompt", "", "prompt ID to run")
|
||||
fs.StringVar(&cfg.promptVersion, "prompt-version", "", "optional prompt definition version")
|
||||
fs.StringVar(&cfg.profileID, "profile", "", "optional execution profile ID; if omitted, prompt default_profile is used")
|
||||
fs.StringVar(&cfg.sessionID, "session-id", "", "optional session ID")
|
||||
fs.Var(&cfg.inputRaw, "input", "input mapping(s): name=path (repeatable, comma-separated)")
|
||||
@@ -630,7 +622,6 @@ func buildRunRequestFromConfig(cfg *runConfig) (promptkit.RunRequest, error) {
|
||||
}
|
||||
return promptkit.RunRequest{
|
||||
PromptID: cfg.promptID,
|
||||
PromptVersion: cfg.promptVersion,
|
||||
ProfileID: cfg.profileID,
|
||||
SessionID: cfg.sessionID,
|
||||
Inputs: inputs,
|
||||
@@ -709,9 +700,8 @@ func printSummary(stderr io.Writer, res *promptkit.RunResult) {
|
||||
if res == nil {
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(stderr, "prompt=%s@%s selected_profile=%s model=%s validation=%s mode=%s validation_errors=%d prompt_hash=%s inputs=%d usage=%d/%d/%d",
|
||||
fmt.Fprintf(stderr, "prompt=%s selected_profile=%s model=%s validation=%s mode=%s validation_errors=%d prompt_hash=%s inputs=%d usage=%d/%d/%d",
|
||||
res.PromptID,
|
||||
res.PromptVersion,
|
||||
res.SelectedProfileID,
|
||||
res.ModelName,
|
||||
res.Validation.Status,
|
||||
@@ -741,8 +731,8 @@ func runErrorMessage(err error) string {
|
||||
|
||||
func printUsage(w io.Writer) {
|
||||
fmt.Fprintln(w, "usage: scriptorium <run|render|serve|inspect|config> ...")
|
||||
fmt.Fprintln(w, " run: scriptorium run [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--schema-dir DIR] --prompt ID [--prompt-version VERSION] [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--out path] [--execution-report path] [--timeout 10m] [--run-timeout 10m] [--transient-retries N]")
|
||||
fmt.Fprintln(w, " render: scriptorium render [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--schema-dir DIR] --prompt ID [--prompt-version VERSION] [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--format text|json] [--out path] [--timeout 10m] [--transient-retries N]")
|
||||
fmt.Fprintln(w, " run: scriptorium run [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--schema-dir DIR] --prompt ID [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--out path] [--execution-report path] [--timeout 10m] [--run-timeout 10m] [--transient-retries N]")
|
||||
fmt.Fprintln(w, " render: scriptorium render [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--schema-dir DIR] --prompt ID [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--format text|json] [--out path] [--timeout 10m] [--transient-retries N]")
|
||||
fmt.Fprintf(w, " serve: scriptorium serve [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--addr %s] [--schema-dir DIR] [--artifact-root DIR] [--max-request-bytes N] [--max-artifact-bytes N] [--max-response-bytes N]\n", defaults.HTTPAddrDefault)
|
||||
fmt.Fprintln(w, " inspect prompt: scriptorium inspect prompt --prompt ID [--prompt-version VERSION] [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--schema-dir DIR] [--format text|json] [--out path]")
|
||||
fmt.Fprintln(w, " inspect profile: scriptorium inspect profile --profile ID [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--schema-dir DIR] [--format text|json] [--out path]")
|
||||
|
||||
@@ -110,9 +110,8 @@ func TestParseRunArgsFlagMapping(t *testing.T) {
|
||||
"--config", configPath,
|
||||
"--promptkit-source", lib.promptDir,
|
||||
"--promptkit-source", sourceFile,
|
||||
"--prompt", "prompt.a",
|
||||
"--prompt-version", "2",
|
||||
"--profile", "profile.a",
|
||||
"--prompt", "support/prompt.a",
|
||||
"--profile", "support/profile.a",
|
||||
"--session-id", "session-1",
|
||||
"--input", "a=b",
|
||||
"--llm-base-url", "http://x/v1",
|
||||
@@ -130,8 +129,8 @@ func TestParseRunArgsFlagMapping(t *testing.T) {
|
||||
if !reflect.DeepEqual(cfg.promptkitSources, []appconfig.SourceSelection{{Directory: lib.promptDir}, {File: sourceFile}}) {
|
||||
t.Fatalf("unexpected source selections: %#v", cfg.promptkitSources)
|
||||
}
|
||||
if cfg.promptID != "prompt.a" || cfg.promptVersion != "2" || cfg.profileID != "profile.a" {
|
||||
t.Fatalf("unexpected prompt/version/profile ids: %q %q %q", cfg.promptID, cfg.promptVersion, cfg.profileID)
|
||||
if cfg.promptID != "support/prompt.a" || cfg.profileID != "support/profile.a" {
|
||||
t.Fatalf("unexpected prompt/profile ids: %q %q", cfg.promptID, cfg.profileID)
|
||||
}
|
||||
if cfg.sessionID != "session-1" || cfg.reasoningEffort != "medium" || !cfg.reasoningEffortSet {
|
||||
t.Fatalf("unexpected session or reasoning configuration: %+v", cfg)
|
||||
@@ -286,7 +285,6 @@ func TestUsageIncludesExecutionAndServeFlags(t *testing.T) {
|
||||
|
||||
usage := stderr.String()
|
||||
for _, want := range []string{
|
||||
"--prompt-version VERSION",
|
||||
"--session-id ID",
|
||||
"--reasoning-effort VALUE",
|
||||
"--artifact-root",
|
||||
@@ -890,22 +888,59 @@ func TestBuildRunRequestPreservesReasoningEffortPresenceAndSessionID(t *testing.
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildRunRequestAllowsOmittedInputsAndMapsPromptVersion(t *testing.T) {
|
||||
func TestBuildRunRequestAllowsOmittedInputsAndPreservesQualifiedIDs(t *testing.T) {
|
||||
req, err := buildRunRequestFromConfig(&runConfig{
|
||||
promptID: "prompt-1",
|
||||
promptVersion: "2",
|
||||
promptID: "support/prompt-1",
|
||||
profileID: "support/profile-1",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected request without inputs to build, got %v", err)
|
||||
}
|
||||
if req.PromptVersion != "2" {
|
||||
t.Fatalf("expected prompt version to be mapped, got %q", req.PromptVersion)
|
||||
if req.PromptID != "support/prompt-1" || req.ProfileID != "support/profile-1" {
|
||||
t.Fatalf("qualified IDs changed: %+v", req)
|
||||
}
|
||||
if req.Inputs != nil {
|
||||
t.Fatalf("expected omitted inputs to remain nil, got %#v", req.Inputs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutionCommandsRejectPromptVersion(t *testing.T) {
|
||||
args := []string{"--prompt", "support/answer", "--prompt-version", "1"}
|
||||
if _, err := parseRunArgs(args); err == nil {
|
||||
t.Fatal("run accepted removed prompt version flag")
|
||||
}
|
||||
if _, err := parseRenderArgs(args); err == nil {
|
||||
t.Fatal("render accepted removed prompt version flag")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunOperationTimeoutAttribution(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
err error
|
||||
deadline bool
|
||||
}{
|
||||
{name: "operation budget", err: fmt.Errorf("wrapped: %w", promptkit.ErrOperationTimeout), deadline: true},
|
||||
{name: "provider deadline", err: fmt.Errorf("provider: %w", context.DeadlineExceeded)},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var stderr bytes.Buffer
|
||||
code := executeRun(&runConfig{runTimeout: 125 * time.Millisecond}, promptkit.RunRequest{}, func(ctx context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) {
|
||||
if req.OperationTimeout != 125*time.Millisecond {
|
||||
t.Errorf("operation timeout = %v", req.OperationTimeout)
|
||||
}
|
||||
if _, ok := ctx.Deadline(); ok {
|
||||
t.Error("adapter added a context deadline")
|
||||
}
|
||||
return nil, tc.err
|
||||
}, io.Discard, &stderr)
|
||||
if code != ExitRuntimeError || strings.Contains(stderr.String(), "execution deadline exceeded") != tc.deadline {
|
||||
t.Fatalf("unexpected outcome: code=%d stderr=%q", code, stderr.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunAndRenderParsingDoesNotRequireCallerSources(t *testing.T) {
|
||||
configPath := writeAppConfigFile(t, "{}\n")
|
||||
if cfg, err := parseRunArgs([]string{"--config", configPath, "--prompt", "p"}); err != nil || len(cfg.promptkitSources) != 0 {
|
||||
@@ -1455,7 +1490,6 @@ func TestPromptkitDefinitionsRenderThroughCLI(t *testing.T) {
|
||||
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||
"--config", configPath,
|
||||
"--prompt", "compat.complete",
|
||||
"--prompt-version", "1.0.0",
|
||||
"--input", "source=" + inputPath,
|
||||
"--var", "topic=testing",
|
||||
"--format", "json",
|
||||
@@ -1468,7 +1502,7 @@ func TestPromptkitDefinitionsRenderThroughCLI(t *testing.T) {
|
||||
if err := json.Unmarshal([]byte(stdout), &payload); err != nil {
|
||||
t.Fatalf("decode rendered fixture: %v\nbody=%s", err, stdout)
|
||||
}
|
||||
if payload["prompt_version"] != "1.0.0" || payload["selected_profile_id"] != "custom-derived" || payload["selected_backend_id"] != "fixture-custom" {
|
||||
if _, present := payload["prompt_version"]; present || payload["selected_profile_id"] != "custom-derived" || payload["selected_backend_id"] != "fixture-custom" {
|
||||
t.Fatalf("unexpected selected definition and target: %#v", payload)
|
||||
}
|
||||
if payload["session_id"] != "fixture-testing" {
|
||||
@@ -2337,7 +2371,7 @@ func TestRenderCommandExplicitProfileOverridesPromptDefault(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderCommandUsesDefinitionInputRulesAndPromptVersions(t *testing.T) {
|
||||
func TestRenderCommandUsesDefinitionInputRules(t *testing.T) {
|
||||
lib := newCLITestLibrary(t)
|
||||
writeProfileFile(t, lib.profileDir, "local", "http://127.0.0.1:1/v1", "model")
|
||||
|
||||
@@ -2351,25 +2385,6 @@ func TestRenderCommandUsesDefinitionInputRulesAndPromptVersions(t *testing.T) {
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)
|
||||
writePromptDefinition(t, lib.promptDir, "versioned.yaml", `prompts:
|
||||
versioned:
|
||||
"1":
|
||||
default_profile: local
|
||||
messages:
|
||||
- role: user
|
||||
content: "one"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
"2":
|
||||
default_profile: local
|
||||
messages:
|
||||
- role: user
|
||||
content: "two"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)
|
||||
writePromptDefinition(t, lib.promptDir, "optional.yaml", `prompts:
|
||||
optional:
|
||||
@@ -2418,9 +2433,6 @@ func TestRenderCommandUsesDefinitionInputRulesAndPromptVersions(t *testing.T) {
|
||||
wantCode int
|
||||
wantText string
|
||||
}{
|
||||
{name: "sole version selected when omitted", args: []string{"--prompt", "sole"}, wantCode: ExitOK, wantText: "prompt_version: 1"},
|
||||
{name: "explicit version selected", args: []string{"--prompt", "versioned", "--prompt-version", "2"}, wantCode: ExitOK, wantText: "prompt_version: 2"},
|
||||
{name: "multiple versions require selection", args: []string{"--prompt", "versioned"}, wantCode: ExitRuntimeError, wantText: "version is ambiguous"},
|
||||
{name: "no declared inputs", args: []string{"--prompt", "sole"}, wantCode: ExitOK, wantText: "prompt: sole"},
|
||||
{name: "optional input omitted", args: []string{"--prompt", "optional"}, wantCode: ExitOK, wantText: "prompt: optional"},
|
||||
{name: "required input omitted", args: []string{"--prompt", "required"}, wantCode: ExitRuntimeError, wantText: "required"},
|
||||
@@ -2563,7 +2575,6 @@ func TestWriteOutputAndSummaryUseSeparateWriters(t *testing.T) {
|
||||
}
|
||||
printSummary(&stderr, &promptkit.RunResult{
|
||||
PromptID: "p",
|
||||
PromptVersion: "1",
|
||||
SelectedProfileID: "exec",
|
||||
ModelName: "m",
|
||||
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic},
|
||||
@@ -2574,7 +2585,7 @@ func TestWriteOutputAndSummaryUseSeparateWriters(t *testing.T) {
|
||||
if stdout.String() != "artifact-body" {
|
||||
t.Fatalf("expected artifact output on stdout, got %q", stdout.String())
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "prompt=p@1") {
|
||||
if !strings.Contains(stderr.String(), "prompt=p ") || strings.Contains(stderr.String(), "prompt_version") {
|
||||
t.Fatalf("expected summary on stderr, got %q", stderr.String())
|
||||
}
|
||||
if strings.Contains(stderr.String(), "cached_tokens=") || strings.Contains(stderr.String(), "cache_write_tokens=") {
|
||||
@@ -2590,7 +2601,6 @@ func TestPrintSummaryIncludesCacheUsageWhenPresent(t *testing.T) {
|
||||
|
||||
printSummary(&stderr, &promptkit.RunResult{
|
||||
PromptID: "p",
|
||||
PromptVersion: "1",
|
||||
SelectedProfileID: "exec",
|
||||
ModelName: "m",
|
||||
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic},
|
||||
@@ -2621,7 +2631,6 @@ func TestPrintSummaryIncludesBackendWhenPresent(t *testing.T) {
|
||||
var stderr bytes.Buffer
|
||||
printSummary(&stderr, &promptkit.RunResult{
|
||||
PromptID: "p",
|
||||
PromptVersion: "1",
|
||||
SelectedProfileID: "exec",
|
||||
SelectedBackendID: "local",
|
||||
ModelName: "m",
|
||||
|
||||
Reference in New Issue
Block a user