Implement positional prompt execution with native reporting and outcomes

This commit is contained in:
2026-11-20 04:42:59 +00:00
parent f9b5121432
commit 6f39bca841
6 changed files with 437 additions and 5 deletions

View File

@@ -445,6 +445,26 @@ upstream analysis or rendering suite.
## Stage 5: Implement Positional Generation, Request Mapping, And Outcomes
**Status: Complete.** `run PROMPT_ID` uses native resource/input/execution
bindings and equivalent command-scoped aliases, typed namespace strings and
public engine views, and one `Engine.Run` dispatch with the invocation context.
Operation durations pass through unchanged. Main results use the native reporter
with the artifact-only byte-preserving text template; structured modes retain
the native result. Completed content-validation failure is domain status 4;
resultless operational failures remain status 1 with preserved typed causes and
safe explanations. Configuration classification uses native diagnostic evidence.
Verified on Go 1.27.4 with focused controlled-provider integration tests, race
checks, and vet: canonical/preferred aliases, input file/literal precedence,
commas and additional equals signs, explicit numeric zero and empty reasoning,
service tier and extra parameters, independent namespaces and exact qualified
IDs, session context, raw stdout, native JSON/YAML results, verbose diagnostic
stream separation, invalid-content delivery/status 4, and operational failures
without a result or provider-payload disclosure. Typed-error/nil-result checks
and the shared Narratio discovery/preparation/generation fixture cover mapping
and context consistency. Actual timeout/cancellation lifecycle checks follow in
Stage 7; full-package compilation still awaits Stage 8 legacy removal.
**Primary files:** the new application `run` declaration, request/outcome
helpers in `internal/adapter/cli`, and run integration tests.

View File

@@ -22,13 +22,13 @@ func Execute(ctx context.Context, identity commandkit.Identity, args []string, s
func newApp(identity commandkit.Identity) (*commandkit.App, error) {
definitions := promptSuite()
declarations := []commandkit.Declaration{
commandkit.NewCommand(commandkit.CommandSpec[struct{}]{Name: "run", Summary: "Run a stored prompt.", Run: commandNotImplemented}),
commandkit.NewCommand(commandkit.CommandSpec[serveInput]{Name: "serve", Summary: "Serve the HTTP API.", Flags: serveFlags(), Run: func(commandkit.Invocation, serveInput) commandkit.Outcome {
return commandkit.Failure("Server startup is not implemented.")
}}),
configurationCommand(),
commandkit.Completion("scriptorium"),
}
declarations = append(declarations, runDeclarations()...)
declarations = append(declarations, ckprompt.Declarations(definitions)...)
return commandkit.New(identity, declarations...)
}
@@ -42,7 +42,3 @@ func serveFlags() []commandkit.Flag[serveInput] {
flags := configurationSelectors(func(input *serveInput) *ckconfig.Selection { return &input.Selection })
return append(flags, serverSelectors(func(input *serveInput) *applicationInput { return &input.Application })...)
}
func commandNotImplemented(commandkit.Invocation, struct{}) commandkit.Outcome {
return commandkit.Failure("Application command is not implemented.")
}

View File

@@ -30,6 +30,13 @@ func nativePromptFixture(t *testing.T, endpoint string) {
- id: writer
backend: support/local
model: controlled-model
temperature: 0.8
max_tokens: 99
top_p: 0.9
timeout_seconds: 4
reasoning_effort: high
extra_params:
stale: true
- id: protected
backend: support/local
model: controlled-model

View File

@@ -0,0 +1,99 @@
package cli
import (
"text/template"
"time"
"go.maximumdirect.net/ckconfig"
"go.maximumdirect.net/ckprompt"
"go.maximumdirect.net/commandkit"
"go.maximumdirect.net/promptkit"
)
type runInput struct {
Selection ckconfig.Selection
PromptID string
Resources ckprompt.ResourceInput
Inputs ckprompt.PromptInputs
Execution ckprompt.ExecutionInput
Namespaces promptkit.Namespaces
OperationTimeout time.Duration
}
var artifactTextTemplate = template.Must(commandkit.ParseTextTemplate("artifact", `{{printf "%s" .Artifact.Body}}`, nil))
func runDeclarations() []commandkit.Declaration {
resources := ckprompt.NewResourceFlags(func(input *runInput) *ckprompt.ResourceInput { return &input.Resources })
inputs := ckprompt.NewPromptInputFlags(func(input *runInput) *ckprompt.PromptInputs { return &input.Inputs })
execution := ckprompt.NewExecutionFlags(func(input *runInput) *ckprompt.ExecutionInput { return &input.Execution })
flags := configurationSelectors(func(input *runInput) *ckconfig.Selection { return &input.Selection })
flags = append(flags, resources.Profile, resources.SessionID)
flags = append(flags, inputs.All()...)
flags = append(flags, execution.All()...)
flags = append(flags, commandkit.ValueFlag(commandkit.FlagMeta{Name: "run-timeout", Usage: "Whole-operation timeout as a Go duration."}, (*time.Duration)(nil), time.ParseDuration,
func(input *runInput, value commandkit.Supplied[time.Duration]) { input.OperationTimeout = value.Value }))
for _, namespace := range []struct {
name string
field func(*runInput) *string
}{
{"prompt", func(input *runInput) *string { return &input.Namespaces.Prompt }},
{"profile", func(input *runInput) *string { return &input.Namespaces.Profile }},
{"backend", func(input *runInput) *string { return &input.Namespaces.Backend }},
} {
flags = append(flags, commandkit.ValueFlag(commandkit.FlagMeta{Name: "llm-" + namespace.name + "-namespace", Usage: "Select the " + namespace.name + " namespace."}, (*string)(nil), parseString,
func(input *runInput, value commandkit.Supplied[string]) { *namespace.field(input) = value.Value }))
}
command := commandkit.NewCommand(commandkit.CommandSpec[runInput]{
Name: "run", Summary: "Run a stored prompt.", Flags: flags,
Arguments: []commandkit.Argument[runInput]{commandkit.ValueArgument(commandkit.ArgumentMeta{Name: "PROMPT_ID", Usage: "Stored prompt identifier."}, true, parseString,
func(input *runInput, value commandkit.Supplied[string]) { input.PromptID = value.Value })},
Run: runPrompt,
})
declarations := []commandkit.Declaration{command}
for _, name := range []string{"profile", "session-id", "var", "input-text", "input-file", "endpoint", "model", "temperature", "max-tokens", "top-p", "timeout-seconds", "service-tier", "reasoning-effort", "api-key-env", "transient-retries", "extra-params", "prompt-namespace", "profile-namespace", "backend-namespace"} {
declarations = append(declarations, commandkit.FlagAliasFor("run", "--"+name, "--llm-"+name))
}
for _, name := range []string{"input", "llm-input"} {
declarations = append(declarations, commandkit.FlagAliasFor("run", "--"+name, "--llm-input-text"))
}
return declarations
}
func requestFromRunInput(input runInput) promptkit.RunRequest {
return promptkit.RunRequest{
PromptID: input.PromptID, ProfileID: input.Resources.Profile.Value, SessionID: input.Resources.SessionID.Value,
Inputs: input.Inputs.Inputs, Vars: input.Inputs.Vars, Execution: input.Execution.Execution,
TransientRetries: input.Execution.TransientRetries, OperationTimeout: input.OperationTimeout,
}
}
func runPrompt(invocation commandkit.Invocation, input runInput) commandkit.Outcome {
state := &acquisitionState{}
engine, err := loadEngine(invocation.Context, input.Selection, applicationInput{}, state)
acquired, err := attachApplicationEvidence(commandkit.Acquired[*promptkit.Engine]{Value: engine}, err, state, 0, false)
queueDiagnostics(invocation.Reporter(), acquired.Diagnostics)
if err != nil {
return requestFailure(err, acquired.Explanation)
}
inspection, err := engine.InspectConfiguration()
if err != nil {
return requestFailure(err, acquired.Explanation)
}
diagnostics, explanation := ckconfig.AdaptDiagnostics("PromptKit configuration:\n", inspection.Sources(), inspection.Diagnostics())
queueDiagnostics(invocation.Reporter(), diagnostics)
engine, err = engine.WithNamespaces(input.Namespaces)
if err != nil {
return requestFailure(err, explanation)
}
result, err := engine.Run(invocation.Context, requestFromRunInput(input))
if err != nil {
return requestFailure(err, explanation)
}
outcome := completedRunOutcome(result)
if result == nil {
return outcome
}
_ = invocation.Reporter().Report(commandkit.Report{Value: result, TextTemplate: artifactTextTemplate})
_ = invocation.Reporter().Info("Prompt execution completed.")
return outcome
}

View File

@@ -0,0 +1,234 @@
package cli
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
"go.maximumdirect.net/commandkit"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
"go.yaml.in/yaml/v4"
)
func TestPositionalRunMapsNativeBindingsAndNamespaces(t *testing.T) {
requests := make(chan map[string]any, 1)
const content = " \x00<artifact>&\n尾-no-newline"
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var request map[string]any
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
t.Error(err)
}
request["authorization"] = r.Header.Get("Authorization")
select {
case requests <- request:
default:
t.Error("unexpected duplicate provider call")
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{"choices": []any{map[string]any{"message": map[string]any{"role": "assistant", "content": content}}}})
}))
defer server.Close()
nativePromptFixture(t, "http://127.0.0.1:1/v1")
t.Setenv("SCRIPTORIUM_CONTROLLED_KEY", "synthetic-credential-canary")
writeNativeFile(t, "note,part=1.txt", "file,transcript=body")
args := []string{"run", "summary", "--config=app.yml", "--prompt-namespace=support", "--profile=writer", "--profile-namespace=support", "--backend-namespace=review", "--session-id=caller-session",
"--input-file=transcript=note,part=1.txt", "--input-text=transcript=first,text=1", "--input-file=transcript=note,part=1.txt", "--input-file=unlisted=missing.txt", "--input=unlisted=last,literal=2", "--var=mode=", "--var=title=Title,with=equals",
"--endpoint=" + server.URL + "/v1", "--model=chosen-model", "--temperature=0", "--max-tokens=0", "--top-p=0", "--timeout-seconds=0", "--reasoning-effort=", "--service-tier=priority", `--extra-params={"seed":0,"test_payload":[0,"x"]}`, "--api-key-env=SCRIPTORIUM_CONTROLLED_KEY", "--transient-retries=0", "--run-timeout=5s", "--format=json"}
outcome, output, diagnostics := executeNative(t, args...)
if outcome.Status != 0 || strings.Contains(output+diagnostics, "synthetic-credential-canary") {
t.Fatalf("run: %+v %q %q", outcome, output, diagnostics)
}
var result promptkit.RunResult
if err := json.Unmarshal([]byte(output), &result); err != nil {
t.Fatal(err)
}
if string(result.Artifact.Body) != content || result.PromptID != "support/summary" || result.SelectedProfileID != "support/writer" || result.SelectedBackendID != "support/local" || result.SessionID != "caller-session" {
t.Fatalf("native result/context: %+v", result)
}
params := result.EffectiveModelParams
if params.Model != "chosen-model" || params.Temperature != 0 || params.MaxTokens != 0 || params.TopP != 0 || params.TimeoutSeconds != 0 || params.ReasoningEffort != "" || params.ServiceTier != "priority" {
t.Fatalf("native explicit overrides: %+v", params)
}
select {
case request := <-requests:
messages := request["messages"].([]any)
message := messages[0].(map[string]any)["content"].(string)
for _, want := range []string{"file,transcript=body", "last,literal=2", "Title,with=equals"} {
if !strings.Contains(message, want) {
t.Fatalf("lost literal/mapping %q: %s", want, message)
}
}
for _, key := range []string{"temperature", "max_tokens", "top_p", "seed"} {
if value, exists := request[key]; !exists || value != float64(0) {
t.Fatalf("explicit zero %s lost: %+v", key, request)
}
}
if request["model"] != "chosen-model" || request["service_tier"] != "priority" || request["authorization"] != "Bearer synthetic-credential-canary" || request["stale"] != nil || request["reasoning_effort"] != nil {
t.Fatalf("provider overrides: %+v", request)
}
default:
t.Fatal("provider was not called")
}
}
func TestRunDeliversRawAndNativeStructuredResults(t *testing.T) {
const content = " <raw>&\x00\nno-final-newline"
var calls atomic.Int64
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls.Add(1)
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{"choices": []any{map[string]any{"message": map[string]any{"content": content}}}})
}))
defer server.Close()
nativePromptFixture(t, server.URL+"/v1")
flags := []string{"support/summary", "--config=app.yml", "--llm-profile=support/writer", "--llm-session-id=session", "--llm-input=transcript=text", "--llm-input-text=unlisted=literal", "--llm-var=mode=", "--llm-var=title=Title"}
for _, format := range []string{"", "text", "json", "yaml"} {
args := append([]string{"run"}, flags...)
if format != "" {
args = append(args, "--format="+format)
}
outcome, output, diagnostics := executeNative(t, args...)
if outcome.Status != 0 {
t.Fatalf("%s: %+v %q %q", format, outcome, output, diagnostics)
}
if format == "" || format == "text" {
if output != content || diagnostics != "" {
t.Fatalf("raw output decorated: %q %q", output, diagnostics)
}
} else {
var report map[string]any
if err := yaml.Unmarshal([]byte(output), &report); err != nil {
t.Fatal(err)
}
if report["prompt_id"] != "support/summary" || report["run_id"] == nil || report["artifact"] == nil || report["result"] != nil || report["schema_version"] != nil {
t.Fatalf("non-native result: %+v", report)
}
}
}
outcome, output, diagnostics := executeNative(t, append(append([]string{"run"}, flags...), "--verbose")...)
if outcome.Status != 0 || output != content || diagnostics == "" {
t.Fatalf("verbose streams: %+v %q %q", outcome, output, diagnostics)
}
if calls.Load() != 5 {
t.Fatalf("generation dispatches=%d", calls.Load())
}
// The same selected prompt/profile/context used above is valid through the
// native preview and requirements paths (Narratio's consistency scenario).
discovered := discoverNative(t, "support/summary", "--profile=support/writer", "--session-id=session", "--var=mode=")
if discovered.PromptID != "support/summary" || discovered.ProfileID != "support/writer" {
t.Fatal(discovered)
}
}
func TestRunDistinguishesCompletedInvalidContentAndOperationalFailure(t *testing.T) {
var fail atomic.Bool
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
if fail.Load() {
w.WriteHeader(401)
fmt.Fprint(w, `{"error":{"message":"provider-private-canary"}}`)
return
}
fmt.Fprint(w, `{"choices":[{"message":{"content":"not-json"}}]}`)
}))
defer server.Close()
nativePromptFixture(t, server.URL+"/v1")
writeNativeFile(t, "invalid-content.yml", "profiles:\n - id: local\n endpoint: "+server.URL+"/v1\n model: controlled\nprompts:\n - id: json\n default_profile: local\n messages:\n - role: user\n content: Return JSON\n output:\n format: json\n validation_mode: json\n")
writeNativeFile(t, "json-app.yml", "promptkit:\n configuration: invalid-content.yml\ntransient_retries: 0\n")
args := []string{"run", "json", "--config=json-app.yml"}
outcome, output, diagnostics := executeNative(t, args...)
if outcome.Status != 4 || outcome.Kind != commandkit.OutcomeDomain || outcome.Domain != "validation_failed" || output != "not-json" || diagnostics == "" {
t.Fatalf("completed invalid content: %+v %q %q", outcome, output, diagnostics)
}
outcome, output, diagnostics = executeNative(t, append(args, "--format=json")...)
var result promptkit.RunResult
if err := json.Unmarshal([]byte(output), &result); err != nil {
t.Fatal(err)
}
if outcome.Status != 4 || result.Validation.Status != promptkit.ValidationFailed || string(result.Artifact.Body) != "not-json" {
t.Fatalf("invalid result lost: %+v %+v", outcome, result)
}
fail.Store(true)
outcome, output, diagnostics = executeNative(t, args...)
var generation *promptkit.GenerationError
if outcome.Status != 1 || output != "" || !errors.Is(outcome.Cause, promptkit.ErrLLMGenerate) || !errors.As(outcome.Cause, &generation) || diagnostics == "" || strings.Contains(diagnostics, "provider-private-canary") {
t.Fatalf("operational failure: %+v %q %q", outcome, output, diagnostics)
}
}
func TestRunUsageAndConfigurationFailures(t *testing.T) {
t.Chdir(t.TempDir())
for _, args := range [][]string{
{"run"}, {"run", "one", "two"}, {"run", "one", "--prompt=two"}, {"run", "one", "--llm-prompt=two"},
{"run", "one", "--backend=local"}, {"run", "one", "--out=result"}, {"run", "one", "--execution-report=result"},
{"run", "one", "--llm-base-url=http://example.invalid"}, {"run", "one", "--timeout=1s"},
{"run", "one", "--prompt-id=two"}, {"run", "one", "--profile-id=local"}, {"run", "one", "--run-timeout=bad"},
} {
outcome, _, _ := executeNative(t, args...)
if outcome.Kind != commandkit.OutcomeUsage || outcome.Status != 2 {
t.Fatalf("%v: %+v", args, outcome)
}
}
writeNativeFile(t, "invalid.yml", "server:\n max_request_bytes: -1\n")
outcome, output, diagnostics := executeNative(t, "run", "one", "--config=invalid.yml")
if outcome.Status != 2 || output != "" || !strings.Contains(diagnostics, "validate.error") {
t.Fatalf("invalid config: %+v %q %q", outcome, output, diagnostics)
}
outcome, output, diagnostics = executeNative(t, "run", "one", "--config=absent.yml")
if outcome.Status != 1 || output != "" || diagnostics == "" {
t.Fatalf("missing config: %+v %q %q", outcome, output, diagnostics)
}
}
func TestTypedRequestFailureMapping(t *testing.T) {
_, invalid := configkit.Load(configkit.Bytes("invalid", []byte("[\n")), configkit.Options[map[string]any]{})
for _, test := range []struct {
err error
status int
}{
{fmt.Errorf("wrapped: %w", invalid), 2}, {promptkit.ErrInvalidConfig, 1},
{errors.Join(invalid, context.Canceled), 1}, {errors.Join(invalid, promptkit.ErrOperationTimeout, context.DeadlineExceeded), 1},
{promptkit.ErrCapacityExceeded, 1}, {promptkit.ErrValidation, 1}, {promptkit.ErrInvalidRequest, 1},
{promptkit.ErrPromptNotFound, 1}, {promptkit.ErrLLMGenerate, 1},
} {
outcome := requestFailure(test.err, commandkit.Explanation{})
if outcome.Status != test.status || !errors.Is(outcome.Cause, test.err) {
t.Fatalf("%v: %+v", test.err, outcome)
}
}
if outcome := completedRunOutcome(nil); outcome.Status != 1 || outcome.Cause == nil {
t.Fatalf("nil successful result accepted: %+v", outcome)
}
// Mapping a duration adds no local positivity policy or second timer.
for _, budget := range []time.Duration{-time.Second, 0, time.Second} {
request := requestFromRunInput(runInput{OperationTimeout: budget})
if request.OperationTimeout != budget {
t.Fatal("operation budget changed", budget)
}
}
}
func TestRunDiagnosticQueueKeepsArtifactOnStdout(t *testing.T) {
app, err := commandkit.New(commandkit.Identity{Name: "reporting"}, commandkit.NewCommand(commandkit.CommandSpec[struct{}]{Name: "run", Run: func(inv commandkit.Invocation, _ struct{}) commandkit.Outcome {
queueDiagnostics(inv.Reporter(), []commandkit.Diagnostic{commandkit.MessageDiagnostic(commandkit.SeverityInfo, "acquisition-info"), commandkit.MessageDiagnostic(commandkit.SeverityWarning, "acquisition-warning")})
_ = inv.Reporter().Report(commandkit.Report{Value: &promptkit.RunResult{Artifact: promptkit.Artifact{Body: []byte("raw")}}, TextTemplate: artifactTextTemplate})
return commandkit.Success()
}}))
if err != nil {
t.Fatal(err)
}
var output, diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), []string{"run", "--verbose"}, commandkit.Streams{Out: &output, Err: &diagnostics})
if outcome.Status != 0 || output.String() != "raw" || !strings.Contains(diagnostics.String(), "acquisition-info") || !strings.Contains(diagnostics.String(), "acquisition-warning") {
t.Fatalf("streams: %+v %q %q", outcome, output.String(), diagnostics.String())
}
}

View File

@@ -0,0 +1,76 @@
package cli
import (
"context"
"errors"
"go.maximumdirect.net/ckconfig"
"go.maximumdirect.net/commandkit"
"go.maximumdirect.net/promptkit"
)
func completedRunOutcome(result *promptkit.RunResult) commandkit.Outcome {
if result == nil {
return commandkit.Failure("Prompt execution returned no result.").WithCause(errors.New("engine returned neither a result nor an error"))
}
if result.Validation.Status == promptkit.ValidationFailed {
return commandkit.Domain("validation_failed", 4).WithExplanation("Generated content failed validation.")
}
return commandkit.Success()
}
func requestFailure(err error, explanation commandkit.Explanation) commandkit.Outcome {
message := "The requested operation failed."
switch {
case errors.Is(err, promptkit.ErrOperationTimeout):
message = "The operation timeout expired."
case errors.Is(err, context.Canceled):
message = "The operation was canceled."
case errors.Is(err, context.DeadlineExceeded):
message = "The caller deadline expired."
default:
if ckconfig.ClassifyConfigurationError(err) == commandkit.OutcomeUsage {
return commandkit.Usage("Invalid configuration.", explanation.Details...).WithCause(err)
}
switch {
case errors.Is(err, promptkit.ErrCapacityExceeded):
message = "Model backend capacity is exhausted."
case errors.Is(err, promptkit.ErrLLMGenerate):
message = "Model generation failed."
case errors.Is(err, promptkit.ErrValidation):
message = "The output validation operation failed."
case errors.Is(err, promptkit.ErrPromptNotFound):
message = "The selected prompt was not found."
case errors.Is(err, promptkit.ErrProfileNotFound):
message = "The selected profile was not found."
case errors.Is(err, promptkit.ErrArtifactLoad):
message = "Could not load request input artifacts."
case errors.Is(err, promptkit.ErrPromptRender):
message = "Could not render the selected prompt."
case errors.Is(err, promptkit.ErrInvalidRequest):
message = "The prompt request is invalid."
case errors.Is(err, promptkit.ErrInvalidConfig):
message = "Could not initialize PromptKit configuration."
}
}
return commandkit.Failure(message, explanation.Details...).WithCause(err)
}
func queueDiagnostics(reporter *commandkit.Reporter, diagnostics []commandkit.Diagnostic) {
deferred := reporter.AtEnd()
for _, diagnostic := range diagnostics {
if diagnostic.Write == nil {
continue
}
switch diagnostic.Severity {
case commandkit.SeverityInfo:
_ = deferred.Info("", diagnostic.Write)
case commandkit.SeverityDebug:
_ = deferred.Debug("", diagnostic.Write)
case commandkit.SeverityProgress:
_ = reporter.Progress("", diagnostic.Write)
default:
_ = deferred.Warn("", diagnostic.Write)
}
}
}