Implement positional prompt execution with native reporting and outcomes
This commit is contained in:
234
internal/adapter/cli/native_run_test.go
Normal file
234
internal/adapter/cli/native_run_test.go
Normal file
@@ -0,0 +1,234 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"go.maximumdirect.net/commandkit"
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
"go.yaml.in/yaml/v4"
|
||||
)
|
||||
|
||||
func TestPositionalRunMapsNativeBindingsAndNamespaces(t *testing.T) {
|
||||
requests := make(chan map[string]any, 1)
|
||||
const content = " \x00<artifact>&\n尾-no-newline"
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var request map[string]any
|
||||
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
request["authorization"] = r.Header.Get("Authorization")
|
||||
select {
|
||||
case requests <- request:
|
||||
default:
|
||||
t.Error("unexpected duplicate provider call")
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"choices": []any{map[string]any{"message": map[string]any{"role": "assistant", "content": content}}}})
|
||||
}))
|
||||
defer server.Close()
|
||||
nativePromptFixture(t, "http://127.0.0.1:1/v1")
|
||||
t.Setenv("SCRIPTORIUM_CONTROLLED_KEY", "synthetic-credential-canary")
|
||||
writeNativeFile(t, "note,part=1.txt", "file,transcript=body")
|
||||
args := []string{"run", "summary", "--config=app.yml", "--prompt-namespace=support", "--profile=writer", "--profile-namespace=support", "--backend-namespace=review", "--session-id=caller-session",
|
||||
"--input-file=transcript=note,part=1.txt", "--input-text=transcript=first,text=1", "--input-file=transcript=note,part=1.txt", "--input-file=unlisted=missing.txt", "--input=unlisted=last,literal=2", "--var=mode=", "--var=title=Title,with=equals",
|
||||
"--endpoint=" + server.URL + "/v1", "--model=chosen-model", "--temperature=0", "--max-tokens=0", "--top-p=0", "--timeout-seconds=0", "--reasoning-effort=", "--service-tier=priority", `--extra-params={"seed":0,"test_payload":[0,"x"]}`, "--api-key-env=SCRIPTORIUM_CONTROLLED_KEY", "--transient-retries=0", "--run-timeout=5s", "--format=json"}
|
||||
outcome, output, diagnostics := executeNative(t, args...)
|
||||
if outcome.Status != 0 || strings.Contains(output+diagnostics, "synthetic-credential-canary") {
|
||||
t.Fatalf("run: %+v %q %q", outcome, output, diagnostics)
|
||||
}
|
||||
var result promptkit.RunResult
|
||||
if err := json.Unmarshal([]byte(output), &result); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(result.Artifact.Body) != content || result.PromptID != "support/summary" || result.SelectedProfileID != "support/writer" || result.SelectedBackendID != "support/local" || result.SessionID != "caller-session" {
|
||||
t.Fatalf("native result/context: %+v", result)
|
||||
}
|
||||
params := result.EffectiveModelParams
|
||||
if params.Model != "chosen-model" || params.Temperature != 0 || params.MaxTokens != 0 || params.TopP != 0 || params.TimeoutSeconds != 0 || params.ReasoningEffort != "" || params.ServiceTier != "priority" {
|
||||
t.Fatalf("native explicit overrides: %+v", params)
|
||||
}
|
||||
select {
|
||||
case request := <-requests:
|
||||
messages := request["messages"].([]any)
|
||||
message := messages[0].(map[string]any)["content"].(string)
|
||||
for _, want := range []string{"file,transcript=body", "last,literal=2", "Title,with=equals"} {
|
||||
if !strings.Contains(message, want) {
|
||||
t.Fatalf("lost literal/mapping %q: %s", want, message)
|
||||
}
|
||||
}
|
||||
for _, key := range []string{"temperature", "max_tokens", "top_p", "seed"} {
|
||||
if value, exists := request[key]; !exists || value != float64(0) {
|
||||
t.Fatalf("explicit zero %s lost: %+v", key, request)
|
||||
}
|
||||
}
|
||||
if request["model"] != "chosen-model" || request["service_tier"] != "priority" || request["authorization"] != "Bearer synthetic-credential-canary" || request["stale"] != nil || request["reasoning_effort"] != nil {
|
||||
t.Fatalf("provider overrides: %+v", request)
|
||||
}
|
||||
default:
|
||||
t.Fatal("provider was not called")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunDeliversRawAndNativeStructuredResults(t *testing.T) {
|
||||
const content = " <raw>&\x00\nno-final-newline"
|
||||
var calls atomic.Int64
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls.Add(1)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"choices": []any{map[string]any{"message": map[string]any{"content": content}}}})
|
||||
}))
|
||||
defer server.Close()
|
||||
nativePromptFixture(t, server.URL+"/v1")
|
||||
flags := []string{"support/summary", "--config=app.yml", "--llm-profile=support/writer", "--llm-session-id=session", "--llm-input=transcript=text", "--llm-input-text=unlisted=literal", "--llm-var=mode=", "--llm-var=title=Title"}
|
||||
for _, format := range []string{"", "text", "json", "yaml"} {
|
||||
args := append([]string{"run"}, flags...)
|
||||
if format != "" {
|
||||
args = append(args, "--format="+format)
|
||||
}
|
||||
outcome, output, diagnostics := executeNative(t, args...)
|
||||
if outcome.Status != 0 {
|
||||
t.Fatalf("%s: %+v %q %q", format, outcome, output, diagnostics)
|
||||
}
|
||||
if format == "" || format == "text" {
|
||||
if output != content || diagnostics != "" {
|
||||
t.Fatalf("raw output decorated: %q %q", output, diagnostics)
|
||||
}
|
||||
} else {
|
||||
var report map[string]any
|
||||
if err := yaml.Unmarshal([]byte(output), &report); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if report["prompt_id"] != "support/summary" || report["run_id"] == nil || report["artifact"] == nil || report["result"] != nil || report["schema_version"] != nil {
|
||||
t.Fatalf("non-native result: %+v", report)
|
||||
}
|
||||
}
|
||||
}
|
||||
outcome, output, diagnostics := executeNative(t, append(append([]string{"run"}, flags...), "--verbose")...)
|
||||
if outcome.Status != 0 || output != content || diagnostics == "" {
|
||||
t.Fatalf("verbose streams: %+v %q %q", outcome, output, diagnostics)
|
||||
}
|
||||
if calls.Load() != 5 {
|
||||
t.Fatalf("generation dispatches=%d", calls.Load())
|
||||
}
|
||||
// The same selected prompt/profile/context used above is valid through the
|
||||
// native preview and requirements paths (Narratio's consistency scenario).
|
||||
discovered := discoverNative(t, "support/summary", "--profile=support/writer", "--session-id=session", "--var=mode=")
|
||||
if discovered.PromptID != "support/summary" || discovered.ProfileID != "support/writer" {
|
||||
t.Fatal(discovered)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunDistinguishesCompletedInvalidContentAndOperationalFailure(t *testing.T) {
|
||||
var fail atomic.Bool
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if fail.Load() {
|
||||
w.WriteHeader(401)
|
||||
fmt.Fprint(w, `{"error":{"message":"provider-private-canary"}}`)
|
||||
return
|
||||
}
|
||||
fmt.Fprint(w, `{"choices":[{"message":{"content":"not-json"}}]}`)
|
||||
}))
|
||||
defer server.Close()
|
||||
nativePromptFixture(t, server.URL+"/v1")
|
||||
writeNativeFile(t, "invalid-content.yml", "profiles:\n - id: local\n endpoint: "+server.URL+"/v1\n model: controlled\nprompts:\n - id: json\n default_profile: local\n messages:\n - role: user\n content: Return JSON\n output:\n format: json\n validation_mode: json\n")
|
||||
writeNativeFile(t, "json-app.yml", "promptkit:\n configuration: invalid-content.yml\ntransient_retries: 0\n")
|
||||
args := []string{"run", "json", "--config=json-app.yml"}
|
||||
outcome, output, diagnostics := executeNative(t, args...)
|
||||
if outcome.Status != 4 || outcome.Kind != commandkit.OutcomeDomain || outcome.Domain != "validation_failed" || output != "not-json" || diagnostics == "" {
|
||||
t.Fatalf("completed invalid content: %+v %q %q", outcome, output, diagnostics)
|
||||
}
|
||||
outcome, output, diagnostics = executeNative(t, append(args, "--format=json")...)
|
||||
var result promptkit.RunResult
|
||||
if err := json.Unmarshal([]byte(output), &result); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if outcome.Status != 4 || result.Validation.Status != promptkit.ValidationFailed || string(result.Artifact.Body) != "not-json" {
|
||||
t.Fatalf("invalid result lost: %+v %+v", outcome, result)
|
||||
}
|
||||
fail.Store(true)
|
||||
outcome, output, diagnostics = executeNative(t, args...)
|
||||
var generation *promptkit.GenerationError
|
||||
if outcome.Status != 1 || output != "" || !errors.Is(outcome.Cause, promptkit.ErrLLMGenerate) || !errors.As(outcome.Cause, &generation) || diagnostics == "" || strings.Contains(diagnostics, "provider-private-canary") {
|
||||
t.Fatalf("operational failure: %+v %q %q", outcome, output, diagnostics)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunUsageAndConfigurationFailures(t *testing.T) {
|
||||
t.Chdir(t.TempDir())
|
||||
for _, args := range [][]string{
|
||||
{"run"}, {"run", "one", "two"}, {"run", "one", "--prompt=two"}, {"run", "one", "--llm-prompt=two"},
|
||||
{"run", "one", "--backend=local"}, {"run", "one", "--out=result"}, {"run", "one", "--execution-report=result"},
|
||||
{"run", "one", "--llm-base-url=http://example.invalid"}, {"run", "one", "--timeout=1s"},
|
||||
{"run", "one", "--prompt-id=two"}, {"run", "one", "--profile-id=local"}, {"run", "one", "--run-timeout=bad"},
|
||||
} {
|
||||
outcome, _, _ := executeNative(t, args...)
|
||||
if outcome.Kind != commandkit.OutcomeUsage || outcome.Status != 2 {
|
||||
t.Fatalf("%v: %+v", args, outcome)
|
||||
}
|
||||
}
|
||||
writeNativeFile(t, "invalid.yml", "server:\n max_request_bytes: -1\n")
|
||||
outcome, output, diagnostics := executeNative(t, "run", "one", "--config=invalid.yml")
|
||||
if outcome.Status != 2 || output != "" || !strings.Contains(diagnostics, "validate.error") {
|
||||
t.Fatalf("invalid config: %+v %q %q", outcome, output, diagnostics)
|
||||
}
|
||||
outcome, output, diagnostics = executeNative(t, "run", "one", "--config=absent.yml")
|
||||
if outcome.Status != 1 || output != "" || diagnostics == "" {
|
||||
t.Fatalf("missing config: %+v %q %q", outcome, output, diagnostics)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTypedRequestFailureMapping(t *testing.T) {
|
||||
_, invalid := configkit.Load(configkit.Bytes("invalid", []byte("[\n")), configkit.Options[map[string]any]{})
|
||||
for _, test := range []struct {
|
||||
err error
|
||||
status int
|
||||
}{
|
||||
{fmt.Errorf("wrapped: %w", invalid), 2}, {promptkit.ErrInvalidConfig, 1},
|
||||
{errors.Join(invalid, context.Canceled), 1}, {errors.Join(invalid, promptkit.ErrOperationTimeout, context.DeadlineExceeded), 1},
|
||||
{promptkit.ErrCapacityExceeded, 1}, {promptkit.ErrValidation, 1}, {promptkit.ErrInvalidRequest, 1},
|
||||
{promptkit.ErrPromptNotFound, 1}, {promptkit.ErrLLMGenerate, 1},
|
||||
} {
|
||||
outcome := requestFailure(test.err, commandkit.Explanation{})
|
||||
if outcome.Status != test.status || !errors.Is(outcome.Cause, test.err) {
|
||||
t.Fatalf("%v: %+v", test.err, outcome)
|
||||
}
|
||||
}
|
||||
if outcome := completedRunOutcome(nil); outcome.Status != 1 || outcome.Cause == nil {
|
||||
t.Fatalf("nil successful result accepted: %+v", outcome)
|
||||
}
|
||||
// Mapping a duration adds no local positivity policy or second timer.
|
||||
for _, budget := range []time.Duration{-time.Second, 0, time.Second} {
|
||||
request := requestFromRunInput(runInput{OperationTimeout: budget})
|
||||
if request.OperationTimeout != budget {
|
||||
t.Fatal("operation budget changed", budget)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunDiagnosticQueueKeepsArtifactOnStdout(t *testing.T) {
|
||||
app, err := commandkit.New(commandkit.Identity{Name: "reporting"}, commandkit.NewCommand(commandkit.CommandSpec[struct{}]{Name: "run", Run: func(inv commandkit.Invocation, _ struct{}) commandkit.Outcome {
|
||||
queueDiagnostics(inv.Reporter(), []commandkit.Diagnostic{commandkit.MessageDiagnostic(commandkit.SeverityInfo, "acquisition-info"), commandkit.MessageDiagnostic(commandkit.SeverityWarning, "acquisition-warning")})
|
||||
_ = inv.Reporter().Report(commandkit.Report{Value: &promptkit.RunResult{Artifact: promptkit.Artifact{Body: []byte("raw")}}, TextTemplate: artifactTextTemplate})
|
||||
return commandkit.Success()
|
||||
}}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var output, diagnostics bytes.Buffer
|
||||
outcome := app.Execute(context.Background(), []string{"run", "--verbose"}, commandkit.Streams{Out: &output, Err: &diagnostics})
|
||||
if outcome.Status != 0 || output.String() != "raw" || !strings.Contains(diagnostics.String(), "acquisition-info") || !strings.Contains(diagnostics.String(), "acquisition-warning") {
|
||||
t.Fatalf("streams: %+v %q %q", outcome, output.String(), diagnostics.String())
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user