package cli import ( "bytes" "context" "encoding/json" "errors" "fmt" "io" "net" "net/http" "net/http/httptest" "strings" "sync/atomic" "testing" "time" "go.maximumdirect.net/commandkit" "go.maximumdirect.net/promptkit" ) func TestRunCancellationAndNativeTimeoutLayers(t *testing.T) { for _, test := range []struct { name, operation, attempt string cancelParent, localTimeout bool parentDeadline bool }{ {"parent cancellation", "10s", "0", true, false, false}, {"operation timeout", "500ms", "0", false, true, false}, {"zero operation budget", "0", "0", true, false, false}, {"negative operation budget", "-1s", "0", true, false, false}, {"attempt timeout", "0", "1", false, false, false}, {"earlier parent deadline", "10s", "0", false, false, true}, } { t.Run(test.name, func(t *testing.T) { started, stopped, release := make(chan struct{}), make(chan struct{}), make(chan struct{}) provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = io.Copy(io.Discard, r.Body) close(started) select { case <-r.Context().Done(): close(stopped) case <-release: } })) defer provider.Close() defer close(release) nativePromptFixture(t, provider.URL+"/v1") var ctx context.Context var cancel context.CancelFunc if test.parentDeadline { ctx, cancel = context.WithTimeout(context.Background(), 500*time.Millisecond) } else { ctx, cancel = context.WithCancel(context.Background()) } defer cancel() var output, diagnostics bytes.Buffer finished := make(chan commandkit.Outcome, 1) args := append(deliveryArgs(), "--run-timeout="+test.operation, "--timeout-seconds="+test.attempt) go func() { finished <- Execute(ctx, commandkit.Identity{Name: "scriptorium"}, args, &output, &diagnostics) }() select { case <-started: case outcome := <-finished: t.Fatalf("ended before provider dispatch: %+v %q", outcome, diagnostics.String()) case <-time.After(5 * time.Second): t.Fatal("provider did not start") } if test.cancelParent { cancel() } var outcome commandkit.Outcome select { case outcome = <-finished: case <-time.After(5 * time.Second): t.Fatal("run did not stop") } if outcome.Status != 1 || output.Len() != 0 || diagnostics.Len() == 0 || errors.Is(outcome.Cause, promptkit.ErrOperationTimeout) != test.localTimeout { t.Fatalf("timeout/cancellation: %+v %q %q", outcome, output.String(), diagnostics.String()) } if test.cancelParent && !errors.Is(outcome.Cause, context.Canceled) { t.Fatalf("parent cause lost: %+v", outcome) } if !test.cancelParent && !errors.Is(outcome.Cause, context.DeadlineExceeded) { t.Fatalf("deadline cause lost: %+v", outcome) } if !test.cancelParent && !test.parentDeadline && ctx.Err() != nil { t.Fatal("engine deadline changed caller context") } select { case <-stopped: case <-time.After(5 * time.Second): t.Fatal("provider request was not canceled") } }) } } func TestServeStartupHTTPAndCancellation(t *testing.T) { var block atomic.Bool started, stopped, release := make(chan struct{}), make(chan struct{}), make(chan struct{}) provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = io.Copy(io.Discard, r.Body) if block.Load() { close(started) select { case <-r.Context().Done(): close(stopped) case <-release: } return } w.Header().Set("Content-Type", "application/json") fmt.Fprint(w, `{"choices":[{"message":{"content":"served"}}]}`) })) defer provider.Close() defer close(release) nativePromptFixture(t, provider.URL+"/v1") writeNativeFile(t, "artifacts/transcript.md", "file from restricted root") ready := make(chan string, 1) listen := func(ctx context.Context, network, addr string) (net.Listener, error) { listener, err := (&net.ListenConfig{}).Listen(ctx, network, addr) if err == nil { ready <- listener.Addr().String() } return listener, err } app, err := commandkit.New(commandkit.Identity{Name: "scriptorium"}, commandkit.NewCommand(commandkit.CommandSpec[serveInput]{Name: "serve", Flags: serveFlags(), Run: func(inv commandkit.Invocation, input serveInput) commandkit.Outcome { return serveWithListener(inv, input, listen) }})) if err != nil { t.Fatal(err) } ctx, cancel := context.WithCancel(context.Background()) defer cancel() var output, diagnostics bytes.Buffer finished := make(chan commandkit.Outcome, 1) go func() { finished <- app.Execute(ctx, []string{"serve", "--config=app.yml", "--addr=127.0.0.1:0", "--artifact-root=artifacts", "--max-request-bytes=0", "--max-artifact-bytes=0", "--max-response-bytes=0", "--verbose"}, commandkit.Streams{Out: &output, Err: &diagnostics}) }() var addr string select { case addr = <-ready: case outcome := <-finished: t.Fatalf("startup: %+v %q", outcome, diagnostics.String()) case <-time.After(5 * time.Second): t.Fatal("server did not listen") } client := &http.Client{Timeout: 5 * time.Second} defer client.CloseIdleConnections() const payload = `{"prompt_id":"support/summary","profile_id":"support/writer","session_id":"server-session","vars":{"mode":"","title":"Title"},"inputs":{"transcript":{"type":"file","uri":"transcript.md"},"unlisted":{"type":"inline","body":"literal"}},"model":{"timeout_seconds":0},"include_execution_report":true}` response, err := client.Post("http://"+addr+"/v1/runs", "application/json", strings.NewReader(payload)) if err != nil { t.Fatal(err) } var value map[string]any decodeErr := json.NewDecoder(response.Body).Decode(&value) response.Body.Close() if response.StatusCode != 200 || decodeErr != nil || value["execution"] == nil { t.Fatalf("HTTP response: %d %+v %v", response.StatusCode, value, decodeErr) } metadata := value["metadata"].(map[string]any) if metadata["selected_profile_id"] != "support/writer" || metadata["session_id"] != "server-session" { t.Fatalf("HTTP contract changed: %+v", value) } block.Store(true) requestFinished := make(chan error, 1) go func() { response, err := client.Post("http://"+addr+"/v1/runs", "application/json", strings.NewReader(payload)) if response != nil { response.Body.Close() } requestFinished <- err }() select { case <-started: case <-time.After(5 * time.Second): t.Fatal("active HTTP request did not reach engine") } cancel() select { case outcome := <-finished: if outcome.Status != 1 || !errors.Is(outcome.Cause, context.Canceled) || output.Len() != 0 || diagnostics.Len() == 0 { t.Fatalf("shutdown: %+v %q %q", outcome, output.String(), diagnostics.String()) } case <-time.After(5 * time.Second): t.Fatal("HTTP server did not close") } select { case <-requestFinished: case <-time.After(5 * time.Second): t.Fatal("active HTTP connection remained open") } select { case <-stopped: case <-time.After(5 * time.Second): t.Fatal("active engine work was not canceled") } if connection, err := net.DialTimeout("tcp", addr, time.Second); err == nil { connection.Close() t.Fatal("listener remained open") } } func TestServeListenAndConfigurationFailures(t *testing.T) { nativePromptFixture(t, "http://127.0.0.1:1/v1") for _, args := range [][]string{ {"serve", "--config=app.yml", "--addr=invalid:address:syntax"}, {"serve", "--config=absent.yml"}, } { outcome, output, diagnostics := executeNative(t, args...) if outcome.Status != 1 || outcome.Cause == nil || output != "" || diagnostics == "" { t.Fatalf("serve failure: %+v %q %q", outcome, output, diagnostics) } } writeNativeFile(t, "invalid.yml", "server:\n max_request_bytes: -1\n") outcome, _, diagnostics := executeNative(t, "serve", "--config=invalid.yml") if outcome.Status != 2 || !strings.Contains(diagnostics, "validate.error") { t.Fatalf("serve configuration classification: %+v %q", outcome, diagnostics) } writeNativeFile(t, "bad-framework.yml", "profiles: [\n") writeNativeFile(t, "bad-framework-app.yml", "promptkit:\n configuration: bad-framework.yml\n") for _, command := range []string{"run", "serve"} { args := []string{command} if command == "run" { args = append(args, "any") } args = append(args, "--config=bad-framework-app.yml") outcome, _, diagnostics := executeNative(t, args...) if outcome.Status != 2 || !strings.Contains(diagnostics, "PromptKit configuration") || !strings.Contains(diagnostics, "yaml.invalid") { t.Fatalf("framework diagnostics: %+v %q", outcome, diagnostics) } } }