236 lines
8.4 KiB
Go
236 lines
8.4 KiB
Go
|
|
package cli
|
||
|
|
|
||
|
|
import (
|
||
|
|
"bytes"
|
||
|
|
"context"
|
||
|
|
"encoding/json"
|
||
|
|
"errors"
|
||
|
|
"fmt"
|
||
|
|
"io"
|
||
|
|
"net"
|
||
|
|
"net/http"
|
||
|
|
"net/http/httptest"
|
||
|
|
"strings"
|
||
|
|
"sync/atomic"
|
||
|
|
"testing"
|
||
|
|
"time"
|
||
|
|
|
||
|
|
"go.maximumdirect.net/commandkit"
|
||
|
|
"go.maximumdirect.net/promptkit"
|
||
|
|
)
|
||
|
|
|
||
|
|
func TestRunCancellationAndNativeTimeoutLayers(t *testing.T) {
|
||
|
|
for _, test := range []struct {
|
||
|
|
name, operation, attempt string
|
||
|
|
cancelParent, localTimeout bool
|
||
|
|
parentDeadline bool
|
||
|
|
}{
|
||
|
|
{"parent cancellation", "10s", "0", true, false, false},
|
||
|
|
{"operation timeout", "500ms", "0", false, true, false},
|
||
|
|
{"zero operation budget", "0", "0", true, false, false},
|
||
|
|
{"negative operation budget", "-1s", "0", true, false, false},
|
||
|
|
{"attempt timeout", "0", "1", false, false, false},
|
||
|
|
{"earlier parent deadline", "10s", "0", false, false, true},
|
||
|
|
} {
|
||
|
|
t.Run(test.name, func(t *testing.T) {
|
||
|
|
started, stopped, release := make(chan struct{}), make(chan struct{}), make(chan struct{})
|
||
|
|
provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
|
|
_, _ = io.Copy(io.Discard, r.Body)
|
||
|
|
close(started)
|
||
|
|
select {
|
||
|
|
case <-r.Context().Done():
|
||
|
|
close(stopped)
|
||
|
|
case <-release:
|
||
|
|
}
|
||
|
|
}))
|
||
|
|
defer provider.Close()
|
||
|
|
defer close(release)
|
||
|
|
nativePromptFixture(t, provider.URL+"/v1")
|
||
|
|
var ctx context.Context
|
||
|
|
var cancel context.CancelFunc
|
||
|
|
if test.parentDeadline {
|
||
|
|
ctx, cancel = context.WithTimeout(context.Background(), 500*time.Millisecond)
|
||
|
|
} else {
|
||
|
|
ctx, cancel = context.WithCancel(context.Background())
|
||
|
|
}
|
||
|
|
defer cancel()
|
||
|
|
var output, diagnostics bytes.Buffer
|
||
|
|
finished := make(chan commandkit.Outcome, 1)
|
||
|
|
args := append(deliveryArgs(), "--run-timeout="+test.operation, "--timeout-seconds="+test.attempt)
|
||
|
|
go func() {
|
||
|
|
finished <- Execute(ctx, commandkit.Identity{Name: "scriptorium"}, args, &output, &diagnostics)
|
||
|
|
}()
|
||
|
|
select {
|
||
|
|
case <-started:
|
||
|
|
case outcome := <-finished:
|
||
|
|
t.Fatalf("ended before provider dispatch: %+v %q", outcome, diagnostics.String())
|
||
|
|
case <-time.After(5 * time.Second):
|
||
|
|
t.Fatal("provider did not start")
|
||
|
|
}
|
||
|
|
if test.cancelParent {
|
||
|
|
cancel()
|
||
|
|
}
|
||
|
|
var outcome commandkit.Outcome
|
||
|
|
select {
|
||
|
|
case outcome = <-finished:
|
||
|
|
case <-time.After(5 * time.Second):
|
||
|
|
t.Fatal("run did not stop")
|
||
|
|
}
|
||
|
|
if outcome.Status != 1 || output.Len() != 0 || diagnostics.Len() == 0 || errors.Is(outcome.Cause, promptkit.ErrOperationTimeout) != test.localTimeout {
|
||
|
|
t.Fatalf("timeout/cancellation: %+v %q %q", outcome, output.String(), diagnostics.String())
|
||
|
|
}
|
||
|
|
if test.cancelParent && !errors.Is(outcome.Cause, context.Canceled) {
|
||
|
|
t.Fatalf("parent cause lost: %+v", outcome)
|
||
|
|
}
|
||
|
|
if !test.cancelParent && !errors.Is(outcome.Cause, context.DeadlineExceeded) {
|
||
|
|
t.Fatalf("deadline cause lost: %+v", outcome)
|
||
|
|
}
|
||
|
|
if !test.cancelParent && !test.parentDeadline && ctx.Err() != nil {
|
||
|
|
t.Fatal("engine deadline changed caller context")
|
||
|
|
}
|
||
|
|
select {
|
||
|
|
case <-stopped:
|
||
|
|
case <-time.After(5 * time.Second):
|
||
|
|
t.Fatal("provider request was not canceled")
|
||
|
|
}
|
||
|
|
})
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestServeStartupHTTPAndCancellation(t *testing.T) {
|
||
|
|
var block atomic.Bool
|
||
|
|
started, stopped, release := make(chan struct{}), make(chan struct{}), make(chan struct{})
|
||
|
|
provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
|
|
_, _ = io.Copy(io.Discard, r.Body)
|
||
|
|
if block.Load() {
|
||
|
|
close(started)
|
||
|
|
select {
|
||
|
|
case <-r.Context().Done():
|
||
|
|
close(stopped)
|
||
|
|
case <-release:
|
||
|
|
}
|
||
|
|
return
|
||
|
|
}
|
||
|
|
w.Header().Set("Content-Type", "application/json")
|
||
|
|
fmt.Fprint(w, `{"choices":[{"message":{"content":"served"}}]}`)
|
||
|
|
}))
|
||
|
|
defer provider.Close()
|
||
|
|
defer close(release)
|
||
|
|
nativePromptFixture(t, provider.URL+"/v1")
|
||
|
|
writeNativeFile(t, "artifacts/transcript.md", "file from restricted root")
|
||
|
|
ready := make(chan string, 1)
|
||
|
|
listen := func(ctx context.Context, network, addr string) (net.Listener, error) {
|
||
|
|
listener, err := (&net.ListenConfig{}).Listen(ctx, network, addr)
|
||
|
|
if err == nil {
|
||
|
|
ready <- listener.Addr().String()
|
||
|
|
}
|
||
|
|
return listener, err
|
||
|
|
}
|
||
|
|
app, err := commandkit.New(commandkit.Identity{Name: "scriptorium"}, commandkit.NewCommand(commandkit.CommandSpec[serveInput]{Name: "serve", Flags: serveFlags(), Run: func(inv commandkit.Invocation, input serveInput) commandkit.Outcome {
|
||
|
|
return serveWithListener(inv, input, listen)
|
||
|
|
}}))
|
||
|
|
if err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
ctx, cancel := context.WithCancel(context.Background())
|
||
|
|
defer cancel()
|
||
|
|
var output, diagnostics bytes.Buffer
|
||
|
|
finished := make(chan commandkit.Outcome, 1)
|
||
|
|
go func() {
|
||
|
|
finished <- app.Execute(ctx, []string{"serve", "--config=app.yml", "--addr=127.0.0.1:0", "--artifact-root=artifacts", "--max-request-bytes=0", "--max-artifact-bytes=0", "--max-response-bytes=0", "--verbose"}, commandkit.Streams{Out: &output, Err: &diagnostics})
|
||
|
|
}()
|
||
|
|
var addr string
|
||
|
|
select {
|
||
|
|
case addr = <-ready:
|
||
|
|
case outcome := <-finished:
|
||
|
|
t.Fatalf("startup: %+v %q", outcome, diagnostics.String())
|
||
|
|
case <-time.After(5 * time.Second):
|
||
|
|
t.Fatal("server did not listen")
|
||
|
|
}
|
||
|
|
client := &http.Client{Timeout: 5 * time.Second}
|
||
|
|
defer client.CloseIdleConnections()
|
||
|
|
const payload = `{"prompt_id":"support/summary","profile_id":"support/writer","session_id":"server-session","vars":{"mode":"","title":"Title"},"inputs":{"transcript":{"type":"file","uri":"transcript.md"},"unlisted":{"type":"inline","body":"literal"}},"model":{"timeout_seconds":0},"include_execution_report":true}`
|
||
|
|
response, err := client.Post("http://"+addr+"/v1/runs", "application/json", strings.NewReader(payload))
|
||
|
|
if err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
var value map[string]any
|
||
|
|
decodeErr := json.NewDecoder(response.Body).Decode(&value)
|
||
|
|
response.Body.Close()
|
||
|
|
if response.StatusCode != 200 || decodeErr != nil || value["execution"] == nil {
|
||
|
|
t.Fatalf("HTTP response: %d %+v %v", response.StatusCode, value, decodeErr)
|
||
|
|
}
|
||
|
|
metadata := value["metadata"].(map[string]any)
|
||
|
|
if metadata["selected_profile_id"] != "support/writer" || metadata["session_id"] != "server-session" {
|
||
|
|
t.Fatalf("HTTP contract changed: %+v", value)
|
||
|
|
}
|
||
|
|
block.Store(true)
|
||
|
|
requestFinished := make(chan error, 1)
|
||
|
|
go func() {
|
||
|
|
response, err := client.Post("http://"+addr+"/v1/runs", "application/json", strings.NewReader(payload))
|
||
|
|
if response != nil {
|
||
|
|
response.Body.Close()
|
||
|
|
}
|
||
|
|
requestFinished <- err
|
||
|
|
}()
|
||
|
|
select {
|
||
|
|
case <-started:
|
||
|
|
case <-time.After(5 * time.Second):
|
||
|
|
t.Fatal("active HTTP request did not reach engine")
|
||
|
|
}
|
||
|
|
cancel()
|
||
|
|
select {
|
||
|
|
case outcome := <-finished:
|
||
|
|
if outcome.Status != 1 || !errors.Is(outcome.Cause, context.Canceled) || output.Len() != 0 || diagnostics.Len() == 0 {
|
||
|
|
t.Fatalf("shutdown: %+v %q %q", outcome, output.String(), diagnostics.String())
|
||
|
|
}
|
||
|
|
case <-time.After(5 * time.Second):
|
||
|
|
t.Fatal("HTTP server did not close")
|
||
|
|
}
|
||
|
|
select {
|
||
|
|
case <-requestFinished:
|
||
|
|
case <-time.After(5 * time.Second):
|
||
|
|
t.Fatal("active HTTP connection remained open")
|
||
|
|
}
|
||
|
|
select {
|
||
|
|
case <-stopped:
|
||
|
|
case <-time.After(5 * time.Second):
|
||
|
|
t.Fatal("active engine work was not canceled")
|
||
|
|
}
|
||
|
|
if connection, err := net.DialTimeout("tcp", addr, time.Second); err == nil {
|
||
|
|
connection.Close()
|
||
|
|
t.Fatal("listener remained open")
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestServeListenAndConfigurationFailures(t *testing.T) {
|
||
|
|
nativePromptFixture(t, "http://127.0.0.1:1/v1")
|
||
|
|
for _, args := range [][]string{
|
||
|
|
{"serve", "--config=app.yml", "--addr=invalid:address:syntax"},
|
||
|
|
{"serve", "--config=absent.yml"},
|
||
|
|
} {
|
||
|
|
outcome, output, diagnostics := executeNative(t, args...)
|
||
|
|
if outcome.Status != 1 || outcome.Cause == nil || output != "" || diagnostics == "" {
|
||
|
|
t.Fatalf("serve failure: %+v %q %q", outcome, output, diagnostics)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
writeNativeFile(t, "invalid.yml", "server:\n max_request_bytes: -1\n")
|
||
|
|
outcome, _, diagnostics := executeNative(t, "serve", "--config=invalid.yml")
|
||
|
|
if outcome.Status != 2 || !strings.Contains(diagnostics, "validate.error") {
|
||
|
|
t.Fatalf("serve configuration classification: %+v %q", outcome, diagnostics)
|
||
|
|
}
|
||
|
|
writeNativeFile(t, "bad-framework.yml", "profiles: [\n")
|
||
|
|
writeNativeFile(t, "bad-framework-app.yml", "promptkit:\n configuration: bad-framework.yml\n")
|
||
|
|
for _, command := range []string{"run", "serve"} {
|
||
|
|
args := []string{command}
|
||
|
|
if command == "run" {
|
||
|
|
args = append(args, "any")
|
||
|
|
}
|
||
|
|
args = append(args, "--config=bad-framework-app.yml")
|
||
|
|
outcome, _, diagnostics := executeNative(t, args...)
|
||
|
|
if outcome.Status != 2 || !strings.Contains(diagnostics, "PromptKit configuration") || !strings.Contains(diagnostics, "yaml.invalid") {
|
||
|
|
t.Fatalf("framework diagnostics: %+v %q", outcome, diagnostics)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|