Refactor the single-run HTTP contract

This commit is contained in:
2026-11-18 19:12:07 +00:00
parent de7c866662
commit f858d57322
6 changed files with 125 additions and 59 deletions

View File

@@ -145,6 +145,8 @@ scaffolding, or conversation and coordination features.
## Stage 4: Refactor the single-run HTTP API ## Stage 4: Refactor the single-run HTTP API
**Status: Complete.**
1. Keep `POST /v1/runs` as the single route: the operation is still a run 1. Keep `POST /v1/runs` as the single route: the operation is still a run
and there is no parallel version to distinguish. This is a direct and there is no parallel version to distinguish. This is a direct
contract replacement at that path, not a promise of old-client contract replacement at that path, not a promise of old-client

View File

@@ -14,5 +14,6 @@
"vars": { "vars": {
"session_date": "2026-05-04" "session_date": "2026-05-04"
}, },
"include_raw_output": false "include_raw_output": false,
"include_execution_report": false
} }

View File

@@ -11,6 +11,36 @@ import (
"go.maximumdirect.net/promptkit" "go.maximumdirect.net/promptkit"
) )
func TestRestrictedArtifactReaderReturnsDetachedBodies(t *testing.T) {
root := t.TempDir()
if err := os.WriteFile(filepath.Join(root, "input.txt"), []byte("original"), 0o644); err != nil {
t.Fatal(err)
}
reader, err := NewRestrictedArtifactReader(root, 0)
if err != nil {
t.Fatal(err)
}
for _, ref := range []promptkit.ArtifactRef{promptkit.Inline("original"), promptkit.File("input.txt")} {
first, err := reader.Read(context.Background(), ref)
if err != nil {
t.Fatal(err)
}
second, err := reader.Read(context.Background(), ref)
if err != nil {
t.Fatal(err)
}
first.Body[0] = 'X'
first.Name = "changed"
if string(second.Body) != "original" || second.Name == "changed" {
t.Fatalf("read results share mutable state: %+v", second)
}
second.Body[1] = 'Y'
if string(first.Body) != "Xriginal" {
t.Fatalf("reader retained a body alias: %q", first.Body)
}
}
}
func TestRestrictedArtifactReaderReadsContainedFiles(t *testing.T) { func TestRestrictedArtifactReaderReadsContainedFiles(t *testing.T) {
root := t.TempDir() root := t.TempDir()
outside := t.TempDir() outside := t.TempDir()

View File

@@ -8,7 +8,6 @@ import (
type runRequestDTO struct { type runRequestDTO struct {
PromptID string `json:"prompt_id"` PromptID string `json:"prompt_id"`
PromptVersion string `json:"prompt_version,omitempty"`
ProfileID string `json:"profile_id,omitempty"` ProfileID string `json:"profile_id,omitempty"`
SessionID string `json:"session_id,omitempty"` SessionID string `json:"session_id,omitempty"`
Inputs map[string]inputRefDTO `json:"inputs"` Inputs map[string]inputRefDTO `json:"inputs"`
@@ -59,28 +58,21 @@ type artifactDTO struct {
type metadataDTO struct { type metadataDTO struct {
RunID string `json:"run_id"` RunID string `json:"run_id"`
PromptID string `json:"prompt_id"` PromptID string `json:"prompt_id"`
PromptVersion string `json:"prompt_version"`
PromptHash string `json:"prompt_hash"` PromptHash string `json:"prompt_hash"`
RenderedPromptHash string `json:"rendered_prompt_hash"` RenderedPromptHash string `json:"rendered_prompt_hash"`
SelectedProfileID string `json:"selected_profile_id"` SelectedProfileID string `json:"selected_profile_id"`
SelectedBackendID string `json:"selected_backend_id,omitempty"` SelectedBackendID string `json:"selected_backend_id,omitempty"`
SessionID string `json:"session_id,omitempty"` SessionID string `json:"session_id,omitempty"`
ModelName string `json:"model_name"`
Endpoint string `json:"endpoint"`
ModelParams modelParamsDTO `json:"model_params"` ModelParams modelParamsDTO `json:"model_params"`
InputHashes map[string]string `json:"input_hashes"` InputHashes map[string]string `json:"input_hashes"`
Usage tokenUsageDTO `json:"usage"` Usage tokenUsageDTO `json:"usage"`
StartTime time.Time `json:"start_time"` StartTime time.Time `json:"start_time"`
EndTime time.Time `json:"end_time"` EndTime time.Time `json:"end_time"`
DurationMS int64 `json:"duration_ms"` DurationMS int64 `json:"duration_ms"`
ValidationMode string `json:"validation_mode"`
ValidationStatus string `json:"validation_status"`
RepairAttemptsUsed int `json:"repair_attempts_used"`
} }
type modelParamsDTO struct { type modelParamsDTO struct {
Endpoint string `json:"endpoint"` Endpoint string `json:"endpoint"`
BackendID string `json:"backend_id,omitempty"`
Model string `json:"model"` Model string `json:"model"`
Temperature float64 `json:"temperature"` Temperature float64 `json:"temperature"`
MaxTokens int `json:"max_tokens"` MaxTokens int `json:"max_tokens"`

View File

@@ -106,38 +106,36 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
model = executionTargetOverrideFromModelOverrideDTO(req.Model) model = executionTargetOverrideFromModelOverrideDTO(req.Model)
} }
runContext := r.Context() var operationTimeout time.Duration
if req.RunTimeoutSeconds != nil && *req.RunTimeoutSeconds > 0 { if req.RunTimeoutSeconds != nil && *req.RunTimeoutSeconds > 0 {
var cancel context.CancelFunc operationTimeout = time.Duration(*req.RunTimeoutSeconds) * time.Second
runContext, cancel = context.WithTimeout(runContext, time.Duration(*req.RunTimeoutSeconds)*time.Second)
defer cancel()
} }
mappedRequest := promptkit.RunRequest{ mappedRequest := promptkit.RunRequest{
PromptID: req.PromptID, PromptID: req.PromptID,
PromptVersion: req.PromptVersion,
ProfileID: req.ProfileID, ProfileID: req.ProfileID,
SessionID: req.SessionID, SessionID: req.SessionID,
Inputs: mappedInputs, Inputs: mappedInputs,
Vars: req.Vars, Vars: req.Vars,
Execution: model, Execution: model,
TransientRetries: req.TransientRetries, TransientRetries: req.TransientRetries,
OperationTimeout: operationTimeout,
} }
var collector *executionreport.Collector var collector *executionreport.Collector
if req.IncludeExecutionReport { if req.IncludeExecutionReport {
collector = executionreport.NewCollector() collector = executionreport.NewCollector()
mappedRequest.Observer = collector.Observe mappedRequest.Observer = collector.Observe
} }
res, err := h.runner.Run(runContext, mappedRequest) res, err := h.runner.Run(r.Context(), mappedRequest)
var execution *executionreport.Execution var execution *executionreport.Execution
if collector != nil { if collector != nil {
execution = collector.Snapshot() execution = collector.Snapshot()
} }
if err != nil { if err != nil {
switch runContext.Err() { switch {
case context.DeadlineExceeded: case errors.Is(err, promptkit.ErrOperationTimeout), errors.Is(err, context.DeadlineExceeded) && errors.Is(r.Context().Err(), context.DeadlineExceeded):
writeRunFailure(w, http.StatusGatewayTimeout, "deadline_exceeded", "run execution deadline exceeded", execution, h.options.MaxResponseBytes) writeRunFailure(w, http.StatusGatewayTimeout, "deadline_exceeded", "run execution deadline exceeded", execution, h.options.MaxResponseBytes)
return return
case context.Canceled: case errors.Is(err, context.Canceled) && errors.Is(r.Context().Err(), context.Canceled):
writeRunFailure(w, http.StatusRequestTimeout, "request_canceled", "run request canceled", execution, h.options.MaxResponseBytes) writeRunFailure(w, http.StatusRequestTimeout, "request_canceled", "run request canceled", execution, h.options.MaxResponseBytes)
return return
} }
@@ -159,14 +157,11 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
Metadata: metadataDTO{ Metadata: metadataDTO{
RunID: res.RunID, RunID: res.RunID,
PromptID: res.PromptID, PromptID: res.PromptID,
PromptVersion: res.PromptVersion,
PromptHash: res.PromptHash, PromptHash: res.PromptHash,
RenderedPromptHash: res.RenderedPromptHash, RenderedPromptHash: res.RenderedPromptHash,
SelectedProfileID: res.SelectedProfileID, SelectedProfileID: res.SelectedProfileID,
SelectedBackendID: res.SelectedBackendID, SelectedBackendID: res.SelectedBackendID,
SessionID: res.SessionID, SessionID: res.SessionID,
ModelName: res.ModelName,
Endpoint: res.Endpoint,
ModelParams: modelParamsDTOFromExecutionTarget(res.EffectiveModelParams), ModelParams: modelParamsDTOFromExecutionTarget(res.EffectiveModelParams),
InputHashes: res.InputHashes, InputHashes: res.InputHashes,
Usage: tokenUsageDTO{ Usage: tokenUsageDTO{
@@ -176,12 +171,9 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
CachedTokens: res.Usage.CachedTokens, CachedTokens: res.Usage.CachedTokens,
CacheWriteTokens: res.Usage.CacheWriteTokens, CacheWriteTokens: res.Usage.CacheWriteTokens,
}, },
StartTime: res.StartTime, StartTime: res.StartTime,
EndTime: res.EndTime, EndTime: res.EndTime,
DurationMS: res.Duration.Milliseconds(), DurationMS: res.Duration.Milliseconds(),
ValidationMode: string(res.Validation.Mode),
ValidationStatus: string(res.Validation.Status),
RepairAttemptsUsed: res.Validation.RepairAttempts,
}, },
Execution: execution, Execution: execution,
} }
@@ -221,7 +213,6 @@ func executionTargetOverrideFromModelOverrideDTO(dto *modelOverrideRequestDTO) *
func modelParamsDTOFromExecutionTarget(target promptkit.ExecutionTarget) modelParamsDTO { func modelParamsDTOFromExecutionTarget(target promptkit.ExecutionTarget) modelParamsDTO {
return modelParamsDTO{ return modelParamsDTO{
Endpoint: target.Endpoint, Endpoint: target.Endpoint,
BackendID: target.BackendID,
Model: target.Model, Model: target.Model,
Temperature: target.Temperature, Temperature: target.Temperature,
MaxTokens: target.MaxTokens, MaxTokens: target.MaxTokens,
@@ -255,16 +246,16 @@ func mapRunError(err error) (int, string, string) {
return http.StatusBadRequest, "profile_required", "profile_id is required when prompt default_profile is not set" return http.StatusBadRequest, "profile_required", "profile_id is required when prompt default_profile is not set"
case errors.Is(err, promptkit.ErrAPIKeyEnvMissing): case errors.Is(err, promptkit.ErrAPIKeyEnvMissing):
return http.StatusBadRequest, "api_key_env_missing", "api_key_env is set but the environment variable is missing" return http.StatusBadRequest, "api_key_env_missing", "api_key_env is set but the environment variable is missing"
case errors.Is(err, ErrFileNotAllowed), errors.Is(err, ErrFileOutsideRoot):
return http.StatusBadRequest, "artifact_not_allowed", "file input artifact is not allowed"
case errors.Is(err, ErrFileTooLarge):
return http.StatusRequestEntityTooLarge, "artifact_too_large", "file input artifact is too large"
case errors.Is(err, promptkit.ErrPromptLoad): case errors.Is(err, promptkit.ErrPromptLoad):
return http.StatusBadRequest, "prompt_load_failed", "failed to load prompt definition" return http.StatusBadRequest, "prompt_load_failed", "failed to load prompt definition"
case errors.Is(err, promptkit.ErrProfileLoad): case errors.Is(err, promptkit.ErrProfileLoad):
return http.StatusBadRequest, "profile_load_failed", "failed to load execution profile" return http.StatusBadRequest, "profile_load_failed", "failed to load execution profile"
case errors.Is(err, promptkit.ErrInvalidRequest): case errors.Is(err, promptkit.ErrInvalidRequest):
return http.StatusBadRequest, "invalid_request", "invalid run request" return http.StatusBadRequest, "invalid_request", "invalid run request"
case errors.Is(err, ErrFileNotAllowed), errors.Is(err, ErrFileOutsideRoot):
return http.StatusBadRequest, "artifact_not_allowed", "file input artifact is not allowed"
case errors.Is(err, ErrFileTooLarge):
return http.StatusRequestEntityTooLarge, "artifact_too_large", "file input artifact is too large"
case errors.Is(err, promptkit.ErrArtifactLoad): case errors.Is(err, promptkit.ErrArtifactLoad):
return http.StatusBadRequest, "artifact_read_failed", "failed to read input artifact" return http.StatusBadRequest, "artifact_read_failed", "failed to read input artifact"
case errors.Is(err, promptkit.ErrPromptRender): case errors.Is(err, promptkit.ErrPromptRender):

View File

@@ -4,6 +4,7 @@ import (
"bytes" "bytes"
"context" "context"
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
@@ -336,11 +337,13 @@ func TestHandlerRunTimeoutValidationAndContextMapping(t *testing.T) {
name, body string name, body string
parent context.Context parent context.Context
wantDeadline time.Time wantDeadline time.Time
wantTimeout time.Duration
}{ }{
{name: "omitted inherits deadline", body: `{"prompt_id":"p"}`, parent: parent, wantDeadline: parentDeadline}, {name: "omitted inherits deadline", body: `{"prompt_id":"p"}`, parent: parent, wantDeadline: parentDeadline},
{name: "null inherits deadline", body: `{"prompt_id":"p","run_timeout_seconds":null}`, parent: parent, wantDeadline: parentDeadline}, {name: "null inherits deadline", body: `{"prompt_id":"p","run_timeout_seconds":null}`, parent: parent, wantDeadline: parentDeadline},
{name: "zero inherits deadline", body: `{"prompt_id":"p","run_timeout_seconds":0}`, parent: parent, wantDeadline: parentDeadline}, {name: "zero inherits deadline", body: `{"prompt_id":"p","run_timeout_seconds":0}`, parent: parent, wantDeadline: parentDeadline},
{name: "positive preserves earlier deadline", body: `{"prompt_id":"p","run_timeout_seconds":10}`, parent: parent, wantDeadline: parentDeadline}, {name: "positive preserves earlier deadline", body: `{"prompt_id":"p","run_timeout_seconds":10}`, parent: parent, wantDeadline: parentDeadline, wantTimeout: 10 * time.Second},
{name: "positive adds operation budget", body: `{"prompt_id":"p","run_timeout_seconds":2}`, parent: context.Background(), wantTimeout: 2 * time.Second},
{name: "omitted adds no deadline", body: `{"prompt_id":"p"}`, parent: context.Background()}, {name: "omitted adds no deadline", body: `{"prompt_id":"p"}`, parent: context.Background()},
{name: "null adds no deadline", body: `{"prompt_id":"p","run_timeout_seconds":null}`, parent: context.Background()}, {name: "null adds no deadline", body: `{"prompt_id":"p","run_timeout_seconds":null}`, parent: context.Background()},
{name: "zero adds no deadline", body: `{"prompt_id":"p","run_timeout_seconds":0}`, parent: context.Background()}, {name: "zero adds no deadline", body: `{"prompt_id":"p","run_timeout_seconds":0}`, parent: context.Background()},
@@ -353,6 +356,9 @@ func TestHandlerRunTimeoutValidationAndContextMapping(t *testing.T) {
if ok != !tc.wantDeadline.IsZero() || (ok && !deadline.Equal(tc.wantDeadline)) { if ok != !tc.wantDeadline.IsZero() || (ok && !deadline.Equal(tc.wantDeadline)) {
t.Fatalf("execution deadline: got %v (present=%t), want %v", deadline, ok, tc.wantDeadline) t.Fatalf("execution deadline: got %v (present=%t), want %v", deadline, ok, tc.wantDeadline)
} }
if req.OperationTimeout != tc.wantTimeout {
t.Fatalf("operation timeout = %v, want %v", req.OperationTimeout, tc.wantTimeout)
}
return &promptkit.RunResult{}, nil return &promptkit.RunResult{}, nil
})) }))
w := httptest.NewRecorder() w := httptest.NewRecorder()
@@ -382,6 +388,22 @@ func TestHandlerRunTimeoutValidationAndContextMapping(t *testing.T) {
h.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(`{"prompt_id":"p","run_timeout_seconds":10}`))) h.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(`{"prompt_id":"p","run_timeout_seconds":10}`)))
assertHTTPErrorCode(t, w, http.StatusBadGateway, "llm_failed") assertHTTPErrorCode(t, w, http.StatusBadGateway, "llm_failed")
expired, cancelExpired := context.WithDeadline(context.Background(), time.Now().Add(-time.Second))
defer cancelExpired()
h = NewHandler(runnerFunc(func(ctx context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) {
return nil, fmt.Errorf("run failed: %w", ctx.Err())
}))
w = httptest.NewRecorder()
h.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(`{"prompt_id":"p"}`)).WithContext(expired))
assertHTTPErrorCode(t, w, http.StatusGatewayTimeout, "deadline_exceeded")
h = NewHandler(runnerFunc(func(ctx context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) {
return nil, errors.Join(promptkit.ErrOperationTimeout, context.Canceled)
}))
w = httptest.NewRecorder()
h.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(`{"prompt_id":"p"}`)).WithContext(canceled))
assertHTTPErrorCode(t, w, http.StatusGatewayTimeout, "deadline_exceeded")
successContext, cancelSuccess := context.WithCancel(context.Background()) successContext, cancelSuccess := context.WithCancel(context.Background())
h = NewHandler(runnerFunc(func(ctx context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) { h = NewHandler(runnerFunc(func(ctx context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) {
cancelSuccess() cancelSuccess()
@@ -447,7 +469,6 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
}, },
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic, IsValid: true}, Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic, IsValid: true},
PromptID: "prompt-1", PromptID: "prompt-1",
PromptVersion: "1.0.0",
PromptHash: "phash", PromptHash: "phash",
RenderedPromptHash: "rhash", RenderedPromptHash: "rhash",
SelectedProfileID: "exec-default", SelectedProfileID: "exec-default",
@@ -508,8 +529,10 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
if metadata["prompt_id"] != "prompt-1" { if metadata["prompt_id"] != "prompt-1" {
t.Fatalf("unexpected metadata.prompt_id: %#v", metadata["prompt_id"]) t.Fatalf("unexpected metadata.prompt_id: %#v", metadata["prompt_id"])
} }
if metadata["prompt_version"] != "1.0.0" { for _, field := range []string{"prompt_version", "model_name", "endpoint", "validation_mode", "validation_status", "repair_attempts_used"} {
t.Fatalf("unexpected metadata.prompt_version: %#v", metadata["prompt_version"]) if _, ok := metadata[field]; ok {
t.Fatalf("redundant metadata field %q: %#v", field, metadata)
}
} }
if metadata["selected_profile_id"] != "exec-default" { if metadata["selected_profile_id"] != "exec-default" {
t.Fatalf("unexpected metadata.selected_profile_id: %#v", metadata["selected_profile_id"]) t.Fatalf("unexpected metadata.selected_profile_id: %#v", metadata["selected_profile_id"])
@@ -517,9 +540,6 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
if metadata["selected_backend_id"] != "local" { if metadata["selected_backend_id"] != "local" {
t.Fatalf("unexpected metadata.selected_backend_id: %#v", metadata["selected_backend_id"]) t.Fatalf("unexpected metadata.selected_backend_id: %#v", metadata["selected_backend_id"])
} }
if metadata["model_name"] != "m1" || metadata["endpoint"] != "http://llm/v1" {
t.Fatalf("unexpected model metadata: name=%#v endpoint=%#v", metadata["model_name"], metadata["endpoint"])
}
usage := metadata["usage"].(map[string]any) usage := metadata["usage"].(map[string]any)
if usage["prompt_tokens"] != float64(1) || usage["completion_tokens"] != float64(2) || usage["total_tokens"] != float64(3) { if usage["prompt_tokens"] != float64(1) || usage["completion_tokens"] != float64(2) || usage["total_tokens"] != float64(3) {
t.Fatalf("unexpected base usage metadata: %#v", usage) t.Fatalf("unexpected base usage metadata: %#v", usage)
@@ -528,8 +548,8 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
t.Fatalf("unexpected cache usage metadata: %#v", usage) t.Fatalf("unexpected cache usage metadata: %#v", usage)
} }
modelParams := metadata["model_params"].(map[string]any) modelParams := metadata["model_params"].(map[string]any)
if modelParams["backend_id"] != "local" { if _, ok := modelParams["backend_id"]; ok || modelParams["model"] != "m1" || modelParams["endpoint"] != "http://llm/v1" {
t.Fatalf("expected model_params.backend_id=local, got %#v", modelParams["backend_id"]) t.Fatalf("unexpected model parameters: %#v", modelParams)
} }
if modelParams["api_key_env"] != envName { if modelParams["api_key_env"] != envName {
t.Fatalf("expected model_params.api_key_env=%q, got %#v", envName, modelParams["api_key_env"]) t.Fatalf("expected model_params.api_key_env=%q, got %#v", envName, modelParams["api_key_env"])
@@ -561,7 +581,7 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
} }
} }
func TestHandlerAllowsOmittedInputsAndMapsPromptVersion(t *testing.T) { func TestHandlerAllowsOmittedInputsAndPreservesQualifiedIDs(t *testing.T) {
r := &fakeRunner{result: &promptkit.RunResult{ r := &fakeRunner{result: &promptkit.RunResult{
Artifact: promptkit.Artifact{Body: []byte("ok")}, Artifact: promptkit.Artifact{Body: []byte("ok")},
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationNone, IsValid: true}, Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationNone, IsValid: true},
@@ -570,8 +590,8 @@ func TestHandlerAllowsOmittedInputsAndMapsPromptVersion(t *testing.T) {
h := NewHandler(r) h := NewHandler(r)
for _, body := range []string{ for _, body := range []string{
`{"prompt_id":"prompt-1","prompt_version":"2"}`, `{"prompt_id":"support/prompt-1","profile_id":"support/profile-1"}`,
`{"prompt_id":"prompt-1","prompt_version":"2","inputs":{}}`, `{"prompt_id":"support/prompt-1","profile_id":"support/profile-1","inputs":{}}`,
} { } {
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(body)) req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(body))
w := httptest.NewRecorder() w := httptest.NewRecorder()
@@ -580,8 +600,8 @@ func TestHandlerAllowsOmittedInputsAndMapsPromptVersion(t *testing.T) {
if w.Code != http.StatusOK { if w.Code != http.StatusOK {
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String()) t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
} }
if r.last.PromptVersion != "2" { if r.last.PromptID != "support/prompt-1" || r.last.ProfileID != "support/profile-1" {
t.Fatalf("expected prompt version to be mapped, got %q", r.last.PromptVersion) t.Fatalf("qualified IDs changed: %+v", r.last)
} }
if r.last.Inputs != nil { if r.last.Inputs != nil {
t.Fatalf("expected omitted or empty inputs to remain nil, got %#v", r.last.Inputs) t.Fatalf("expected omitted or empty inputs to remain nil, got %#v", r.last.Inputs)
@@ -589,6 +609,16 @@ func TestHandlerAllowsOmittedInputsAndMapsPromptVersion(t *testing.T) {
} }
} }
func TestHandlerRejectsRemovedPromptVersion(t *testing.T) {
runner := &fakeRunner{result: &promptkit.RunResult{}}
w := httptest.NewRecorder()
NewHandler(runner).ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(`{"prompt_id":"p","prompt_version":"1"}`)))
assertHTTPErrorCode(t, w, http.StatusBadRequest, "invalid_json")
if runner.last.PromptID != "" {
t.Fatal("runner invoked for removed selector")
}
}
func TestHandlerDelegatesDefinitionInputRequirements(t *testing.T) { func TestHandlerDelegatesDefinitionInputRequirements(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
@@ -786,7 +816,6 @@ func TestHandlerPostRunsSuccessUsingPromptDefaultProfile(t *testing.T) {
r := &fakeRunner{result: &promptkit.RunResult{ r := &fakeRunner{result: &promptkit.RunResult{
Artifact: promptkit.Artifact{Body: []byte("ok")}, Artifact: promptkit.Artifact{Body: []byte("ok")},
PromptID: "prompt-1", PromptID: "prompt-1",
PromptVersion: "1.0.0",
SelectedProfileID: "prompt-default", SelectedProfileID: "prompt-default",
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic, IsValid: true}, Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic, IsValid: true},
EffectiveModelParams: promptkit.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"}, EffectiveModelParams: promptkit.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
@@ -1191,9 +1220,13 @@ func TestHandlerDeadlineCancelsBuiltInRetryWait(t *testing.T) {
})) }))
defer provider.Close() defer provider.Close()
h := newRetryHandler(t, provider.URL+"/v1", intPointer(1), 0, nil) h := newRetryHandler(t, provider.URL+"/v1", intPointer(1), 0, nil)
failures := observeRunnerFailures(h)
w := httptest.NewRecorder() w := httptest.NewRecorder()
h.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(`{"prompt_id":"p","run_timeout_seconds":1}`))) h.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(`{"prompt_id":"p","run_timeout_seconds":1}`)))
assertHTTPErrorCode(t, w, http.StatusGatewayTimeout, "deadline_exceeded") assertHTTPErrorCode(t, w, http.StatusGatewayTimeout, "deadline_exceeded")
if err := <-failures; !errors.Is(err, promptkit.ErrOperationTimeout) {
t.Fatalf("engine did not attribute retry wait to operation timeout: %v", err)
}
if atomic.LoadInt32(&hits) != 1 { if atomic.LoadInt32(&hits) != 1 {
t.Fatalf("expected one provider call before canceled retry wait, got %d", atomic.LoadInt32(&hits)) t.Fatalf("expected one provider call before canceled retry wait, got %d", atomic.LoadInt32(&hits))
} }
@@ -1221,6 +1254,7 @@ func TestHandlerDeadlineCancelsCapacityWaitAndReleasesAdmission(t *testing.T) {
}() }()
queue := 1 queue := 1
h := newRetryHandler(t, provider.URL+"/v1", intPointer(0), 1, &queue) h := newRetryHandler(t, provider.URL+"/v1", intPointer(0), 1, &queue)
failures := observeRunnerFailures(h)
first := httptest.NewRecorder() first := httptest.NewRecorder()
done := make(chan struct{}) done := make(chan struct{})
go func() { go func() {
@@ -1235,6 +1269,9 @@ func TestHandlerDeadlineCancelsCapacityWaitAndReleasesAdmission(t *testing.T) {
waiting := httptest.NewRecorder() waiting := httptest.NewRecorder()
h.ServeHTTP(waiting, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(`{"prompt_id":"p","run_timeout_seconds":1}`))) h.ServeHTTP(waiting, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(`{"prompt_id":"p","run_timeout_seconds":1}`)))
assertHTTPErrorCode(t, waiting, http.StatusGatewayTimeout, "deadline_exceeded") assertHTTPErrorCode(t, waiting, http.StatusGatewayTimeout, "deadline_exceeded")
if err := <-failures; !errors.Is(err, promptkit.ErrOperationTimeout) {
t.Fatalf("engine did not attribute capacity wait to operation timeout: %v", err)
}
if atomic.LoadInt32(&hits) != 1 { if atomic.LoadInt32(&hits) != 1 {
t.Fatalf("capacity waiter reached provider: hits=%d", atomic.LoadInt32(&hits)) t.Fatalf("capacity waiter reached provider: hits=%d", atomic.LoadInt32(&hits))
} }
@@ -1254,27 +1291,39 @@ func TestHandlerDeadlineCancelsCapacityWaitAndReleasesAdmission(t *testing.T) {
} }
} }
func observeRunnerFailures(h *Handler) <-chan error {
failures := make(chan error, 1)
runner := h.runner
h.runner = runnerFunc(func(ctx context.Context, request promptkit.RunRequest) (*promptkit.RunResult, error) {
result, err := runner.Run(ctx, request)
if err != nil {
failures <- err
}
return result, err
})
return failures
}
func newRetryHandler(t *testing.T, endpoint string, retries *int, concurrency int, queue *int) *Handler { func newRetryHandler(t *testing.T, endpoint string, retries *int, concurrency int, queue *int) *Handler {
t.Helper() t.Helper()
promptDir := t.TempDir() promptDir := t.TempDir()
profileDir := t.TempDir() profileDir := t.TempDir()
if err := os.WriteFile(filepath.Join(promptDir, "prompt.yaml"), []byte(`prompts: if err := os.WriteFile(filepath.Join(promptDir, "prompt.yaml"), []byte(`prompts:
p: - id: p
"1": default_profile: limited
default_profile: limited messages:
messages: - role: user
- role: user content: hi
content: hi output:
output: format: text
format: text validation_mode: none
validation_mode: none
`), 0o644); err != nil { `), 0o644); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if err := os.WriteFile(filepath.Join(profileDir, "profile.yaml"), []byte("profiles:\n limited:\n backend: limited\n model: test\n"), 0o644); err != nil { if err := os.WriteFile(filepath.Join(profileDir, "profile.yaml"), []byte("profiles:\n - id: limited\n backend: limited\n model: test\n"), 0o644); err != nil {
t.Fatal(err) t.Fatal(err)
} }
backend := promptkit.BackendConfig{Endpoint: &endpoint, ConcurrencyLimit: &concurrency, QueueCapacity: queue} backend := promptkit.BackendConfig{ID: "limited", Endpoint: &endpoint, ConcurrencyLimit: &concurrency, QueueCapacity: queue}
engine := newHandlerEngineFromDirectories(t, promptDir, profileDir, retries, &backend) engine := newHandlerEngineFromDirectories(t, promptDir, profileDir, retries, &backend)
return NewHandler(engine) return NewHandler(engine)
} }
@@ -1627,6 +1676,7 @@ func TestHandlerPublicErrorMapping(t *testing.T) {
{name: "file denied", err: ErrFileNotAllowed, status: http.StatusBadRequest, code: "artifact_not_allowed", message: "file input artifact is not allowed"}, {name: "file denied", err: ErrFileNotAllowed, status: http.StatusBadRequest, code: "artifact_not_allowed", message: "file input artifact is not allowed"},
{name: "file outside root", err: ErrFileOutsideRoot, status: http.StatusBadRequest, code: "artifact_not_allowed", message: "file input artifact is not allowed"}, {name: "file outside root", err: ErrFileOutsideRoot, status: http.StatusBadRequest, code: "artifact_not_allowed", message: "file input artifact is not allowed"},
{name: "file too large", err: ErrFileTooLarge, status: http.StatusRequestEntityTooLarge, code: "artifact_too_large", message: "file input artifact is too large"}, {name: "file too large", err: ErrFileTooLarge, status: http.StatusRequestEntityTooLarge, code: "artifact_too_large", message: "file input artifact is too large"},
{name: "file too large with broad category", err: errors.Join(ErrFileTooLarge, promptkit.ErrInvalidRequest, promptkit.ErrArtifactLoad), status: http.StatusRequestEntityTooLarge, code: "artifact_too_large", message: "file input artifact is too large"},
{name: "artifact", err: wrap(promptkit.ErrArtifactLoad, fmt.Errorf("read failed")), status: http.StatusBadRequest, code: "artifact_read_failed", message: "failed to read input artifact", avoidCause: "read failed"}, {name: "artifact", err: wrap(promptkit.ErrArtifactLoad, fmt.Errorf("read failed")), status: http.StatusBadRequest, code: "artifact_read_failed", message: "failed to read input artifact", avoidCause: "read failed"},
{name: "prompt render", err: wrap(promptkit.ErrPromptRender, fmt.Errorf("render failed")), status: http.StatusBadRequest, code: "prompt_render_failed", message: "failed to render prompt", avoidCause: "render failed"}, {name: "prompt render", err: wrap(promptkit.ErrPromptRender, fmt.Errorf("render failed")), status: http.StatusBadRequest, code: "prompt_render_failed", message: "failed to render prompt", avoidCause: "render failed"},
{name: "capacity", err: &promptkit.CapacityError{BackendID: "private-backend"}, status: http.StatusServiceUnavailable, code: "capacity_exceeded", message: "model backend capacity is exhausted", avoidCause: "private-backend"}, {name: "capacity", err: &promptkit.CapacityError{BackendID: "private-backend"}, status: http.StatusServiceUnavailable, code: "capacity_exceeded", message: "model backend capacity is exhausted", avoidCause: "private-backend"},
@@ -1829,7 +1879,7 @@ func newHandlerEngineFromDirectories(t *testing.T, promptDir, profileDir string,
} }
underlays := []configkit.Underlay{{Inputs: promptInputs}, {Inputs: profileInputs}} underlays := []configkit.Underlay{{Inputs: promptInputs}, {Inputs: profileInputs}}
if backend != nil { if backend != nil {
underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{{Source: promptkit.BackendSource("test backend", map[string]promptkit.BackendConfig{"limited": *backend})}}}) underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{{Source: promptkit.BackendSource("test backend", []promptkit.BackendConfig{*backend})}}})
} }
engine, err := promptkit.NewEngine(promptkit.Config{ engine, err := promptkit.NewEngine(promptkit.Config{