128 lines
5.3 KiB
Go
128 lines
5.3 KiB
Go
package promptkit_test
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"io"
|
|
"math"
|
|
"net/http"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
|
|
"go.maximumdirect.net/configkit"
|
|
"go.maximumdirect.net/promptkit"
|
|
)
|
|
|
|
func TestConcurrentInvocationObservationAcrossProviders(t *testing.T) {
|
|
pendingStarted, release := make(chan struct{}), make(chan struct{})
|
|
releasePending := sync.OnceFunc(func() { close(release) })
|
|
defer releasePending()
|
|
var retryCalls atomic.Int32
|
|
transport := costRoundTrip(func(req *http.Request) (*http.Response, error) {
|
|
var body struct {
|
|
Model string `json:"model"`
|
|
}
|
|
if err := json.NewDecoder(req.Body).Decode(&body); err != nil {
|
|
return nil, err
|
|
}
|
|
status, payload := 200, ""
|
|
switch body.Model {
|
|
case "router":
|
|
if retryCalls.Add(1) == 1 {
|
|
status, payload = 503, `{"error":{}}`
|
|
} else {
|
|
payload = `{"usage":{"prompt_tokens":100,"completion_tokens":20,"total_tokens":120,"prompt_tokens_details":{"cached_tokens":40,"cache_write_tokens":5},"completion_tokens_details":{"reasoning_tokens":3},"cost":0.02},"choices":[{"message":{"content":"ok"}}]}`
|
|
}
|
|
case "failure":
|
|
payload = `{"usage":{"prompt_tokens":30,"completion_tokens":6,"total_tokens":36,"prompt_tokens_details":{"cached_tokens":15,"cache_write_tokens":1},"completion_tokens_details":{"reasoning_tokens":1},"cost":0.01},"choices":[]}`
|
|
case "vllm":
|
|
payload = `{"usage":{"prompt_tokens":20,"completion_tokens":8,"total_tokens":28,"prompt_tokens_details":{"cached_tokens":10,"created_cache_tokens":4},"completion_tokens_details":{"reasoning_tokens":2}},"choices":[{"message":{"content":"ok"}}]}`
|
|
case "pending":
|
|
close(pendingStarted)
|
|
<-release
|
|
payload = `{"usage":{"prompt_tokens":10,"completion_tokens":2,"total_tokens":12,"prompt_tokens_details":{"cached_tokens":0,"created_cache_tokens":0},"completion_tokens_details":{"reasoning_tokens":0}},"choices":[{"message":{"content":"ok"}}]}`
|
|
default:
|
|
t.Errorf("unexpected model %q", body.Model)
|
|
}
|
|
return &http.Response{StatusCode: status, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(payload))}, nil
|
|
})
|
|
engine := func(provider, model string) *promptkit.Engine {
|
|
t.Helper()
|
|
input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("workflow", map[string]any{
|
|
"backends": []any{map[string]any{"id": "backend", "endpoint": "https://example.invalid/v1", "provider": provider, "zero_cost": true}},
|
|
"profiles": []any{map[string]any{"id": "profile", "backend": "backend", "model": model}},
|
|
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},
|
|
})}
|
|
result, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}, HTTPClient: &http.Client{Transport: transport}}, promptkit.WithBundledOpenRouter(false))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return result
|
|
}
|
|
router, vllm := engine("openrouter", "router"), engine("vllm", "vllm")
|
|
var group promptkit.ExecutionGroup
|
|
if empty := group.Snapshot(); len(empty.Members) != 0 || empty.CostUSD != (promptkit.ObservedCost{Known: true, Complete: true}) {
|
|
t.Fatalf("empty=%+v", empty)
|
|
}
|
|
ctx := promptkit.WithExecutionObserver(context.Background(), group.Observe)
|
|
results := make(chan error, 4)
|
|
var completed sync.WaitGroup
|
|
var pendingDone sync.WaitGroup
|
|
start := func(e *promptkit.Engine, model string, fail bool) {
|
|
completed.Add(1)
|
|
go func() {
|
|
defer completed.Done()
|
|
req := promptkit.RunRequest{PromptID: "prompt"}
|
|
if model != "" {
|
|
req.Execution = &promptkit.ExecutionTargetOverride{Model: model}
|
|
}
|
|
_, err := e.Run(ctx, req)
|
|
if fail && err != nil {
|
|
err = nil
|
|
} else if fail {
|
|
err = io.ErrUnexpectedEOF
|
|
}
|
|
results <- err
|
|
}()
|
|
}
|
|
pendingDone.Add(1)
|
|
go func() {
|
|
defer pendingDone.Done()
|
|
_, err := vllm.Run(ctx, promptkit.RunRequest{PromptID: "prompt", Execution: &promptkit.ExecutionTargetOverride{Model: "pending"}})
|
|
results <- err
|
|
}()
|
|
<-pendingStarted
|
|
if pending := group.Snapshot(); len(pending.Members) != 1 || pending.CostUSD != (promptkit.ObservedCost{}) || pending.TerminalCoverageComplete {
|
|
t.Fatalf("pending=%+v", pending)
|
|
}
|
|
start(router, "", false)
|
|
start(router, "failure", true)
|
|
start(vllm, "", false)
|
|
completed.Wait()
|
|
for range 3 {
|
|
if err := <-results; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
snapshot := group.Snapshot()
|
|
if len(snapshot.Members) != 4 || snapshot.TerminalCoverageComplete || snapshot.Attempts != 4 || !snapshot.CostUSD.Known || !snapshot.CostUSD.Complete || snapshot.CostUSD.Source != promptkit.CostSourceMixed || math.Abs(snapshot.CostUSD.Value-.03) > 1e-12 {
|
|
t.Fatalf("snapshot=%+v", snapshot)
|
|
}
|
|
if snapshot.Usage.PromptTokens.Value != 150 || snapshot.Usage.ReasoningTokens.Value != 6 || snapshot.Usage.CachedTokens.Value != 65 || snapshot.Usage.CacheWriteTokens.Value != 10 || snapshot.Usage.PromptTokens.Complete {
|
|
t.Fatalf("usage=%+v", snapshot.Usage)
|
|
}
|
|
if _, ok := snapshot.Usage.CacheHitRatio(); ok {
|
|
t.Fatal("ratio available despite an unreported retry")
|
|
}
|
|
releasePending()
|
|
if err := <-results; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
pendingDone.Wait()
|
|
if final := group.Snapshot(); !final.TerminalCoverageComplete || len(final.Members) != 4 || final.Attempts != 5 || math.Abs(final.CostUSD.Value-.03) > 1e-12 {
|
|
t.Fatalf("final=%+v", final)
|
|
}
|
|
}
|