package promptkit_test import ( "context" "encoding/json" "io" "math" "net/http" "strings" "sync" "sync/atomic" "testing" "go.maximumdirect.net/configkit" "go.maximumdirect.net/promptkit" ) func TestConcurrentInvocationObservationAcrossProviders(t *testing.T) { pendingStarted, release := make(chan struct{}), make(chan struct{}) releasePending := sync.OnceFunc(func() { close(release) }) defer releasePending() var retryCalls atomic.Int32 transport := costRoundTrip(func(req *http.Request) (*http.Response, error) { var body struct { Model string `json:"model"` } if err := json.NewDecoder(req.Body).Decode(&body); err != nil { return nil, err } status, payload := 200, "" switch body.Model { case "router": if retryCalls.Add(1) == 1 { status, payload = 503, `{"error":{}}` } else { payload = `{"usage":{"prompt_tokens":100,"completion_tokens":20,"total_tokens":120,"prompt_tokens_details":{"cached_tokens":40,"cache_write_tokens":5},"completion_tokens_details":{"reasoning_tokens":3},"cost":0.02},"choices":[{"message":{"content":"ok"}}]}` } case "failure": payload = `{"usage":{"prompt_tokens":30,"completion_tokens":6,"total_tokens":36,"prompt_tokens_details":{"cached_tokens":15,"cache_write_tokens":1},"completion_tokens_details":{"reasoning_tokens":1},"cost":0.01},"choices":[]}` case "vllm": payload = `{"usage":{"prompt_tokens":20,"completion_tokens":8,"total_tokens":28,"prompt_tokens_details":{"cached_tokens":10,"created_cache_tokens":4},"completion_tokens_details":{"reasoning_tokens":2}},"choices":[{"message":{"content":"ok"}}]}` case "pending": close(pendingStarted) <-release payload = `{"usage":{"prompt_tokens":10,"completion_tokens":2,"total_tokens":12,"prompt_tokens_details":{"cached_tokens":0,"created_cache_tokens":0},"completion_tokens_details":{"reasoning_tokens":0}},"choices":[{"message":{"content":"ok"}}]}` default: t.Errorf("unexpected model %q", body.Model) } return &http.Response{StatusCode: status, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(payload))}, nil }) engine := func(provider, model string) *promptkit.Engine { t.Helper() input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("workflow", map[string]any{ "backends": []any{map[string]any{"id": "backend", "endpoint": "https://example.invalid/v1", "provider": provider, "zero_cost": true}}, "profiles": []any{map[string]any{"id": "profile", "backend": "backend", "model": model}}, "prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}}, })} result, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}, HTTPClient: &http.Client{Transport: transport}}, promptkit.WithBundledOpenRouter(false)) if err != nil { t.Fatal(err) } return result } router, vllm := engine("openrouter", "router"), engine("vllm", "vllm") var group promptkit.ExecutionGroup if empty := group.Snapshot(); len(empty.Members) != 0 || empty.CostUSD != (promptkit.ObservedCost{Known: true, Complete: true}) { t.Fatalf("empty=%+v", empty) } ctx := promptkit.WithExecutionObserver(context.Background(), group.Observe) results := make(chan error, 4) var completed sync.WaitGroup var pendingDone sync.WaitGroup start := func(e *promptkit.Engine, model string, fail bool) { completed.Add(1) go func() { defer completed.Done() req := promptkit.RunRequest{PromptID: "prompt"} if model != "" { req.Execution = &promptkit.ExecutionTargetOverride{Model: model} } _, err := e.Run(ctx, req) if fail && err != nil { err = nil } else if fail { err = io.ErrUnexpectedEOF } results <- err }() } pendingDone.Add(1) go func() { defer pendingDone.Done() _, err := vllm.Run(ctx, promptkit.RunRequest{PromptID: "prompt", Execution: &promptkit.ExecutionTargetOverride{Model: "pending"}}) results <- err }() <-pendingStarted if pending := group.Snapshot(); len(pending.Members) != 1 || pending.CostUSD != (promptkit.ObservedCost{}) || pending.TerminalCoverageComplete { t.Fatalf("pending=%+v", pending) } start(router, "", false) start(router, "failure", true) start(vllm, "", false) completed.Wait() for range 3 { if err := <-results; err != nil { t.Fatal(err) } } snapshot := group.Snapshot() if len(snapshot.Members) != 4 || snapshot.TerminalCoverageComplete || snapshot.Attempts != 4 || !snapshot.CostUSD.Known || !snapshot.CostUSD.Complete || snapshot.CostUSD.Source != promptkit.CostSourceMixed || math.Abs(snapshot.CostUSD.Value-.03) > 1e-12 { t.Fatalf("snapshot=%+v", snapshot) } if snapshot.Usage.PromptTokens.Value != 150 || snapshot.Usage.ReasoningTokens.Value != 6 || snapshot.Usage.CachedTokens.Value != 65 || snapshot.Usage.CacheWriteTokens.Value != 10 || snapshot.Usage.PromptTokens.Complete { t.Fatalf("usage=%+v", snapshot.Usage) } if _, ok := snapshot.Usage.CacheHitRatio(); ok { t.Fatal("ratio available despite an unreported retry") } releasePending() if err := <-results; err != nil { t.Fatal(err) } pendingDone.Wait() if final := group.Snapshot(); !final.TerminalCoverageComplete || len(final.Members) != 4 || final.Attempts != 5 || math.Abs(final.CostUSD.Value-.03) > 1e-12 { t.Fatalf("final=%+v", final) } }