From cd0e1f5d396b07fb544176bd6234a5bc2dbbb0f1 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 22 Nov 2026 18:25:32 +0000 Subject: [PATCH] Capture optional provider reported costs --- docs/integrations/openai-compatible-chat.md | 3 ++ docs/roadmap/implementation.md | 2 + internal/llm/openai_compatible_client.go | 18 ++++++++- internal/llm/openai_compatible_client_test.go | 40 +++++++++++++++++++ llm_adapter_internal_test.go | 5 ++- 5 files changed, 66 insertions(+), 2 deletions(-) diff --git a/docs/integrations/openai-compatible-chat.md b/docs/integrations/openai-compatible-chat.md index 2841129..ddbad95 100644 --- a/docs/integrations/openai-compatible-chat.md +++ b/docs/integrations/openai-compatible-chat.md @@ -170,6 +170,9 @@ come from `usage.prompt_tokens_details.cache_write_tokens`; vLLM uses its top-level `usage.cache_write_tokens` mapping. Missing, null, malformed, fractional, or unrepresentable new details remain unknown without invalidating otherwise usable content. Existing common count decoding remains strict. +OpenRouter's `usage.cost` is an optional USD amount; malformed or non-finite +amounts are ignored without invalidating content. The generic and vLLM +providers do not interpret that field. Each common token field retains whether the provider reported a non-null integer, including zero. Omitted and `null` fields are unknown for diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 95ddf1a..0950ac4 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -304,6 +304,8 @@ Validation: pure accounting and public serialization tests; no pricing library. ## Stage 10 — Optional Cost At The Model-Client Boundary +**Completed.** + - Decode OpenRouter `usage.cost` independently using local tolerant finite-float handling. Missing/null/string/object/overflow values are unknown; invalid framing or common envelope fields retain existing failures. Generic/vLLM do diff --git a/internal/llm/openai_compatible_client.go b/internal/llm/openai_compatible_client.go index 71518f4..6800223 100644 --- a/internal/llm/openai_compatible_client.go +++ b/internal/llm/openai_compatible_client.go @@ -105,6 +105,7 @@ type AttemptOutcome struct { StatusCode int Usage domain.TokenUsage UsagePresence *domain.TokenUsagePresence + CostUSD *float64 RetryNotBefore time.Time Evidence HTTPExecutionEvidence StructuredOutputRequested *bool @@ -299,6 +300,9 @@ func (p *PreparedGeneration) execute(ctx context.Context, captureEvidence bool) TotalTokens: wireResp.Usage.TotalTokens != nil, CachedTokens: wireResp.Usage.PromptTokensDetails.CachedTokens != nil, CacheWriteTokens: writes != nil, ReasoningTokens: reasoning != nil, } + if p.provider == "openrouter" { + outcome.CostUSD = optionalFloat(wireResp.Usage.Cost) + } if len(wireResp.Choices) == 0 { outcome.Err = fmt.Errorf("%w: no choices returned", ErrMalformedResponse) @@ -319,7 +323,7 @@ func (p *PreparedGeneration) execute(ctx context.Context, captureEvidence bool) } return outcome } - outcome.Response = &domain.GenerateResponse{Content: *content, Usage: outcome.Usage, UsagePresence: outcome.UsagePresence} + outcome.Response = &domain.GenerateResponse{Content: *content, Usage: outcome.Usage, UsagePresence: outcome.UsagePresence, CostUSD: outcome.CostUSD} outcome.GenerationCompleted = true if captureEvidence { if contextErr := observedContextError(requestContext, ctx, nil); contextErr != nil { @@ -568,6 +572,7 @@ type openAIChatResponse struct { } `json:"prompt_tokens_details"` CompletionTokensDetails json.RawMessage `json:"completion_tokens_details"` CacheWriteTokens json.RawMessage `json:"cache_write_tokens"` + Cost json.RawMessage `json:"cost"` } `json:"usage"` } @@ -582,6 +587,17 @@ func optionalInteger(raw json.RawMessage) *int { return value } +func optionalFloat(raw json.RawMessage) *float64 { + if len(raw) == 0 { + return nil + } + var value *float64 + if json.Unmarshal(raw, &value) != nil || (value != nil && (math.IsNaN(*value) || math.IsInf(*value, 0))) { + return nil + } + return value +} + type openAIResponseFormat struct { Type string `json:"type"` JSONSchema *openAIJSONSchemaEnvelope `json:"json_schema,omitempty"` diff --git a/internal/llm/openai_compatible_client_test.go b/internal/llm/openai_compatible_client_test.go index 1f3d385..b4d3d43 100644 --- a/internal/llm/openai_compatible_client_test.go +++ b/internal/llm/openai_compatible_client_test.go @@ -921,6 +921,46 @@ func TestProviderSpecificUsageDetails(t *testing.T) { } } +func TestProviderReportedCostIsOptional(t *testing.T) { + for _, tc := range []struct { + name, provider, cost string + known bool + value float64 + }{ + {"charge", "openrouter", `0.002`, true, .002}, + {"zero", "openrouter", `0`, true, 0}, + {"null", "openrouter", `null`, false, 0}, + {"string", "openrouter", `"0.2"`, false, 0}, + {"object", "openrouter", `{}`, false, 0}, + {"overflow", "openrouter", `1e9999`, false, 0}, + {"generic ignores cost", "openai-compatible", `0.02`, false, 0}, + {"vllm ignores cost", "vllm", `0.02`, false, 0}, + } { + t.Run(tc.name, func(t *testing.T) { + provider := newRecordingProvider(t) + provider.respond(http.StatusOK, `{"usage":{"cost":`+tc.cost+`},"choices":[{"message":{"content":"ok"}}]}`) + request := provider.generateRequest() + request.Target.Provider = tc.provider + response, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{}).Generate(context.Background(), request) + if err != nil || response == nil || (response.CostUSD != nil) != tc.known || (response.CostUSD != nil && *response.CostUSD != tc.value) { + t.Fatalf("response=%+v err=%v", response, err) + } + }) + } + provider := newRecordingProvider(t) + provider.respond(http.StatusOK, `{"usage":{"cost":0.02},"choices":[]}`) + request := provider.generateRequest() + request.Target.Provider = "openrouter" + prepared, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{}).PrepareGeneration(request) + if err != nil { + t.Fatal(err) + } + outcome := prepared.Execute(context.Background()) + if !errors.Is(outcome.Err, ErrMalformedResponse) || outcome.CostUSD == nil || *outcome.CostUSD != .02 { + t.Fatalf("outcome=%+v", outcome) + } +} + func checkOptionalResponseFieldOmission(t *testing.T) { provider := newRecordingProvider(t) client := NewOpenAICompatibleClient(OpenAICompatibleConfig{}) diff --git a/llm_adapter_internal_test.go b/llm_adapter_internal_test.go index 9fc2ec0..6f299fa 100644 --- a/llm_adapter_internal_test.go +++ b/llm_adapter_internal_test.go @@ -74,10 +74,12 @@ func TestPublicLLMClientAdapterCopiesUsagePresence(t *testing.T) { func TestPublicLLMClientAdapterPreservesOwnedResponseAlongsideError(t *testing.T) { presence := &TokenUsagePresence{PromptTokens: true} + cost := .125 source := &GenerateResponse{ Content: "returned with error", Usage: TokenUsage{PromptTokens: 6}, UsagePresence: presence, + CostUSD: &cost, } marker := errors.New("injected failure") adapter := publicLLMClientAdapter{client: responseAndErrorClient{response: source, err: marker}} @@ -88,7 +90,8 @@ func TestPublicLLMClientAdapterPreservesOwnedResponseAlongsideError(t *testing.T } source.Content = "mutated after return" presence.PromptTokens = false - if got.Content != "returned with error" || got.UsagePresence == nil || !got.UsagePresence.PromptTokens { + cost = .5 + if got.Content != "returned with error" || got.UsagePresence == nil || !got.UsagePresence.PromptTokens || got.CostUSD == nil || *got.CostUSD != .125 { t.Fatalf("adapter response was not owned: %+v", got) } }