diff --git a/convert.go b/convert.go index e3982be..8753254 100644 --- a/convert.go +++ b/convert.go @@ -168,6 +168,7 @@ func toDomainTokenUsagePresence(presence *TokenUsagePresence) *domain.TokenUsage return &domain.TokenUsagePresence{ PromptTokens: presence.PromptTokens, CompletionTokens: presence.CompletionTokens, + ReasoningTokens: presence.ReasoningTokens, TotalTokens: presence.TotalTokens, CachedTokens: presence.CachedTokens, CacheWriteTokens: presence.CacheWriteTokens, @@ -356,6 +357,7 @@ func fromDomainTokenUsage(usage domain.TokenUsage) TokenUsage { return TokenUsage{ PromptTokens: usage.PromptTokens, CompletionTokens: usage.CompletionTokens, + ReasoningTokens: usage.ReasoningTokens, TotalTokens: usage.TotalTokens, CachedTokens: usage.CachedTokens, CacheWriteTokens: usage.CacheWriteTokens, @@ -366,6 +368,7 @@ func toDomainTokenUsage(usage TokenUsage) domain.TokenUsage { return domain.TokenUsage{ PromptTokens: usage.PromptTokens, CompletionTokens: usage.CompletionTokens, + ReasoningTokens: usage.ReasoningTokens, TotalTokens: usage.TotalTokens, CachedTokens: usage.CachedTokens, CacheWriteTokens: usage.CacheWriteTokens, diff --git a/docs/integrations/openai-compatible-chat.md b/docs/integrations/openai-compatible-chat.md index 38528f4..2841129 100644 --- a/docs/integrations/openai-compatible-chat.md +++ b/docs/integrations/openai-compatible-chat.md @@ -163,7 +163,15 @@ data, a second JSON value, absent choices, missing content, `null` content, non-string content, and size overflow are malformed responses and return no partial result. -Each of the five token fields retains whether the provider reported a +For OpenRouter and vLLM, the client also reads reasoning tokens from +`usage.completion_tokens_details.reasoning_tokens`. OpenRouter cache writes +come from `usage.prompt_tokens_details.cache_write_tokens`; vLLM uses +`usage.prompt_tokens_details.created_cache_tokens`. The generic provider keeps +its top-level `usage.cache_write_tokens` mapping. Missing, null, malformed, +fractional, or unrepresentable new details remain unknown without invalidating +otherwise usable content. Existing common count decoding remains strict. + +Each common token field retains whether the provider reported a non-null integer, including zero. Omitted and `null` fields are unknown for observations. A strictly decoded successful envelope can yield observed usage even when a later missing choice or content makes the attempt malformed. diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index e0453c2..7ad1466 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -223,6 +223,8 @@ Validation: focused adapter and smoke coordinator tests with controlled HTTP. ## Stage 7 — Reasoning Values And Provider-Specific Token Decoding +**Completed.** + Touch public/domain token declarations, facade conversions, and `internal/llm/openai_compatible_client.go` plus focused provider files. diff --git a/internal/domain/domain.go b/internal/domain/domain.go index d2c2ab9..2c6cc4a 100644 --- a/internal/domain/domain.go +++ b/internal/domain/domain.go @@ -334,6 +334,7 @@ type GenerateResponse struct { type TokenUsagePresence struct { PromptTokens bool CompletionTokens bool + ReasoningTokens bool TotalTokens bool CachedTokens bool CacheWriteTokens bool @@ -343,6 +344,7 @@ type TokenUsagePresence struct { type TokenUsage struct { PromptTokens int CompletionTokens int + ReasoningTokens int TotalTokens int CachedTokens int CacheWriteTokens int diff --git a/internal/domain/observation.go b/internal/domain/observation.go index b722505..2836c63 100644 --- a/internal/domain/observation.go +++ b/internal/domain/observation.go @@ -56,6 +56,7 @@ type ObservedTokenCount struct { type ObservedTokenUsage struct { PromptTokens ObservedTokenCount CompletionTokens ObservedTokenCount + ReasoningTokens ObservedTokenCount TotalTokens ObservedTokenCount CachedTokens ObservedTokenCount CacheWriteTokens ObservedTokenCount diff --git a/internal/llm/openai_compatible_client.go b/internal/llm/openai_compatible_client.go index 2b0059a..71518f4 100644 --- a/internal/llm/openai_compatible_client.go +++ b/internal/llm/openai_compatible_client.go @@ -88,6 +88,7 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera // PreparedGeneration holds one encoded request and resolved authentication for // repeated attempts. Each execution creates a fresh HTTP request and context. type PreparedGeneration struct { + provider string endpoint string payload []byte apiKey string @@ -156,7 +157,7 @@ func (c *OpenAICompatibleClient) PrepareGeneration(req domain.GenerateRequest) ( httpClient = &http.Client{Timeout: defaults.LLMRequestTimeoutDefault} } return &PreparedGeneration{ - endpoint: endpoint, payload: payload, apiKey: apiKey, + endpoint: endpoint, payload: payload, apiKey: apiKey, provider: EffectiveProvider(req.Target.Provider), timeoutSeconds: req.Target.TimeoutSeconds, httpClient: httpClient, structuredOutputRequested: wirePayload["response_format"] != nil, }, nil @@ -266,16 +267,37 @@ func (p *PreparedGeneration) execute(ctx context.Context, captureEvidence bool) } return outcome } + var writes, reasoning *int + switch p.provider { + case "openrouter", "vllm": + var details map[string]json.RawMessage + if json.Unmarshal(wireResp.Usage.CompletionTokensDetails, &details) == nil { + reasoning = optionalInteger(details["reasoning_tokens"]) + } + if p.provider == "openrouter" { + writes = optionalInteger(wireResp.Usage.PromptTokensDetails.CacheWriteTokens) + } else { + writes = optionalInteger(wireResp.Usage.PromptTokensDetails.CreatedCacheTokens) + } + default: + if len(wireResp.Usage.CacheWriteTokens) > 0 && json.Unmarshal(wireResp.Usage.CacheWriteTokens, &writes) != nil { + outcome.Err = fmt.Errorf("%w: failed to decode response", ErrMalformedResponse) + if contextErr := observedContextError(requestContext, ctx, outcome.Err); contextErr != nil { + outcome.Err = contextErr + } + return outcome + } + } outcome.ResponseDecoded = true outcome.Usage = domain.TokenUsage{ PromptTokens: tokenCount(wireResp.Usage.PromptTokens), CompletionTokens: tokenCount(wireResp.Usage.CompletionTokens), TotalTokens: tokenCount(wireResp.Usage.TotalTokens), CachedTokens: tokenCount(wireResp.Usage.PromptTokensDetails.CachedTokens), - CacheWriteTokens: tokenCount(wireResp.Usage.CacheWriteTokens), + CacheWriteTokens: tokenCount(writes), ReasoningTokens: tokenCount(reasoning), } outcome.UsagePresence = &domain.TokenUsagePresence{ PromptTokens: wireResp.Usage.PromptTokens != nil, CompletionTokens: wireResp.Usage.CompletionTokens != nil, TotalTokens: wireResp.Usage.TotalTokens != nil, CachedTokens: wireResp.Usage.PromptTokensDetails.CachedTokens != nil, - CacheWriteTokens: wireResp.Usage.CacheWriteTokens != nil, + CacheWriteTokens: writes != nil, ReasoningTokens: reasoning != nil, } if len(wireResp.Choices) == 0 { @@ -540,12 +562,26 @@ type openAIChatResponse struct { CompletionTokens *int `json:"completion_tokens"` TotalTokens *int `json:"total_tokens"` PromptTokensDetails struct { - CachedTokens *int `json:"cached_tokens"` + CachedTokens *int `json:"cached_tokens"` + CacheWriteTokens json.RawMessage `json:"cache_write_tokens"` + CreatedCacheTokens json.RawMessage `json:"created_cache_tokens"` } `json:"prompt_tokens_details"` - CacheWriteTokens *int `json:"cache_write_tokens"` + CompletionTokensDetails json.RawMessage `json:"completion_tokens_details"` + CacheWriteTokens json.RawMessage `json:"cache_write_tokens"` } `json:"usage"` } +func optionalInteger(raw json.RawMessage) *int { + if len(raw) == 0 { + return nil + } + var value *int + if json.Unmarshal(raw, &value) != nil { + return nil + } + return value +} + type openAIResponseFormat struct { Type string `json:"type"` JSONSchema *openAIJSONSchemaEnvelope `json:"json_schema,omitempty"` diff --git a/internal/llm/openai_compatible_client_test.go b/internal/llm/openai_compatible_client_test.go index 6276132..1f3d385 100644 --- a/internal/llm/openai_compatible_client_test.go +++ b/internal/llm/openai_compatible_client_test.go @@ -882,6 +882,45 @@ func TestOpenAICompatibleUsagePresence(t *testing.T) { } } +func TestProviderSpecificUsageDetails(t *testing.T) { + for _, tc := range []struct { + name, provider, usage string + writes, reasoning int + writesKnown, reasoningKnown bool + }{ + {name: "openrouter nested", provider: "openrouter", usage: `"prompt_tokens_details":{"cached_tokens":2,"cache_write_tokens":3,"created_cache_tokens":7},"completion_tokens_details":{"reasoning_tokens":4},"cache_write_tokens":9`, writes: 3, reasoning: 4, writesKnown: true, reasoningKnown: true}, + {name: "vllm nested", provider: "vllm", usage: `"prompt_tokens_details":{"cached_tokens":2,"cache_write_tokens":3,"created_cache_tokens":7},"completion_tokens_details":{"reasoning_tokens":0},"cache_write_tokens":9`, writes: 7, writesKnown: true, reasoningKnown: true}, + {name: "generic ignores details", provider: "openai-compatible", usage: `"prompt_tokens_details":{"cache_write_tokens":3},"completion_tokens_details":{"reasoning_tokens":4},"cache_write_tokens":9`, writes: 9, writesKnown: true}, + {name: "missing selected field", provider: "openrouter", usage: `"prompt_tokens_details":{"created_cache_tokens":7},"cache_write_tokens":9`}, + {name: "invalid optional fields", provider: "vllm", usage: `"prompt_tokens_details":{"created_cache_tokens":"7","cached_tokens":2},"completion_tokens_details":{"reasoning_tokens":2.5},"cache_write_tokens":{}`}, + {name: "overflow and null", provider: "openrouter", usage: `"prompt_tokens_details":{"cache_write_tokens":999999999999999999999999},"completion_tokens_details":{"reasoning_tokens":null}`}, + {name: "vllm disabled details", provider: "vllm", usage: `"prompt_tokens":12`}, + } { + t.Run(tc.name, func(t *testing.T) { + provider := newRecordingProvider(t) + provider.respond(http.StatusOK, `{"usage":{`+tc.usage+`},"choices":[{"message":{"content":"ok"}}]}`) + request := provider.generateRequest() + request.Target.Provider = tc.provider + response, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{}).Generate(context.Background(), request) + if err != nil || response.Usage.CacheWriteTokens != tc.writes || response.Usage.ReasoningTokens != tc.reasoning || response.UsagePresence.CacheWriteTokens != tc.writesKnown || response.UsagePresence.ReasoningTokens != tc.reasoningKnown { + t.Fatalf("response=%+v error=%v", response, err) + } + }) + } + provider := newRecordingProvider(t) + provider.respond(http.StatusOK, `{"usage":{"completion_tokens_details":{"reasoning_tokens":5}},"choices":[]}`) + request := provider.generateRequest() + request.Target.Provider = "openrouter" + prepared, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{}).PrepareGeneration(request) + if err != nil { + t.Fatal(err) + } + outcome := prepared.Execute(context.Background()) + if !errors.Is(outcome.Err, ErrMalformedResponse) || outcome.Usage.ReasoningTokens != 5 || !outcome.UsagePresence.ReasoningTokens { + t.Fatalf("outcome=%+v", outcome) + } +} + func checkOptionalResponseFieldOmission(t *testing.T) { provider := newRecordingProvider(t) client := NewOpenAICompatibleClient(OpenAICompatibleConfig{}) diff --git a/json_contract_test.go b/json_contract_test.go index e1ea2ac..2e9f8d6 100644 --- a/json_contract_test.go +++ b/json_contract_test.go @@ -19,9 +19,9 @@ func stringPointerForTest(value string) *string { func TestGenerateResponseUsagePresenceJSON(t *testing.T) { value := promptkit.GenerateResponse{ Content: "ok", - Usage: promptkit.TokenUsage{TotalTokens: 5}, + Usage: promptkit.TokenUsage{TotalTokens: 5, ReasoningTokens: 2}, UsagePresence: &promptkit.TokenUsagePresence{ - PromptTokens: true, CompletionTokens: true, TotalTokens: true, + PromptTokens: true, CompletionTokens: true, TotalTokens: true, ReasoningTokens: true, CachedTokens: true, CacheWriteTokens: true, }, } @@ -30,6 +30,9 @@ func TestGenerateResponseUsagePresenceJSON(t *testing.T) { t.Fatal(err) } object := decodeJSONObject(t, payload) + if string(decodeJSONObject(t, object["usage"])["reasoning_tokens"]) != "2" { + t.Fatalf("reasoning field missing: %s", payload) + } if _, ok := object["usage_presence"]; !ok { t.Fatalf("missing usage_presence: %s", payload) } diff --git a/observations.go b/observations.go index c132318..4ead910 100644 --- a/observations.go +++ b/observations.go @@ -95,6 +95,7 @@ type ObservedTokenCount struct { type ObservedTokenUsage struct { PromptTokens ObservedTokenCount CompletionTokens ObservedTokenCount + ReasoningTokens ObservedTokenCount TotalTokens ObservedTokenCount CachedTokens ObservedTokenCount CacheWriteTokens ObservedTokenCount @@ -157,7 +158,7 @@ func fromDomainExecutionEvent(value domain.ExecutionEvent) ExecutionEvent { Outcome: ExecutionOutcome(value.Outcome), Failure: ExecutionFailureKind(value.Failure), StatusCode: value.StatusCode, RetriesExhausted: value.RetriesExhausted, Usage: ObservedTokenUsage{PromptTokens: count(value.Usage.PromptTokens), CompletionTokens: count(value.Usage.CompletionTokens), - TotalTokens: count(value.Usage.TotalTokens), CachedTokens: count(value.Usage.CachedTokens), CacheWriteTokens: count(value.Usage.CacheWriteTokens)}, + TotalTokens: count(value.Usage.TotalTokens), ReasoningTokens: count(value.Usage.ReasoningTokens), CachedTokens: count(value.Usage.CachedTokens), CacheWriteTokens: count(value.Usage.CacheWriteTokens)}, Attempts: value.Attempts, TransientRetries: value.TransientRetries, RepairCalls: value.RepairCalls, } } diff --git a/smoke_diagnostics.go b/smoke_diagnostics.go index 25a95a4..d69ae59 100644 --- a/smoke_diagnostics.go +++ b/smoke_diagnostics.go @@ -368,6 +368,7 @@ func fromDomainObservedTokenUsage(usage domain.ObservedTokenUsage) ObservedToken return ObservedTokenUsage{ PromptTokens: fromDomainObservedTokenCount(usage.PromptTokens), CompletionTokens: fromDomainObservedTokenCount(usage.CompletionTokens), + ReasoningTokens: fromDomainObservedTokenCount(usage.ReasoningTokens), TotalTokens: fromDomainObservedTokenCount(usage.TotalTokens), CachedTokens: fromDomainObservedTokenCount(usage.CachedTokens), CacheWriteTokens: fromDomainObservedTokenCount(usage.CacheWriteTokens), diff --git a/types.go b/types.go index 47a686c..1ff2aeb 100644 --- a/types.go +++ b/types.go @@ -568,6 +568,8 @@ type TokenUsage struct { PromptTokens int `json:"prompt_tokens"` // CompletionTokens is the reported generated-token count. CompletionTokens int `json:"completion_tokens"` + // ReasoningTokens is the reported reasoning subset of completion tokens. + ReasoningTokens int `json:"reasoning_tokens"` // TotalTokens is the reported total-token count. TotalTokens int `json:"total_tokens"` // CachedTokens is the reported cached-input-token count. @@ -697,6 +699,7 @@ type GenerateResponse struct { type TokenUsagePresence struct { PromptTokens bool `json:"prompt_tokens"` CompletionTokens bool `json:"completion_tokens"` + ReasoningTokens bool `json:"reasoning_tokens"` TotalTokens bool `json:"total_tokens"` CachedTokens bool `json:"cached_tokens"` CacheWriteTokens bool `json:"cache_write_tokens"`