diff --git a/cache_hit_contract_test.go b/cache_hit_contract_test.go new file mode 100644 index 0000000..7662dae --- /dev/null +++ b/cache_hit_contract_test.go @@ -0,0 +1,53 @@ +package promptkit_test + +import ( + "math" + "testing" + + "go.maximumdirect.net/promptkit" +) + +func TestCacheHitRatioUsesCompleteConsistentCounts(t *testing.T) { + known := func(v int) promptkit.ObservedTokenCount { + return promptkit.ObservedTokenCount{Value: v, Known: true, Complete: true} + } + for _, tc := range []struct { + name string + input, reads promptkit.ObservedTokenCount + ratio float64 + ok bool + }{ + {name: "fraction", input: known(8), reads: known(3), ratio: .375, ok: true}, + {name: "zero reads", input: known(8), reads: known(0), ok: true}, + {name: "zero input", input: known(0), reads: known(0)}, + {name: "negative reads", input: known(8), reads: known(-1)}, + {name: "reads exceed input", input: known(8), reads: known(9)}, + {name: "unknown", input: known(8)}, + {name: "incomplete", input: known(8), reads: promptkit.ObservedTokenCount{Value: 3, Known: true}}, + } { + t.Run(tc.name, func(t *testing.T) { + usage := promptkit.ObservedTokenUsage{PromptTokens: tc.input, CachedTokens: tc.reads} + got, ok := usage.CacheHitRatio() + if ok != tc.ok || math.Abs(got-tc.ratio) > 1e-12 { + t.Fatalf("ratio=%v available=%t", got, ok) + } + }) + } +} + +func TestGroupCacheRatioUsesSummedTokensAndDeduplicatesTerminals(t *testing.T) { + known := func(v int) promptkit.ObservedTokenCount { + return promptkit.ObservedTokenCount{Value: v, Known: true, Complete: true} + } + var group promptkit.ExecutionGroup + first := promptkit.ExecutionEvent{RunID: "a", Kind: promptkit.ExecutionFinished, Usage: promptkit.ObservedTokenUsage{PromptTokens: known(10), CachedTokens: known(10), ReasoningTokens: known(1)}} + second := promptkit.ExecutionEvent{RunID: "b", Kind: promptkit.ExecutionFinished, Usage: promptkit.ObservedTokenUsage{PromptTokens: known(90), CachedTokens: known(0), ReasoningTokens: known(2)}} + group.Observe(first) + group.Observe(first) + group.Observe(second) + snapshot := group.Snapshot() + ratio, ok := snapshot.Usage.CacheHitRatio() + if !ok || math.Abs(ratio-.1) > 1e-12 || snapshot.Usage.ReasoningTokens != known(3) { + t.Fatalf("summary=%+v ratio=%v available=%t", snapshot, ratio, ok) + } +} diff --git a/docs/internal/observation.md b/docs/internal/observation.md index c90ee62..732be3c 100644 --- a/docs/internal/observation.md +++ b/docs/internal/observation.md @@ -56,6 +56,10 @@ attempts above the first in a generation, and repair generations with at least one dispatch. A terminal event is an aggregate and must not be added to its attempt events. +Reasoning counts use the same per-field rules and are a subset of completion +tokens. The public cache-hit helper derives a ratio from complete, consistent +summed input and cache-read counts; it does not aggregate ratios or use writes. + An execution group is a zero-ready synchronized collector of selected events. It enrolls by run ID and aggregates each run's first terminal counters and presence-aware usage exactly once. Snapshots copy members and totals under one diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 7ad1466..3db1644 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -254,6 +254,8 @@ Validation: model-client parsing tests and public token JSON round trips. ## Stage 8 — Token Aggregation And Cache-Hit Helper +**Completed.** + Touch observation usage accumulators/recorder/group, runner `addTokenUsage`, public event/group conversions, and related fixtures. diff --git a/execution_group.go b/execution_group.go index b09b9c0..88bd121 100644 --- a/execution_group.go +++ b/execution_group.go @@ -37,7 +37,7 @@ func (g *ExecutionGroup) Observe(event ExecutionEvent) { return domain.ObservedTokenCount{Value: c.Value, Known: c.Known, Complete: c.Complete} } g.group.Observe(domain.ExecutionEvent{Kind: domain.ExecutionEventKind(event.Kind), RunID: event.RunID, Attempts: event.Attempts, TransientRetries: event.TransientRetries, RepairCalls: event.RepairCalls, - Usage: domain.ObservedTokenUsage{PromptTokens: count(event.Usage.PromptTokens), CompletionTokens: count(event.Usage.CompletionTokens), TotalTokens: count(event.Usage.TotalTokens), CachedTokens: count(event.Usage.CachedTokens), CacheWriteTokens: count(event.Usage.CacheWriteTokens)}}) + Usage: domain.ObservedTokenUsage{PromptTokens: count(event.Usage.PromptTokens), CompletionTokens: count(event.Usage.CompletionTokens), ReasoningTokens: count(event.Usage.ReasoningTokens), TotalTokens: count(event.Usage.TotalTokens), CachedTokens: count(event.Usage.CachedTokens), CacheWriteTokens: count(event.Usage.CacheWriteTokens)}}) } // Snapshot returns an internally consistent, caller-owned summary. Coverage @@ -52,5 +52,5 @@ func (g *ExecutionGroup) Snapshot() ExecutionGroupSummary { members[i] = ExecutionGroupMember{RunID: member.RunID, TerminalObserved: member.TerminalObserved} } return ExecutionGroupSummary{Members: members, TerminalCoverageComplete: value.TerminalCoverageComplete, Attempts: value.Attempts, TransientRetries: value.TransientRetries, RepairCalls: value.RepairCalls, - Usage: ObservedTokenUsage{PromptTokens: count(value.Usage.PromptTokens), CompletionTokens: count(value.Usage.CompletionTokens), TotalTokens: count(value.Usage.TotalTokens), CachedTokens: count(value.Usage.CachedTokens), CacheWriteTokens: count(value.Usage.CacheWriteTokens)}} + Usage: ObservedTokenUsage{PromptTokens: count(value.Usage.PromptTokens), CompletionTokens: count(value.Usage.CompletionTokens), ReasoningTokens: count(value.Usage.ReasoningTokens), TotalTokens: count(value.Usage.TotalTokens), CachedTokens: count(value.Usage.CachedTokens), CacheWriteTokens: count(value.Usage.CacheWriteTokens)}} } diff --git a/internal/observation/group.go b/internal/observation/group.go index def85a6..4f3150a 100644 --- a/internal/observation/group.go +++ b/internal/observation/group.go @@ -36,7 +36,7 @@ func (g *Group) Observe(event domain.ExecutionEvent) { g.repairs += event.RepairCalls if g.terminals == 0 { complete := domain.ObservedTokenCount{Complete: true} - g.usage = domain.ObservedTokenUsage{PromptTokens: complete, CompletionTokens: complete, TotalTokens: complete, CachedTokens: complete, CacheWriteTokens: complete} + g.usage = domain.ObservedTokenUsage{PromptTokens: complete, CompletionTokens: complete, ReasoningTokens: complete, TotalTokens: complete, CachedTokens: complete, CacheWriteTokens: complete} } mergeGroupUsage(&g.usage, event.Usage) g.terminals++ @@ -52,7 +52,7 @@ func (g *Group) Snapshot() domain.ExecutionGroupSummary { } if len(g.members) == 0 { zero := domain.ObservedTokenCount{Known: true, Complete: true} - result.Usage = domain.ObservedTokenUsage{PromptTokens: zero, CompletionTokens: zero, TotalTokens: zero, CachedTokens: zero, CacheWriteTokens: zero} + result.Usage = domain.ObservedTokenUsage{PromptTokens: zero, CompletionTokens: zero, ReasoningTokens: zero, TotalTokens: zero, CachedTokens: zero, CacheWriteTokens: zero} } return result } @@ -67,6 +67,7 @@ func mergeGroupUsage(total *domain.ObservedTokenUsage, next domain.ObservedToken } merge(&total.PromptTokens, next.PromptTokens) merge(&total.CompletionTokens, next.CompletionTokens) + merge(&total.ReasoningTokens, next.ReasoningTokens) merge(&total.TotalTokens, next.TotalTokens) merge(&total.CachedTokens, next.CachedTokens) merge(&total.CacheWriteTokens, next.CacheWriteTokens) diff --git a/internal/observation/recorder.go b/internal/observation/recorder.go index a076a94..58b09f8 100644 --- a/internal/observation/recorder.go +++ b/internal/observation/recorder.go @@ -260,7 +260,7 @@ type usageField struct { known, complete bool } type usageTotals struct { - prompt, completion, total, cached, cacheWrite usageField + prompt, completion, reasoning, total, cached, cacheWrite usageField } // UsageAccumulator combines presence-aware usage across dispatched attempts. @@ -286,7 +286,7 @@ func (u UsageAccumulator) Observed() domain.ObservedTokenUsage { } func completeUsageTotals() usageTotals { - return usageTotals{prompt: usageField{complete: true}, completion: usageField{complete: true}, + return usageTotals{prompt: usageField{complete: true}, completion: usageField{complete: true}, reasoning: usageField{complete: true}, total: usageField{complete: true}, cached: usageField{complete: true}, cacheWrite: usageField{complete: true}} } @@ -294,6 +294,7 @@ func (u *usageTotals) add(usage domain.TokenUsage, presence *domain.TokenUsagePr known := usagePresence(usage, presence) u.prompt.add(usage.PromptTokens, known.PromptTokens) u.completion.add(usage.CompletionTokens, known.CompletionTokens) + u.reasoning.add(usage.ReasoningTokens, known.ReasoningTokens) u.total.add(usage.TotalTokens, known.TotalTokens) u.cached.add(usage.CachedTokens, known.CachedTokens) u.cacheWrite.add(usage.CacheWriteTokens, known.CacheWriteTokens) @@ -302,7 +303,8 @@ func (u *usageTotals) add(usage domain.TokenUsage, presence *domain.TokenUsagePr func (u usageTotals) snapshot(attempts uint64) domain.ObservedTokenUsage { return domain.ObservedTokenUsage{ PromptTokens: u.prompt.snapshot(attempts), CompletionTokens: u.completion.snapshot(attempts), - TotalTokens: u.total.snapshot(attempts), CachedTokens: u.cached.snapshot(attempts), + ReasoningTokens: u.reasoning.snapshot(attempts), + TotalTokens: u.total.snapshot(attempts), CachedTokens: u.cached.snapshot(attempts), CacheWriteTokens: u.cacheWrite.snapshot(attempts), } } @@ -328,6 +330,7 @@ func observedUsage(usage domain.TokenUsage, presence *domain.TokenUsagePresence) return domain.ObservedTokenUsage{ PromptTokens: observedCount(usage.PromptTokens, known.PromptTokens), CompletionTokens: observedCount(usage.CompletionTokens, known.CompletionTokens), + ReasoningTokens: observedCount(usage.ReasoningTokens, known.ReasoningTokens), TotalTokens: observedCount(usage.TotalTokens, known.TotalTokens), CachedTokens: observedCount(usage.CachedTokens, known.CachedTokens), CacheWriteTokens: observedCount(usage.CacheWriteTokens, known.CacheWriteTokens), @@ -340,7 +343,8 @@ func usagePresence(usage domain.TokenUsage, presence *domain.TokenUsagePresence) } return domain.TokenUsagePresence{ PromptTokens: usage.PromptTokens != 0, CompletionTokens: usage.CompletionTokens != 0, - TotalTokens: usage.TotalTokens != 0, CachedTokens: usage.CachedTokens != 0, + ReasoningTokens: usage.ReasoningTokens != 0, + TotalTokens: usage.TotalTokens != 0, CachedTokens: usage.CachedTokens != 0, CacheWriteTokens: usage.CacheWriteTokens != 0, } } diff --git a/internal/observation/usage_accumulator_test.go b/internal/observation/usage_accumulator_test.go index f0a588a..1ec9e5f 100644 --- a/internal/observation/usage_accumulator_test.go +++ b/internal/observation/usage_accumulator_test.go @@ -25,3 +25,18 @@ func TestUsageAccumulatorPreservesPresenceAndNoAttemptSemantics(t *testing.T) { t.Fatalf("unreported completion usage = %+v", observed.CompletionTokens) } } + +func TestReasoningUsagePresence(t *testing.T) { + var usage UsageAccumulator + usage.Add(domain.TokenUsage{}, &domain.TokenUsagePresence{ReasoningTokens: true}) + usage.Add(domain.TokenUsage{ReasoningTokens: 4}, nil) + got := usage.Observed().ReasoningTokens + if got != (domain.ObservedTokenCount{Value: 4, Known: true, Complete: true}) { + t.Fatalf("usage=%+v", got) + } + usage.Add(domain.TokenUsage{}, nil) + got = usage.Observed().ReasoningTokens + if got != (domain.ObservedTokenCount{Value: 4, Known: true, Complete: false}) { + t.Fatalf("partial usage=%+v", got) + } +} diff --git a/internal/usecase/prepared_execution_test.go b/internal/usecase/prepared_execution_test.go index 88634ca..4345bef 100644 --- a/internal/usecase/prepared_execution_test.go +++ b/internal/usecase/prepared_execution_test.go @@ -317,7 +317,7 @@ func TestRunnerRunPreparedUsesFrozenValidationForInitialAndRepairOutputs(t *test responses: []*domain.GenerateResponse{{ Content: `{"repaired":true}`, Usage: domain.TokenUsage{ - PromptTokens: 2, CompletionTokens: 3, TotalTokens: 5, + PromptTokens: 2, CompletionTokens: 3, ReasoningTokens: 2, TotalTokens: 5, CachedTokens: 7, CacheWriteTokens: 11, }, }}, @@ -328,7 +328,7 @@ func TestRunnerRunPreparedUsesFrozenValidationForInitialAndRepairOutputs(t *test client := &sequenceLLM{responses: []*domain.GenerateResponse{{ Content: `{"broken":true}`, Usage: domain.TokenUsage{ - PromptTokens: 13, CompletionTokens: 17, TotalTokens: 19, + PromptTokens: 13, CompletionTokens: 17, ReasoningTokens: 4, TotalTokens: 19, CachedTokens: 23, CacheWriteTokens: 29, }, }}} @@ -370,7 +370,7 @@ func TestRunnerRunPreparedUsesFrozenValidationForInitialAndRepairOutputs(t *test t.Fatalf("unexpected repaired validation result: %+v", result.Validation) } wantUsage := domain.TokenUsage{ - PromptTokens: 15, CompletionTokens: 20, TotalTokens: 24, + PromptTokens: 15, CompletionTokens: 20, ReasoningTokens: 6, TotalTokens: 24, CachedTokens: 30, CacheWriteTokens: 40, } if result.Usage != wantUsage { diff --git a/internal/usecase/runner.go b/internal/usecase/runner.go index 1a675fc..5853aef 100644 --- a/internal/usecase/runner.go +++ b/internal/usecase/runner.go @@ -309,6 +309,7 @@ func addTokenUsage(total, next domain.TokenUsage) domain.TokenUsage { return domain.TokenUsage{ PromptTokens: total.PromptTokens + next.PromptTokens, CompletionTokens: total.CompletionTokens + next.CompletionTokens, + ReasoningTokens: total.ReasoningTokens + next.ReasoningTokens, TotalTokens: total.TotalTokens + next.TotalTokens, CachedTokens: total.CachedTokens + next.CachedTokens, CacheWriteTokens: total.CacheWriteTokens + next.CacheWriteTokens, diff --git a/observation_contract_test.go b/observation_contract_test.go index bc37d36..fa54f59 100644 --- a/observation_contract_test.go +++ b/observation_contract_test.go @@ -16,6 +16,19 @@ func observationEngine(t *testing.T, client promptkit.LLMClient) *promptkit.Engi return newSimpleEngine(t, "hello", promptkit.WithLLMClient(client)) } +func TestInjectedReasoningPresenceReachesObservation(t *testing.T) { + engine := observationEngine(t, &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok", UsagePresence: &promptkit.TokenUsagePresence{ReasoningTokens: true}}}) + var events []promptkit.ExecutionEvent + result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", Observer: func(event promptkit.ExecutionEvent) { events = append(events, event) }}) + if err != nil || result == nil || len(events) != 3 { + t.Fatalf("result=%+v events=%+v error=%v", result, events, err) + } + want := promptkit.ObservedTokenCount{Known: true, Complete: true} + if events[1].Usage.ReasoningTokens != want || events[2].Usage.ReasoningTokens != want { + t.Fatalf("events=%+v", events) + } +} + func TestContinuationObservationAncestry(t *testing.T) { client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "private answer"}} engine := observationEngine(t, client) diff --git a/observations.go b/observations.go index 4ead910..1347ddb 100644 --- a/observations.go +++ b/observations.go @@ -101,6 +101,16 @@ type ObservedTokenUsage struct { CacheWriteTokens ObservedTokenCount } +// CacheHitRatio returns the fraction of input tokens served from cache when +// both counts are known, complete, and consistent. Cache writes are independent. +func (u ObservedTokenUsage) CacheHitRatio() (float64, bool) { + input, reads := u.PromptTokens, u.CachedTokens + if !input.Known || !input.Complete || !reads.Known || !reads.Complete || input.Value <= 0 || reads.Value < 0 || reads.Value > input.Value { + return 0, false + } + return float64(reads.Value) / float64(input.Value), true +} + // ExecutionEvent contains metadata only, not provider text, request content, // credentials, URLs, headers, or error objects. It has no stable JSON contract. // Identifiers themselves may be sensitive. Nonterminal aggregate counters are