Collect costs for dispatched model attempts

This commit is contained in:
2026-11-22 18:27:01 +00:00
parent cd0e1f5d39
commit 81a2a7591d
9 changed files with 129 additions and 6 deletions

View File

@@ -18,7 +18,7 @@ Timeout attribution is finalized and the child scope released before terminal
delivery, so observer latency cannot change the already selected outcome.
Prepared-content notification is a separate opt-in callback, not an event.
The facade masks inherited recorder and prepared-claim state when entering
The facade masks inherited recorder, cost, and prepared-claim state when entering
preparation or execution. Delegated calls from an injected collaborator retain
the caller's cancellation, deadline, and unrelated context values, but do not
reuse the enclosing invocation's accounting. Ordinary runs compose the request
@@ -56,6 +56,11 @@ attempts above the first in a generation, and repair generations with at least
one dispatch. A terminal event is an aggregate and must not be added to its
attempt events.
The generation coordinator resolves cost once per dispatched attempt and adds
it to invocation-local accounting independently of event delivery. A recorder
can read that same accumulator at termination; groups merge first terminal
costs independently of terminal coverage.
Reasoning counts use the same per-field rules and are a subset of completion
tokens. The public cache-hit helper derives a ratio from complete, consistent
summed input and cache-read counts; it does not aggregate ratios or use writes.

View File

@@ -325,6 +325,8 @@ Validation: provider decoder and injected-adapter tests.
## Stage 11 — Per-Attempt Cost Collection And Observation
**Completed.**
Touch generation `recordAttempt` and built-in/injected execution paths,
observation attempt data/events, recorder terminal cost, and group aggregation.

View File

@@ -130,7 +130,7 @@ func (c *Coordinator) generate(
if result.Dispatched {
attempts++
}
recordAttempt(result, false, captureEvidence, &evidence, &usage, recorder, generation, ordinal)
recordAttempt(result, false, captureEvidence, &evidence, &usage, recorder, observation.CostFromContext(ctx), req.Target.ZeroCost, generation, ordinal)
if result.Outcome.Err == nil {
if err := ctx.Err(); err != nil {
if captureEvidence {
@@ -185,7 +185,7 @@ func (c *Coordinator) generateInjected(ctx context.Context, req domain.GenerateR
generation = 1
}
result := c.executeInjected(ctx, req)
recordAttempt(result, true, captureEvidence, &evidence, &usage, observation.FromContext(ctx), generation, 1)
recordAttempt(result, true, captureEvidence, &evidence, &usage, observation.FromContext(ctx), observation.CostFromContext(ctx), req.Target.ZeroCost, generation, 1)
if captureEvidence {
evidence.Usage = usage.Observed()
}
@@ -219,6 +219,9 @@ func (c *Coordinator) executeInjected(ctx context.Context, req domain.GenerateRe
requested := req.StructuredOutput != nil
result.Outcome.StructuredOutputRequested = &requested
result.Outcome.Response, result.Outcome.Err = c.injected.Generate(ctx, req)
if result.Outcome.Response != nil {
result.Outcome.CostUSD = result.Outcome.Response.CostUSD
}
result.RequestDuration = elapsed(start, c.now())
if result.Outcome.Err == nil && result.Outcome.Response == nil {
result.Outcome.Err = llm.ErrMalformedResponse
@@ -240,8 +243,15 @@ type attemptResult struct {
// recordAttempt runs after the attempt has released its permit, including when
// the recorder calls application code. Injected clients expose no HTTP facts.
func recordAttempt(result attemptResult, injected, capture bool, evidence *GenerationEvidence, usage *observation.UsageAccumulator, recorder *observation.Recorder, generation, ordinal uint64) {
func recordAttempt(result attemptResult, injected, capture bool, evidence *GenerationEvidence, usage *observation.UsageAccumulator, recorder *observation.Recorder, cost *observation.CostAccumulator, zeroCost bool, generation, ordinal uint64) {
outcome := result.Outcome
var resolved domain.ObservedCost
if result.Dispatched {
resolved = observation.ResolveCost(outcome.CostUSD, zeroCost)
if cost != nil {
cost.Add(resolved)
}
}
failure := ClassifyFailure(outcome.Err)
if capture {
attempt := AttemptEvidence{
@@ -277,7 +287,7 @@ func recordAttempt(result attemptResult, injected, capture bool, evidence *Gener
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: ordinal,
Dispatched: result.Dispatched, CapacityWait: result.CapacityWait, GenerationDuration: result.RequestDuration,
Outcome: attemptOutcome(outcome.Err), Failure: failure, StatusCode: outcome.StatusCode,
Usage: outcome.Usage, UsagePresence: outcome.UsagePresence})
Usage: outcome.Usage, UsagePresence: outcome.UsagePresence, CostUSD: resolved})
}
}

View File

@@ -58,6 +58,60 @@ func TestObservedRetryAndExhaustion(t *testing.T) {
}
}
func TestDispatchedAttemptCostsAreCollectedWithoutObserver(t *testing.T) {
calls := 0
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) {
calls++
if calls == 1 {
return response(503, `{"error":{}}`), nil
}
return response(200, `{"usage":{"cost":0.02},"choices":[{"message":{"content":"ok"}}]}`), nil
})
var cost observation.CostAccumulator
ctx := observation.WithCostAccumulator(context.Background(), &cost)
req := testRequest(1)
req.Target.Provider = "openrouter"
if _, err := coordinator.Generate(ctx, req); err != nil {
t.Fatal(err)
}
got := cost.Observed()
if got.Value != .02 || !got.Known || got.Complete || got.Source != domain.CostSourceProvider {
t.Fatalf("cost=%+v", got)
}
configured := testCoordinator(t, func(*http.Request) (*http.Response, error) { return nil, errors.New("transport failed") })
var zero observation.CostAccumulator
req.Target.ZeroCost = true
if _, err := configured.Generate(observation.WithCostAccumulator(context.Background(), &zero), req); err == nil {
t.Fatal("expected error")
}
if got := zero.Observed(); got != (domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceConfiguredZero}) {
t.Fatalf("fallback=%+v", got)
}
}
func TestInjectedFailedResponseCostIsObservedOnce(t *testing.T) {
configured := testCoordinator(t, func(*http.Request) (*http.Response, error) { t.Fatal("unexpected HTTP"); return nil, nil })
charge := .25
coordinator := NewInjected(configured.manager, fixedClient{response: &domain.GenerateResponse{CostUSD: &charge}, err: errors.New("failed")})
var cost observation.CostAccumulator
var events []domain.ExecutionEvent
recorder, err := observation.New(func(event domain.ExecutionEvent) { events = append(events, event) })
if err != nil {
t.Fatal(err)
}
recorder.SetCostAccumulator(&cost)
ctx := observation.WithRecorder(observation.WithCostAccumulator(context.Background(), &cost), recorder)
if _, err := coordinator.Generate(ctx, testRequest(0)); err == nil {
t.Fatal("expected error")
}
recorder.Finish(observation.FinishData{})
want := domain.ObservedCost{Value: .25, Known: true, Complete: true, Source: domain.CostSourceProvider}
if len(events) != 2 || events[0].CostUSD != want || events[1].CostUSD != want || cost.Observed() != want {
t.Fatalf("events=%+v cost=%+v", events, cost.Observed())
}
}
func TestObservedInterruptedWaitAndCanceledCapacity(t *testing.T) {
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) { return response(503, `{"error":{}}`), nil })
ctx, cancel := context.WithCancel(context.Background())

View File

@@ -1,11 +1,25 @@
package observation
import (
"context"
"math"
"go.maximumdirect.net/promptkit/internal/domain"
)
type costContextKey struct{}
// WithCostAccumulator carries one invocation's cost state.
func WithCostAccumulator(ctx context.Context, accumulator *CostAccumulator) context.Context {
return context.WithValue(ctx, costContextKey{}, accumulator)
}
// CostFromContext returns the current invocation's cost state, if installed.
func CostFromContext(ctx context.Context) *CostAccumulator {
value, _ := ctx.Value(costContextKey{}).(*CostAccumulator)
return value
}
// ResolveCost applies reported-cost precedence over a configured-zero fallback.
func ResolveCost(reported *float64, zeroCost bool) domain.ObservedCost {
if reported != nil && !math.IsNaN(*reported) && !math.IsInf(*reported, 0) {

View File

@@ -14,6 +14,7 @@ type Group struct {
terminals int
attempts, retries, repairs uint64
usage domain.ObservedTokenUsage
cost CostAccumulator
}
// Observe enrolls a run and incorporates its first terminal event only.
@@ -39,6 +40,7 @@ func (g *Group) Observe(event domain.ExecutionEvent) {
g.usage = domain.ObservedTokenUsage{PromptTokens: complete, CompletionTokens: complete, ReasoningTokens: complete, TotalTokens: complete, CachedTokens: complete, CacheWriteTokens: complete}
}
mergeGroupUsage(&g.usage, event.Usage)
g.cost.Add(event.CostUSD)
g.terminals++
}
@@ -47,6 +49,9 @@ func (g *Group) Snapshot() domain.ExecutionGroupSummary {
g.mu.Lock()
defer g.mu.Unlock()
result := domain.ExecutionGroupSummary{Members: make([]domain.ExecutionGroupMember, 0, len(g.members)), TerminalCoverageComplete: len(g.members) == g.terminals, Attempts: g.attempts, TransientRetries: g.retries, RepairCalls: g.repairs, Usage: g.usage}
if g.terminals > 0 || len(g.members) == 0 {
result.CostUSD = g.cost.Observed()
}
for id, terminal := range g.members {
result.Members = append(result.Members, domain.ExecutionGroupMember{RunID: id, TerminalObserved: terminal})
}

View File

@@ -41,6 +41,25 @@ func TestGroupTerminalAccounting(t *testing.T) {
}
}
func TestGroupCostCoverageIsIndependentOfTerminalCoverage(t *testing.T) {
var group Group
if got := group.Snapshot().CostUSD; got != (domain.ObservedCost{Known: true, Complete: true}) {
t.Fatalf("empty=%+v", got)
}
group.Observe(domain.ExecutionEvent{Kind: domain.ExecutionStarted, RunID: "pending"})
if got := group.Snapshot().CostUSD; got != (domain.ObservedCost{}) {
t.Fatalf("pending=%+v", got)
}
charge := domain.ObservedCost{Value: .25, Known: true, Complete: true, Source: domain.CostSourceProvider}
terminal := domain.ExecutionEvent{Kind: domain.ExecutionFinished, RunID: "done", CostUSD: charge}
group.Observe(terminal)
group.Observe(terminal)
got := group.Snapshot()
if got.TerminalCoverageComplete || got.CostUSD != charge {
t.Fatalf("snapshot=%+v", got)
}
}
func TestGroupConcurrentSnapshots(t *testing.T) {
var group Group
var wg sync.WaitGroup

View File

@@ -30,6 +30,7 @@ type Recorder struct {
generationDuration time.Duration
retryDelay time.Duration
usage UsageAccumulator
cost *CostAccumulator
}
// Selection contains resolved identities only. Unresolved fields stay empty.
@@ -53,6 +54,7 @@ type AttemptData struct {
StatusCode int
Usage domain.TokenUsage
UsagePresence *domain.TokenUsagePresence
CostUSD domain.ObservedCost
}
// DelayData describes one completed or interrupted retry wait.
@@ -99,6 +101,9 @@ func (r *Recorder) RunID() string { return r.runID }
// SetParentRunID records the immediate source identity before start delivery.
func (r *Recorder) SetParentRunID(id string) { r.parentRunID = id }
// SetCostAccumulator selects the invocation-owned cost state for terminal reporting.
func (r *Recorder) SetCostAccumulator(cost *CostAccumulator) { r.cost = cost }
// Start delivers the opening event with only identities already resolved.
func (r *Recorder) Start() {
event := r.event(domain.ExecutionStarted)
@@ -138,6 +143,7 @@ func (r *Recorder) Attempt(data AttemptData) {
event.StatusCode = data.StatusCode
if data.Dispatched {
event.Usage = observedUsage(data.Usage, data.UsagePresence)
event.CostUSD = data.CostUSD
}
r.deliver(event)
}
@@ -176,6 +182,9 @@ func (r *Recorder) Finish(data FinishData) {
event.Duration = 0
}
event.Usage = r.usage.Observed()
if r.cost != nil {
event.CostUSD = r.cost.Observed()
}
r.deliver(event)
}
@@ -222,7 +231,7 @@ type executionBoundary struct{ context.Context }
func (c executionBoundary) Value(key any) any {
switch key.(type) {
case contextKey, claimSinkKey:
case contextKey, claimSinkKey, costContextKey:
return nil
default:
return c.Context.Value(key)

View File

@@ -19,10 +19,15 @@ func TestExecutionBoundaryPreservesCallerContext(t *testing.T) {
recorder := newRecorder("outer", time.Now, nil)
sink := recorder
parent = WithClaimSink(WithRecorder(parent, recorder), &sink)
var cost CostAccumulator
parent = WithCostAccumulator(parent, &cost)
child := WithoutExecutionState(parent)
if FromContext(child) != nil || FromContext(parent) != recorder {
t.Fatal("execution boundary did not isolate recorder state")
}
if CostFromContext(child) != nil || CostFromContext(parent) != &cost {
t.Fatal("execution boundary did not isolate cost state")
}
RegisterClaim(child, nil)
if sink != recorder {
t.Fatal("execution boundary retained the parent claim sink")