diff --git a/cache_hit_contract_test.go b/cache_hit_contract_test.go index 7662dae..216ff14 100644 --- a/cache_hit_contract_test.go +++ b/cache_hit_contract_test.go @@ -1,12 +1,29 @@ package promptkit_test import ( + "context" "math" "testing" "go.maximumdirect.net/promptkit" ) +func TestDuplicateContextAndRequestGroupEnrollment(t *testing.T) { + charge := .5 + engine := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + return &promptkit.GenerateResponse{Content: "ok", CostUSD: &charge}, nil + }))) + var group promptkit.ExecutionGroup + ctx := promptkit.WithExecutionObserver(context.Background(), group.Observe) + if _, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", Observer: group.Observe}); err != nil { + t.Fatal(err) + } + got := group.Snapshot() + if len(got.Members) != 1 || got.Attempts != 1 || got.CostUSD.Value != .5 { + t.Fatalf("summary=%+v", got) + } +} + func TestCacheHitRatioUsesCompleteConsistentCounts(t *testing.T) { known := func(v int) promptkit.ObservedTokenCount { return promptkit.ObservedTokenCount{Value: v, Known: true, Complete: true} diff --git a/cost.go b/cost.go index 2449c85..0685784 100644 --- a/cost.go +++ b/cost.go @@ -6,19 +6,31 @@ import "go.maximumdirect.net/promptkit/internal/domain" type CostSource string const ( - CostSourceNone CostSource = "" - CostSourceProvider CostSource = "provider" + // CostSourceNone denotes no known sourced contribution. + CostSourceNone CostSource = "" + // CostSourceProvider denotes a reported provider or injected-client amount. + CostSourceProvider CostSource = "provider" + // CostSourceConfiguredZero denotes an explicitly configured zero fallback. CostSourceConfiguredZero CostSource = "configured_zero" - CostSourceMixed CostSource = "mixed" + // CostSourceMixed denotes both provider and configured-zero contributions. + CostSourceMixed CostSource = "mixed" ) // ObservedCost reports an available USD subtotal and its coverage and source. -// Complete describes coverage of dispatched attempts, not billing verification. +// Known means at least one contribution is available. Complete means all +// included dispatched attempts have an amount under the selected policy; it +// does not verify billing. A completed run with no dispatched attempts, or an +// empty group, has known complete zero with no source. A group with members but +// no terminals has unknown cost. Amounts are unrounded binary floating point. type ObservedCost struct { - Value float64 `json:"value"` - Known bool `json:"known"` - Complete bool `json:"complete"` - Source CostSource `json:"source"` + // Value is the known USD subtotal. + Value float64 `json:"value"` + // Known reports whether a subtotal is available. + Known bool `json:"known"` + // Complete reports whether all included dispatched attempts are covered. + Complete bool `json:"complete"` + // Source summarizes the known contributions. + Source CostSource `json:"source"` } func fromDomainCost(value domain.ObservedCost) ObservedCost { diff --git a/cost_run_contract_test.go b/cost_run_contract_test.go index e348da0..d08b5ea 100644 --- a/cost_run_contract_test.go +++ b/cost_run_contract_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "io" + "math" "net/http" "strings" "testing" @@ -62,6 +63,7 @@ func TestOrdinaryRunCostMatchesTerminalWithOrWithoutObserver(t *testing.T) { func TestOrdinaryRunCostFailureAndFallback(t *testing.T) { charge := .125 + invalid := math.Inf(1) for _, tc := range []struct { name string response *promptkit.GenerateResponse @@ -70,6 +72,7 @@ func TestOrdinaryRunCostFailureAndFallback(t *testing.T) { }{ {"reported takes precedence", &promptkit.GenerateResponse{Content: "ok", CostUSD: &charge}, nil, promptkit.ObservedCost{Value: .125, Known: true, Complete: true, Source: promptkit.CostSourceProvider}}, {"failed dispatch fallback", nil, errors.New("failure"), promptkit.ObservedCost{Known: true, Complete: true, Source: promptkit.CostSourceConfiguredZero}}, + {"nonfinite report falls back", &promptkit.GenerateResponse{Content: "ok", CostUSD: &invalid}, nil, promptkit.ObservedCost{Known: true, Complete: true, Source: promptkit.CostSourceConfiguredZero}}, } { t.Run(tc.name, func(t *testing.T) { engine := costEngine(t, true, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { diff --git a/docs/consumers/pkg-promptkit.md b/docs/consumers/pkg-promptkit.md index 591fee3..a3ce342 100644 --- a/docs/consumers/pkg-promptkit.md +++ b/docs/consumers/pkg-promptkit.md @@ -728,6 +728,40 @@ counts, but its zero counts remain unknown to observations; supply explicit presence when measured zeros matter. Existing `RunResult.Usage` sums returned values independently of observation presence. +For an application-owned invocation spanning several calls or engines, attach +an observer once to a context and propagate it to those calls: + +```go +var group promptkit.ExecutionGroup +ctx = promptkit.WithExecutionObserver(ctx, group.Observe) +result, err := engine.Run(ctx, request) +// After joining all application-owned work, inspect group.Snapshot(), even on error. +``` + +Explicit request observers run first, followed by contextual observers in +registration order. Coordinated continuations pick up the context through +`Run`; prepared handles retain the request observer but use the execution +context's observers, not the preparation context's. Each nested call has its +own run ID and accounting. Duplicate registration can deliver duplicate events; +groups count each run's first terminal once. + +Reasoning tokens are a subset of completion tokens; cache reads and writes are +input details, not extra tokens. `ObservedTokenUsage.CacheHitRatio()` derives +the cache-read fraction only when input and read totals are known, complete, +positive-denominator, and consistent. It divides aggregate counts rather than +averaging per-run ratios and is independent of cache writes. + +`RunResult.CostUSD` is available even without an observer; attempt and terminal +events and group snapshots also report cost. `Known` denotes an available +subtotal, `Complete` covers dispatched attempts, and `Source` distinguishes +provider reports, an explicitly configured zero fallback, and mixed sources. +Reported cost, including zero, takes precedence over fallback. Unknown failed +attempts make a subtotal incomplete; configured zero may cover them. Terminal +events can report cost when a failed operation returns no result. Neither +complete reporting nor provider attribution verifies an invoice; consumers +choose presentation precision. See [backend definitions](../formats.md#backend-definitions) +and the [integration contract](../integrations/openai-compatible-chat.md#response-handling). + Callbacks run synchronously, in order for one operation, and can overlap for different operations. Synchronize shared callback state and avoid blocking on work that needs the same run's admission slot. Callback panics propagate; @@ -879,15 +913,15 @@ outcome, err := engine.Coordinate(ctx, promptkit.CoordinateRequest{ }) ``` -`assess` is application code and explicitly selects its observer on any nested -run. A completed turn releases its admission lease before the callback, so that +`assess` is application code and can select its observer explicitly or inherit +one from the context. A completed turn releases its admission lease before the callback, so that assessment may use the same backend. `stopped` and `exhausted` describe control flow, not content acceptance. Cancellation before callback delivery can return one completed response in `Undelivered`; errors retain their ordinary identities. ### Collect Selected Execution Facts -Groups are independent of continuation and enrollment is explicit: +Groups are independent of continuation and can be enrolled explicitly: ```go var group, validatorGroup promptkit.ExecutionGroup @@ -904,7 +938,7 @@ _, _ = summary, validatorSummary ``` Only the first terminal event per run contributes counts and usage; coverage of -currently observed members is separate from token-usage completeness. Empty +currently observed members is separate from token and cost completeness. Empty groups report complete, known zero totals, while members without terminals do not invent totals. Overlapping groups are not generally additive. See the [offline example](../../examples/go-library/observe/main.go) for complete setup. diff --git a/docs/formats.md b/docs/formats.md index 49ef9d3..a57490e 100644 --- a/docs/formats.md +++ b/docs/formats.md @@ -223,7 +223,7 @@ Backend definitions are supplied through caller-selected configuration sources. The conventional backend IDs are names only; PromptKit does not install definitions for them. -A backend definition may have `endpoint`, `api_key_env`, +A backend definition may have `provider`, `zero_cost`, `endpoint`, `api_key_env`, `concurrency_limit`, `queue_capacity`, `extra_params`, and `smoke_tests`. The final endpoint is required and must be an absolute HTTP(S) URL with a host, no user information, query, or fragment. An omitted or null API-key environment @@ -237,6 +237,17 @@ unrecognized backend field is application data and is not automatically redacted by Promptkit. Applications own sensitivity declarations and disclosure policy for their extension fields. +`provider` selects response and diagnostic conventions independently of the +backend ID, endpoint, or model. Its final value is trimmed and lowercased; +omitted, null, or blank selects `openai-compatible` at runtime. Supported +nonblank values are `openai-compatible`, `openrouter`, and `vllm`. `zero_cost` +is an optional Boolean: true supplies a zero-cost fallback when a dispatched +attempt has no usable reported cost; omitted, null, and false disable it. +Neither setting is sent to the provider. They follow normal layering and +inheritance, and endpoint overrides retain the selected backend's policy. +Resource inspection shows declared normalized values without inventing runtime +defaults; configuration inspection retains source history. + `smoke_tests` is optional backend metadata for explicit diagnostic operations. It may contain `default_profile`, `reachability`, and `authentication`: @@ -274,6 +285,10 @@ Without a selected adapter, authentication diagnostics report an unsupported selection. An endpoint-only authentication block overrides the destination of a provider default; `kind: none` suppresses that default. +Previously, a supplied authentication object required `kind`. Endpoint-only +objects now use a provider default where available; clearing the object +restores that default rather than suppressing it. Use `kind: none` to suppress +authentication testing. These values are configuration only: engine construction does not contact a backend. @@ -364,7 +379,7 @@ does not disable the caller context or HTTP transport timeout. ### Bundled OpenRouter Configuration -The default bundle installs root backend `openrouter`, endpoint +The default bundle installs root backend `openrouter`, provider `openrouter`, endpoint `https://openrouter.ai/api/v1`, optional credential environment name `OPENROUTER_API_KEY`, and smoke metadata selecting `gpt-luna-latest` with the `openrouter_key` authentication adapter. Existing diagnostic endpoint fallback diff --git a/docs/integrations/openai-compatible-chat.md b/docs/integrations/openai-compatible-chat.md index ddbad95..7c9efb7 100644 --- a/docs/integrations/openai-compatible-chat.md +++ b/docs/integrations/openai-compatible-chat.md @@ -173,6 +173,10 @@ otherwise usable content. Existing common count decoding remains strict. OpenRouter's `usage.cost` is an optional USD amount; malformed or non-finite amounts are ignored without invalidating content. The generic and vLLM providers do not interpret that field. +vLLM prompt-token details depend on server configuration, including +`--enable-prompt-tokens-details`; reasoning details depend on a configured +reasoning parser. Servers without these details remain usable. vLLM cache +writes describe local prefix-cache creation, not a billable write charge. Each common token field retains whether the provider reported a non-null integer, including zero. Omitted and `null` fields are unknown for diff --git a/docs/internal/capacity.md b/docs/internal/capacity.md index b827f84..f883f90 100644 --- a/docs/internal/capacity.md +++ b/docs/internal/capacity.md @@ -79,7 +79,7 @@ other backend IDs. The coordinator does not retry injected clients; it normalizes a nil successful response as malformed generation. The built-in HTTP client remains responsible for provider transport behavior. The runner, rather than the -capacity layer, sums all five usage fields from the initial response and every +capacity layer, sums usage fields from the initial response and every completed repair response into the successful run result. An invocation-local observer receives attempt events after active permit diff --git a/docs/internal/generation.md b/docs/internal/generation.md index 4437c15..900088a 100644 --- a/docs/internal/generation.md +++ b/docs/internal/generation.md @@ -35,6 +35,12 @@ unknown usage. Transport, timeout, malformed, and noneligible status failures stop the sequence without retry exhaustion. Context is checked before each wait, acquisition, and dispatch. +At every dispatched completion the coordinator resolves any reported cost +against the frozen backend zero-cost policy and adds it once to the run's +[cost accumulator](observation.md), independently of observer selection. +Injected responses may retain reported cost alongside an error. Undispatched +acquisitions and retry waits contribute no cost. + The public client adapter translates explicitly supplied public transport and malformed-response identities into internal categories while retaining the original error chain. This keeps injected attempt classifications consistent diff --git a/docs/internal/llm.md b/docs/internal/llm.md index 0d33bf7..96d26c0 100644 --- a/docs/internal/llm.md +++ b/docs/internal/llm.md @@ -142,6 +142,14 @@ known. Strictly decoded usage can still be observed if later choice/content checks fail; invalid framing or a non-success error body contributes none. Existing successful-result accounting remains value-based. +The captured provider selects additional response details and authentication +diagnostic defaults. Optional new reasoning, cache-write, and OpenRouter cost +fields are decoded independently, without relaxing common-field or framing +validation. Cost remains an optional raw amount at this boundary; shared +accounting applies configured-zero fallback after dispatch. The integration +contract specifies the [wire mappings](../integrations/openai-compatible-chat.md#response-handling) +and [probe recognition](../integrations/openai-compatible-chat.md#explicit-smoke-diagnostics). + For a non-success response, `ProviderHTTPError` retains the HTTP status and only normalized detail from the bounded recognized envelope. It retains `ErrUnexpectedStatus` through unwrapping. The client owns response closure; diff --git a/docs/internal/overview.md b/docs/internal/overview.md index 0c3d69b..4d18209 100644 --- a/docs/internal/overview.md +++ b/docs/internal/overview.md @@ -21,7 +21,7 @@ contributor workflow and validation. | `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) | | `internal/domain` | Defines internal framework values for requests, immutable conversation exchanges, artifacts, prompt definitions, requirement inspection, profiles, execution targets, rendering, generation, validation, and source-neutral smoke diagnostics, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, output contracts, and resource-identity syntax and qualification. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [exchange storage](../../internal/domain/exchange.go), [requirement values](../../internal/domain/requirements.go), [endpoint invariant](../../internal/domain/endpoint.go) | | `internal/engine` | Assembles an already captured configuration into shared capacity, rendering, validation, generation, repair, runner, and smoke-test collaborators. Immutable namespace views share those collaborators and qualify selections without rewriting captured references. Built-in engines provide the diagnostic HTTP collaborator; injected-client engines do not. It delegates resource inspection and listing to the capture. | [Engine assembly](../../internal/engine/engine.go), [internal sources](sources.md#typed-configuration-loading), [internal runner](runner.md) | -| `internal/generation` | Coordinates frozen built-in HTTP attempts and one-shot injected calls through active backend permits, including bounded status retries, cancellable backoff, ordinary attempt observations, and invocation-local evidence for diagnostics. | [Internal generation coordination](generation.md) | +| `internal/generation` | Coordinates frozen built-in HTTP attempts and one-shot injected calls through active backend permits, including bounded retries, per-attempt cost, ordinary observations, and separate diagnostic evidence. | [Internal generation coordination](generation.md) | | `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) | | `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) | | `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible provider values, and detaches previously accepted values without repeating validation. | [JSON values](../../internal/jsonvalue/jsonvalue.go) | @@ -30,8 +30,8 @@ contributor workflow and validation. | `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data, and conservatively enumerates dependencies from captured template syntax. | [Go-template renderer](../../internal/prompt/go_renderer.go), [requirement discovery](../../internal/prompt/requirements.go) | | `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) | | `internal/validate` | Creates operation-local validation plans from inline schema documents or bound filesystem resources and validates basic, JSON, and JSON Schema output. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) | -| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including bounded response decoding, authentication, deadline handling, per-invocation HTTP evidence and built-in diagnostic probes, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) | -| `internal/observation` | Owns invocation-local run IDs, presence-aware usage and phase accounting, synchronous metadata event delivery and composition, and explicit terminal-based group aggregation; smoke diagnostics keep their private evidence outside ordinary observer events. | [Internal execution observation](observation.md) | +| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible requests, provider-specific response interpretation and authentication probes, bounded decoding, deadlines, and diagnostic evidence. | [Internal model client](llm.md) | +| `internal/observation` | Owns invocation-local run IDs, presence-aware token and cost accounting, contextual observer selection, synchronous event delivery, and terminal-based group aggregation; smoke diagnostics retain separate evidence. | [Internal execution observation](observation.md) | | `internal/usecase` | Selects captured prompts and profiles, enumerates requirements from captured prompt syntax, applies request overrides and completion checks, coordinates ordinary and single-claim prepared execution across rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair, advances a single conversation branch, and runs explicit smoke diagnostics with private cases, shared generation/capacity, and optional built-in HTTP probes. | [Internal runner](runner.md), [requirement inspection](../../internal/usecase/requirements_inspection.go), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) | The internal engine assembles these internal components without exposing their diff --git a/docs/internal/runner.md b/docs/internal/runner.md index 61bd8ad..1c1a262 100644 --- a/docs/internal/runner.md +++ b/docs/internal/runner.md @@ -216,10 +216,16 @@ ordinary `Run` reuses its run ID and carries it to the generation coordinator through context. Prompt and profile identities are enriched only after their respective resolutions succeed, before admission. The facade starts observation before request conversion and emits its terminal event after public error -mapping and after the runner's deferred admission release. Preparation and -prepared execution follows the same terminal mapping after a successful claim; +mapping and after the runner's deferred admission release. Preparation emits no +events; prepared execution follows the same terminal mapping after a successful claim; invalid or discarded handles do not acquire an observer. +Ordinary runs establish fresh private cost state before observation; a claimed +prepared execution establishes its own state after claim. Contextual observers +remain available at public boundaries, while enclosing recorder and cost state +are masked. Prepared handles retain the request observer and frozen provider +policy, not preparation-context selection or an accumulator. + The facade establishes an optional operation scope for ordinary runs before observation and preparation. A claimed prepared execution establishes its own scope from the retained duration; preparation and idle time do not consume it. @@ -273,11 +279,13 @@ run identifier, and UTC timing. The same effective session reaches initial generation and any repair attempt through the rendered prompt. The same effective target and presence metadata, including backend identity and direct credential during execution, reaches generation and every repair attempt. -Result usage is the field-wise sum of all five usage values from the initial +Result usage is the field-wise sum of usage values from the initial response and every completed repair response. Final raw output, artifact, and validation state still come from the last candidate. A repair error returns no partial run result or partial result usage, although the invocation-local observer retains completed-attempt usage and reports it in the terminal event. +Result cost instead snapshots the shared accumulator across all dispatched +attempts, including retries and repairs, whether or not observation is enabled. ## Failure Categories diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 67dcd60..52f3f5e 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -417,6 +417,8 @@ implementation has reached its final buildable state. ## Stage 15 — Canonical Documentation And Final Verification +**Completed.** + - Finalize GoDoc for helpers, backend fields, reasoning counts, cost values and sources, client ownership, and observer/prepared lifetimes. Update stale authentication Kind requirements in shared settings and public docs. diff --git a/docs/roadmap/invocation-scoped-token-usage.md b/docs/roadmap/invocation-scoped-token-usage.md index 08a7086..1582153 100644 --- a/docs/roadmap/invocation-scoped-token-usage.md +++ b/docs/roadmap/invocation-scoped-token-usage.md @@ -1,6 +1,7 @@ # Invocation-Scoped Execution Observation -Status: accepted scope; implementation pending. +Status: accepted scope; implementation complete. Current behavior is documented +in the linked canonical contracts; roadmap retirement is separate housekeeping. ## Purpose And Scope diff --git a/engine.go b/engine.go index 872283e..abbc825 100644 --- a/engine.go +++ b/engine.go @@ -535,7 +535,8 @@ func (e *Engine) prepareRequest(ctx context.Context, req RunRequest) (*domain.Pr // returns no handle on error. A nil Engine returns an error matching // ErrInvalidConfig. The effective transient retry budget is frozen privately // for RunPrepared and is absent from Details. A supplied Observer is retained -// privately for RunPrepared; preparation itself emits no events. +// privately for RunPrepared; preparation itself emits no events and does not +// retain contextual observers. // RunRequest.OperationTimeout and RunRequest.OnPrepared are retained without // starting a timer or invoking the notification during preparation. func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*PreparedExecution, error) { @@ -692,8 +693,9 @@ func (e *Engine) run(ctx context.Context, req RunRequest) (*RunResult, error) { // error. RunPrepared uses the retry choice frozen during PrepareExecution. An // operational error returns no partial RunResult. The retained Observer and // execution-context observers begin only after a successful atomic claim, with -// identities from the frozen snapshot, and receive a terminal event on normal return after public error -// mapping. Invalid, foreign, discarded, or reused handles emit no events. +// identities from the frozen snapshot, and receive a terminal event on normal +// return after public error mapping. Invalid, foreign, discarded, or reused +// handles emit no events. // Callbacks run outside the handle and capacity locks; panics propagate and // do not promise a terminal event. // The retained RunRequest.OperationTimeout starts after claim; the retained diff --git a/examples/go-library/observe/main.go b/examples/go-library/observe/main.go index 4963d31..e05a4e5 100644 --- a/examples/go-library/observe/main.go +++ b/examples/go-library/observe/main.go @@ -49,7 +49,11 @@ func (t *scriptedTransport) RoundTrip(req *http.Request) (*http.Response, error) func modelResponse(content string, promptTokens, completionTokens int) *http.Response { body, _ := json.Marshal(map[string]any{ "choices": []any{map[string]any{"message": map[string]string{"content": content}}}, - "usage": map[string]int{"prompt_tokens": promptTokens, "completion_tokens": completionTokens}, + "usage": map[string]any{ + "prompt_tokens": promptTokens, "completion_tokens": completionTokens, + "prompt_tokens_details": map[string]int{"cached_tokens": 2, "cache_write_tokens": 1}, + "completion_tokens_details": map[string]int{"reasoning_tokens": 1}, "cost": .01, + }, }) return httpResponse(http.StatusOK, string(body)) } @@ -80,6 +84,10 @@ type summary struct { ExhaustedOutcome promptkit.CoordinationOutcome `json:"exhausted_outcome"` GroupMembers int `json:"group_members"` GroupAttempts uint64 `json:"group_attempts"` + GroupCost promptkit.ObservedCost `json:"group_cost"` + GroupReasoning promptkit.ObservedTokenCount `json:"group_reasoning"` + GroupCacheRatioAvailable bool `json:"group_cache_ratio_available"` + AssessmentCacheRatio float64 `json:"assessment_cache_ratio"` PreparedMessages int `json:"prepared_messages"` PreparedParentMatches bool `json:"prepared_parent_matches"` } @@ -97,6 +105,8 @@ func run() (summary, error) { filesystem := configkit.NewFilesystem("example schemas", assets) backend := promptkit.LocalBackend("https://example.invalid/v1", 1) backend.ID = backendID + provider, zeroCost := "openrouter", true + backend.Provider, backend.ZeroCost = &provider, &zeroCost inputs = append(inputs, configkit.Input{Source: promptkit.SchemaRootSource("example schemas", "schemas"), Destination: configkit.Keys("promptkit"), Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "."}}, promptkit.SettingsInput(promptkit.BackendSource("example backend", []promptkit.BackendConfig{ @@ -117,8 +127,9 @@ func run() (summary, error) { var proposalLog eventLog var executions promptkit.ExecutionGroup - proposal, err := engine.Run(context.Background(), promptkit.RunRequest{ - PromptID: "example.proposal", Inputs: map[string]promptkit.ArtifactRef{"idea": promptkit.Inline("Reduce queue cleanup toil.")}, CaptureExchange: true, Observer: promptkit.ComposeObservers(proposalLog.record, executions.Observe), + ctx := promptkit.WithExecutionObserver(context.Background(), executions.Observe) + proposal, err := engine.Run(ctx, promptkit.RunRequest{ + PromptID: "example.proposal", Inputs: map[string]promptkit.ArtifactRef{"idea": promptkit.Inline("Reduce queue cleanup toil.")}, CaptureExchange: true, Observer: proposalLog.record, }) if err != nil { return summary{}, err @@ -133,8 +144,8 @@ func run() (summary, error) { var assessmentLog eventLog var preparedDetails promptkit.PreparedRun decision := "rejected" - coordination, err := engine.Coordinate(context.Background(), promptkit.CoordinateRequest{ - Start: proposal.Exchange, AdditionalTurns: 1, Observer: executions.Observe, + coordination, err := engine.Coordinate(ctx, promptkit.CoordinateRequest{ + Start: proposal.Exchange, AdditionalTurns: 1, Next: func(ctx context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) { if turn.Result == nil { return &promptkit.RunRequest{PromptID: "example.proposal", Inputs: map[string]promptkit.ArtifactRef{"idea": promptkit.Inline("Refine the proposal.")}, OperationTimeout: 10 * time.Second, OnPrepared: func(_ context.Context, details promptkit.PreparedRun) { preparedDetails = details }}, nil @@ -142,7 +153,7 @@ func run() (summary, error) { if !turn.Result.Validation.IsValid { return nil, fmt.Errorf("follow-up failed structural validation") } - assessment, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "example.assessment", Inputs: map[string]promptkit.ArtifactRef{"proposal": promptkit.Inline(turn.Result.RawOutput)}, OperationTimeout: 10 * time.Second, Observer: promptkit.ComposeObservers(assessmentLog.record, executions.Observe)}) + assessment, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "example.assessment", Inputs: map[string]promptkit.ArtifactRef{"proposal": promptkit.Inline(turn.Result.RawOutput)}, OperationTimeout: 10 * time.Second, Observer: assessmentLog.record}) if err != nil { return nil, err } @@ -181,6 +192,8 @@ func run() (summary, error) { return summary{}, err } group := executions.Snapshot() + _, ratioAvailable := group.Usage.CacheHitRatio() + assessmentRatio, _ := assessmentTerminal.Usage.CacheHitRatio() return summary{ ProposalAttempts: proposalTerminal.Attempts, ProposalTransientRetries: proposalTerminal.TransientRetries, @@ -192,6 +205,10 @@ func run() (summary, error) { ExhaustedOutcome: exhausted.Outcome, GroupMembers: len(group.Members), GroupAttempts: group.Attempts, + GroupCost: group.CostUSD, + GroupReasoning: group.Usage.ReasoningTokens, + GroupCacheRatioAvailable: ratioAvailable, + AssessmentCacheRatio: assessmentRatio, PreparedMessages: len(preparedDetails.Messages), PreparedParentMatches: preparedDetails.ParentRunID == proposal.RunID, }, nil diff --git a/examples/go-library/observe/main_test.go b/examples/go-library/observe/main_test.go index e986bf2..3fa6ba3 100644 --- a/examples/go-library/observe/main_test.go +++ b/examples/go-library/observe/main_test.go @@ -3,7 +3,10 @@ package main import ( "bytes" "encoding/json" + "math" "testing" + + "go.maximumdirect.net/promptkit" ) func TestObserveExample(t *testing.T) { @@ -12,18 +15,22 @@ func TestObserveExample(t *testing.T) { t.Fatal(err) } var got struct { - ProposalAttempts *int `json:"proposal_attempts"` - ProposalRetries *int `json:"proposal_transient_retries"` - UsageComplete *bool `json:"proposal_usage_complete"` - AssessmentAttempts *int `json:"assessment_attempts"` - AssessmentRetries *int `json:"assessment_transient_retries"` - Outcome string `json:"assessment_outcome"` - CoordinationOutcome string `json:"coordination_outcome"` - ExhaustedOutcome string `json:"exhausted_outcome"` - GroupMembers int `json:"group_members"` - GroupAttempts int `json:"group_attempts"` - PreparedMessages int `json:"prepared_messages"` - PreparedParentMatches bool `json:"prepared_parent_matches"` + ProposalAttempts *int `json:"proposal_attempts"` + ProposalRetries *int `json:"proposal_transient_retries"` + UsageComplete *bool `json:"proposal_usage_complete"` + AssessmentAttempts *int `json:"assessment_attempts"` + AssessmentRetries *int `json:"assessment_transient_retries"` + Outcome string `json:"assessment_outcome"` + CoordinationOutcome string `json:"coordination_outcome"` + ExhaustedOutcome string `json:"exhausted_outcome"` + GroupMembers int `json:"group_members"` + GroupAttempts int `json:"group_attempts"` + GroupCost promptkit.ObservedCost `json:"group_cost"` + GroupReasoning promptkit.ObservedTokenCount `json:"group_reasoning"` + GroupCacheRatioAvailable bool `json:"group_cache_ratio_available"` + AssessmentCacheRatio float64 `json:"assessment_cache_ratio"` + PreparedMessages int `json:"prepared_messages"` + PreparedParentMatches bool `json:"prepared_parent_matches"` } if err := json.Unmarshal(output.Bytes(), &got); err != nil { t.Fatal(err) @@ -33,4 +40,7 @@ func TestObserveExample(t *testing.T) { got.AssessmentRetries == nil || *got.AssessmentRetries != 0 || got.Outcome != "accepted" || got.CoordinationOutcome != "stopped" || got.ExhaustedOutcome != "exhausted" || got.GroupMembers != 3 || got.GroupAttempts != 4 || got.PreparedMessages < 3 || !got.PreparedParentMatches { t.Fatalf("unexpected observation summary: %+v", got) } + if !got.GroupCost.Known || !got.GroupCost.Complete || got.GroupCost.Source != promptkit.CostSourceMixed || math.Abs(got.GroupCost.Value-.03) > 1e-12 || got.GroupReasoning.Value != 3 || got.GroupReasoning.Complete || got.GroupCacheRatioAvailable || math.Abs(got.AssessmentCacheRatio-2.0/7.0) > 1e-12 { + t.Fatalf("unexpected usage summary: %+v", got) + } } diff --git a/execution_group.go b/execution_group.go index ad7b0b2..3282439 100644 --- a/execution_group.go +++ b/execution_group.go @@ -5,7 +5,7 @@ import ( "go.maximumdirect.net/promptkit/internal/observation" ) -// ExecutionGroup collects facts from explicitly selected execution events. +// ExecutionGroup collects facts from selected execution events. // Its zero value is ready for concurrent use. Do not copy it after first use. type ExecutionGroup struct{ group observation.Group } @@ -21,7 +21,8 @@ type ExecutionGroupMember struct { // member has a terminal, independently of usage completeness. An empty group // has complete coverage and known complete zero usage; a group with members // but no terminals has unknown, incomplete usage. Overlapping groups are not -// generally additive. +// generally additive. CostUSD covers included terminals independently of +// terminal coverage, using the same known, complete, and source merge rules. type ExecutionGroupSummary struct { Members []ExecutionGroupMember `json:"members"` TerminalCoverageComplete bool `json:"terminal_coverage_complete"` diff --git a/internal/generation/observation_test.go b/internal/generation/observation_test.go index a9e5b56..08c2e45 100644 --- a/internal/generation/observation_test.go +++ b/internal/generation/observation_test.go @@ -112,6 +112,35 @@ func TestInjectedFailedResponseCostIsObservedOnce(t *testing.T) { } } +func TestCanceledDispatchCostFallbackAndUndispatchedEmptyCost(t *testing.T) { + configured := testCoordinator(t, func(*http.Request) (*http.Response, error) { t.Fatal("unexpected HTTP"); return nil, nil }) + ctx, cancel := context.WithCancel(context.Background()) + coordinator := NewInjected(configured.manager, cancelingCostClient{cancel: cancel}) + var cost observation.CostAccumulator + req := testRequest(0) + req.Target.ZeroCost = true + if _, err := coordinator.Generate(observation.WithCostAccumulator(ctx, &cost), req); !errors.Is(err, context.Canceled) { + t.Fatalf("dispatch error=%v", err) + } + if got := cost.Observed(); got != (domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceConfiguredZero}) { + t.Fatalf("dispatched=%+v", got) + } + var untouched observation.CostAccumulator + if _, err := coordinator.Generate(observation.WithCostAccumulator(ctx, &untouched), req); !errors.Is(err, context.Canceled) { + t.Fatalf("pre-dispatch error=%v", err) + } + if got := untouched.Observed(); got != (domain.ObservedCost{Known: true, Complete: true}) { + t.Fatalf("undispatched=%+v", got) + } +} + +type cancelingCostClient struct{ cancel context.CancelFunc } + +func (c cancelingCostClient) Generate(ctx context.Context, _ domain.GenerateRequest) (*domain.GenerateResponse, error) { + c.cancel() + return nil, ctx.Err() +} + func TestObservedInterruptedWaitAndCanceledCapacity(t *testing.T) { coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) { return response(503, `{"error":{}}`), nil }) ctx, cancel := context.WithCancel(context.Background()) diff --git a/observations.go b/observations.go index e9d101a..f16abf0 100644 --- a/observations.go +++ b/observations.go @@ -18,8 +18,10 @@ import ( type ExecutionObserver func(ExecutionEvent) // WithExecutionObserver adds an observer to executions using the returned -// context. Inherited observers run before the newly added observer. A nil -// observer leaves the context unchanged. +// context. Request observers run first, then inherited contextual observers +// in registration order. A nil observer leaves the context unchanged. +// Prepared executions use their execution context, not their preparation +// context. Callers own context propagation and synchronization of observers. func WithExecutionObserver(ctx context.Context, observer ExecutionObserver) context.Context { if observer == nil { return ctx @@ -102,7 +104,9 @@ type ObservedTokenUsage struct { } // CacheHitRatio returns the fraction of input tokens served from cache when -// both counts are known, complete, and consistent. Cache writes are independent. +// both counts are known, complete, and consistent, with positive input and +// reads between zero and input inclusive. Otherwise it returns (0, false). +// Cache writes are independent; aggregate ratios use summed counts. func (u ObservedTokenUsage) CacheHitRatio() (float64, bool) { input, reads := u.PromptTokens, u.CachedTokens if !input.Known || !input.Complete || !reads.Known || !reads.Complete || input.Value <= 0 || reads.Value < 0 || reads.Value > input.Value { diff --git a/prepared_execution.go b/prepared_execution.go index 9e43956..2508fec 100644 --- a/prepared_execution.go +++ b/prepared_execution.go @@ -15,6 +15,8 @@ const preparedExecutionString = "promptkit.PreparedExecution{opaque}" // A successful PrepareExecution may retain an Observer function in this // handle's private payload. The callback and other private execution settings // are cleared on claim completion or Discard. Details never exposes them. +// Contextual observers are selected from the RunPrepared context, not retained +// from preparation. // It also retains RunRequest.CaptureExchange and RunRequest.ContinueFrom for // execution of the frozen snapshot. Only RunPrepared can produce its completed // [Exchange]; preparation alone does not. diff --git a/resource_inspection.go b/resource_inspection.go index e9977e4..e0db9b2 100644 --- a/resource_inspection.go +++ b/resource_inspection.go @@ -28,8 +28,10 @@ type ProfileInspection struct { // caller-owned. Nil pointers denote absent fields; non-nil pointers preserve // explicit zero values and empty collections in JSON and YAML. type BackendInspection struct { - BackendID string `json:"id" yaml:"id"` - Provider *string `json:"provider,omitempty" yaml:"provider,omitempty"` + BackendID string `json:"id" yaml:"id"` + // Provider is the normalized declaration, not a manufactured runtime default. + Provider *string `json:"provider,omitempty" yaml:"provider,omitempty"` + // ZeroCost preserves an explicitly declared false as well as true. ZeroCost *bool `json:"zero_cost,omitempty" yaml:"zero_cost,omitempty"` Endpoint string `json:"endpoint" yaml:"endpoint"` APIKeyEnv *string `json:"api_key_env,omitempty" yaml:"api_key_env,omitempty"` diff --git a/types.go b/types.go index 7e9c66c..de9dc49 100644 --- a/types.go +++ b/types.go @@ -574,7 +574,7 @@ type TokenUsage struct { ReasoningTokens int `json:"reasoning_tokens"` // TotalTokens is the reported total-token count. TotalTokens int `json:"total_tokens"` - // CachedTokens is the reported cached-input-token count. + // CachedTokens is the reported input-token cache-read count. CachedTokens int `json:"cached_tokens"` // CacheWriteTokens is the reported cache-write-token count. CacheWriteTokens int `json:"cache_write_tokens"` @@ -660,7 +660,9 @@ type StructuredOutputJSONSpec struct { // treated as known and zero counts as unknown for observation only; the legacy // interface cannot distinguish omitted zero from measured zero. An explicit // UsagePresence can report known zero counts. Result usage sums the supplied -// Usage values independently of presence. +// Usage values independently of presence. A non-nil CostUSD is copied and +// retained for attempt accounting even when returned alongside an error; +// non-finite values are treated as unknown before configured-zero fallback. type LLMClient interface { Generate(context.Context, GenerateRequest) (*GenerateResponse, error) }