Document invocation observation and cost reporting

This commit is contained in:
2026-11-22 18:38:11 +00:00
parent d4509b163e
commit 20c5ea47af
22 changed files with 231 additions and 52 deletions

View File

@@ -49,7 +49,11 @@ func (t *scriptedTransport) RoundTrip(req *http.Request) (*http.Response, error)
func modelResponse(content string, promptTokens, completionTokens int) *http.Response {
body, _ := json.Marshal(map[string]any{
"choices": []any{map[string]any{"message": map[string]string{"content": content}}},
"usage": map[string]int{"prompt_tokens": promptTokens, "completion_tokens": completionTokens},
"usage": map[string]any{
"prompt_tokens": promptTokens, "completion_tokens": completionTokens,
"prompt_tokens_details": map[string]int{"cached_tokens": 2, "cache_write_tokens": 1},
"completion_tokens_details": map[string]int{"reasoning_tokens": 1}, "cost": .01,
},
})
return httpResponse(http.StatusOK, string(body))
}
@@ -80,6 +84,10 @@ type summary struct {
ExhaustedOutcome promptkit.CoordinationOutcome `json:"exhausted_outcome"`
GroupMembers int `json:"group_members"`
GroupAttempts uint64 `json:"group_attempts"`
GroupCost promptkit.ObservedCost `json:"group_cost"`
GroupReasoning promptkit.ObservedTokenCount `json:"group_reasoning"`
GroupCacheRatioAvailable bool `json:"group_cache_ratio_available"`
AssessmentCacheRatio float64 `json:"assessment_cache_ratio"`
PreparedMessages int `json:"prepared_messages"`
PreparedParentMatches bool `json:"prepared_parent_matches"`
}
@@ -97,6 +105,8 @@ func run() (summary, error) {
filesystem := configkit.NewFilesystem("example schemas", assets)
backend := promptkit.LocalBackend("https://example.invalid/v1", 1)
backend.ID = backendID
provider, zeroCost := "openrouter", true
backend.Provider, backend.ZeroCost = &provider, &zeroCost
inputs = append(inputs,
configkit.Input{Source: promptkit.SchemaRootSource("example schemas", "schemas"), Destination: configkit.Keys("promptkit"), Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "."}},
promptkit.SettingsInput(promptkit.BackendSource("example backend", []promptkit.BackendConfig{
@@ -117,8 +127,9 @@ func run() (summary, error) {
var proposalLog eventLog
var executions promptkit.ExecutionGroup
proposal, err := engine.Run(context.Background(), promptkit.RunRequest{
PromptID: "example.proposal", Inputs: map[string]promptkit.ArtifactRef{"idea": promptkit.Inline("Reduce queue cleanup toil.")}, CaptureExchange: true, Observer: promptkit.ComposeObservers(proposalLog.record, executions.Observe),
ctx := promptkit.WithExecutionObserver(context.Background(), executions.Observe)
proposal, err := engine.Run(ctx, promptkit.RunRequest{
PromptID: "example.proposal", Inputs: map[string]promptkit.ArtifactRef{"idea": promptkit.Inline("Reduce queue cleanup toil.")}, CaptureExchange: true, Observer: proposalLog.record,
})
if err != nil {
return summary{}, err
@@ -133,8 +144,8 @@ func run() (summary, error) {
var assessmentLog eventLog
var preparedDetails promptkit.PreparedRun
decision := "rejected"
coordination, err := engine.Coordinate(context.Background(), promptkit.CoordinateRequest{
Start: proposal.Exchange, AdditionalTurns: 1, Observer: executions.Observe,
coordination, err := engine.Coordinate(ctx, promptkit.CoordinateRequest{
Start: proposal.Exchange, AdditionalTurns: 1,
Next: func(ctx context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
if turn.Result == nil {
return &promptkit.RunRequest{PromptID: "example.proposal", Inputs: map[string]promptkit.ArtifactRef{"idea": promptkit.Inline("Refine the proposal.")}, OperationTimeout: 10 * time.Second, OnPrepared: func(_ context.Context, details promptkit.PreparedRun) { preparedDetails = details }}, nil
@@ -142,7 +153,7 @@ func run() (summary, error) {
if !turn.Result.Validation.IsValid {
return nil, fmt.Errorf("follow-up failed structural validation")
}
assessment, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "example.assessment", Inputs: map[string]promptkit.ArtifactRef{"proposal": promptkit.Inline(turn.Result.RawOutput)}, OperationTimeout: 10 * time.Second, Observer: promptkit.ComposeObservers(assessmentLog.record, executions.Observe)})
assessment, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "example.assessment", Inputs: map[string]promptkit.ArtifactRef{"proposal": promptkit.Inline(turn.Result.RawOutput)}, OperationTimeout: 10 * time.Second, Observer: assessmentLog.record})
if err != nil {
return nil, err
}
@@ -181,6 +192,8 @@ func run() (summary, error) {
return summary{}, err
}
group := executions.Snapshot()
_, ratioAvailable := group.Usage.CacheHitRatio()
assessmentRatio, _ := assessmentTerminal.Usage.CacheHitRatio()
return summary{
ProposalAttempts: proposalTerminal.Attempts,
ProposalTransientRetries: proposalTerminal.TransientRetries,
@@ -192,6 +205,10 @@ func run() (summary, error) {
ExhaustedOutcome: exhausted.Outcome,
GroupMembers: len(group.Members),
GroupAttempts: group.Attempts,
GroupCost: group.CostUSD,
GroupReasoning: group.Usage.ReasoningTokens,
GroupCacheRatioAvailable: ratioAvailable,
AssessmentCacheRatio: assessmentRatio,
PreparedMessages: len(preparedDetails.Messages),
PreparedParentMatches: preparedDetails.ParentRunID == proposal.RunID,
}, nil

View File

@@ -3,7 +3,10 @@ package main
import (
"bytes"
"encoding/json"
"math"
"testing"
"go.maximumdirect.net/promptkit"
)
func TestObserveExample(t *testing.T) {
@@ -12,18 +15,22 @@ func TestObserveExample(t *testing.T) {
t.Fatal(err)
}
var got struct {
ProposalAttempts *int `json:"proposal_attempts"`
ProposalRetries *int `json:"proposal_transient_retries"`
UsageComplete *bool `json:"proposal_usage_complete"`
AssessmentAttempts *int `json:"assessment_attempts"`
AssessmentRetries *int `json:"assessment_transient_retries"`
Outcome string `json:"assessment_outcome"`
CoordinationOutcome string `json:"coordination_outcome"`
ExhaustedOutcome string `json:"exhausted_outcome"`
GroupMembers int `json:"group_members"`
GroupAttempts int `json:"group_attempts"`
PreparedMessages int `json:"prepared_messages"`
PreparedParentMatches bool `json:"prepared_parent_matches"`
ProposalAttempts *int `json:"proposal_attempts"`
ProposalRetries *int `json:"proposal_transient_retries"`
UsageComplete *bool `json:"proposal_usage_complete"`
AssessmentAttempts *int `json:"assessment_attempts"`
AssessmentRetries *int `json:"assessment_transient_retries"`
Outcome string `json:"assessment_outcome"`
CoordinationOutcome string `json:"coordination_outcome"`
ExhaustedOutcome string `json:"exhausted_outcome"`
GroupMembers int `json:"group_members"`
GroupAttempts int `json:"group_attempts"`
GroupCost promptkit.ObservedCost `json:"group_cost"`
GroupReasoning promptkit.ObservedTokenCount `json:"group_reasoning"`
GroupCacheRatioAvailable bool `json:"group_cache_ratio_available"`
AssessmentCacheRatio float64 `json:"assessment_cache_ratio"`
PreparedMessages int `json:"prepared_messages"`
PreparedParentMatches bool `json:"prepared_parent_matches"`
}
if err := json.Unmarshal(output.Bytes(), &got); err != nil {
t.Fatal(err)
@@ -33,4 +40,7 @@ func TestObserveExample(t *testing.T) {
got.AssessmentRetries == nil || *got.AssessmentRetries != 0 || got.Outcome != "accepted" || got.CoordinationOutcome != "stopped" || got.ExhaustedOutcome != "exhausted" || got.GroupMembers != 3 || got.GroupAttempts != 4 || got.PreparedMessages < 3 || !got.PreparedParentMatches {
t.Fatalf("unexpected observation summary: %+v", got)
}
if !got.GroupCost.Known || !got.GroupCost.Complete || got.GroupCost.Source != promptkit.CostSourceMixed || math.Abs(got.GroupCost.Value-.03) > 1e-12 || got.GroupReasoning.Value != 3 || got.GroupReasoning.Complete || got.GroupCacheRatioAvailable || math.Abs(got.AssessmentCacheRatio-2.0/7.0) > 1e-12 {
t.Fatalf("unexpected usage summary: %+v", got)
}
}