From 04569e44df272611353a0aef636a5cfc327c47b0 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sat, 14 Nov 2026 01:18:55 +0000 Subject: [PATCH] Capture completed conversation exchanges --- convert.go | 6 ++ docs/roadmap/implementation.md | 2 + exchange_capture_contract_test.go | 91 ++++++++++++++++++++++++++ internal/domain/domain.go | 2 + internal/usecase/prepared_execution.go | 8 ++- internal/usecase/runner.go | 10 ++- prepared_execution_contract_test.go | 11 +++- 7 files changed, 126 insertions(+), 4 deletions(-) create mode 100644 exchange_capture_contract_test.go diff --git a/convert.go b/convert.go index b53b7d2..516100d 100644 --- a/convert.go +++ b/convert.go @@ -102,7 +102,13 @@ func fromDomainRunResult(result *domain.RunResult) *RunResult { if result == nil { return nil } + var exchange *Exchange + if result.Exchange != nil { + exchange = &Exchange{internal: result.Exchange} + } return &RunResult{ + Exchange: exchange, + ParentRunID: result.ParentRunID, RunID: result.RunID, Artifact: fromDomainArtifact(result.Artifact), RawOutput: result.RawOutput, diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 79372d5..7c91846 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -363,6 +363,8 @@ Do not add a separate preparation or validation algorithm for continuation. ## Stage 3 — Capture Ordinary And Prepared Executions +**Status:** Complete. + **Scope:** completed-result creation and prepared payload lifetime. 1. Carry capture/parent state through `preparedOperation` and diff --git a/exchange_capture_contract_test.go b/exchange_capture_contract_test.go new file mode 100644 index 0000000..0a8d3ac --- /dev/null +++ b/exchange_capture_contract_test.go @@ -0,0 +1,91 @@ +package promptkit_test + +import ( + "context" + "encoding/json" + "errors" + "reflect" + "strings" + "testing" + + "go.maximumdirect.net/promptkit" +) + +func TestExchangeCaptureAndPreparedContinuation(t *testing.T) { + client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "answer"}} + engine := newSimpleEngine(t, "question", promptkit.WithLLMClient(client)) + ctx := context.Background() + plain, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt"}) + if err != nil || plain.Exchange != nil { + t.Fatalf("plain run = %#v, %v", plain, err) + } + initial, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true}) + if err != nil || initial.Exchange == nil { + t.Fatalf("captured run = %#v, %v", initial, err) + } + want := []promptkit.RenderedMessage{{Role: promptkit.RoleUser, Content: "question"}, {Role: promptkit.RoleAssistant, Content: "answer"}} + if details := initial.Exchange.Details(); details.RunID != initial.RunID || !reflect.DeepEqual(details.Messages, want) { + t.Fatalf("details = %#v", details) + } + encoded, err := json.Marshal(initial) + if err != nil || strings.Contains(string(encoded), "exchange") || strings.Contains(string(encoded), "question") { + t.Fatalf("result JSON = %s, %v", encoded, err) + } + handle, err := engine.PrepareExecution(ctx, promptkit.RunRequest{PromptID: "prompt", ContinueFrom: initial.Exchange}) + if err != nil { + t.Fatal(err) + } + if handle.Details().ParentRunID != initial.RunID { + t.Fatal("prepared ancestry missing") + } + continued, err := engine.RunPrepared(ctx, handle) + if err != nil || continued.Exchange == nil || continued.ParentRunID != initial.RunID { + t.Fatalf("continued = %#v, %v", continued, err) + } + want = append(want, promptkit.RenderedMessage{Role: promptkit.RoleUser, Content: "question"}, promptkit.RenderedMessage{Role: promptkit.RoleAssistant, Content: "answer"}) + if details := continued.Exchange.Details(); details.ParentRunID != initial.RunID || !reflect.DeepEqual(details.Messages, want) { + t.Fatalf("continued details = %#v", details) + } +} + +func TestExchangeCapturesInvalidFinalOutput(t *testing.T) { + client := &fakeLLMClient{responses: []*promptkit.GenerateResponse{{Content: ""}, {Content: " "}}} + engine := newSimpleEngine(t, "question", promptkit.WithLLMClient(client)) + result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true, + Validation: &promptkit.OutputContract{Format: promptkit.FormatText, ValidationMode: promptkit.ValidationBasic, RepairAttempts: 1}}) + if err != nil || result == nil || result.Validation.Status != promptkit.ValidationFailed || result.Exchange == nil { + t.Fatalf("invalid result = %#v, %v", result, err) + } + if messages := result.Exchange.Details().Messages; len(messages) != 2 || messages[1].Content != " " { + t.Fatalf("history = %#v", messages) + } +} + +func TestExchangeFailuresLeaveSourceReusable(t *testing.T) { + client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "answer"}} + engine := newSimpleEngine(t, "question", promptkit.WithLLMClient(client)) + ctx := context.Background() + initial, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true}) + if err != nil { + t.Fatal(err) + } + original := initial.Exchange.Details() + request := promptkit.RunRequest{PromptID: "prompt", ContinueFrom: initial.Exchange} + handle, err := engine.PrepareExecution(ctx, request) + if err != nil { + t.Fatal(err) + } + handle.Discard() + if result, err := engine.RunPrepared(ctx, handle); result != nil || !errors.Is(err, promptkit.ErrInvalidRequest) { + t.Fatalf("discarded = %#v, %v", result, err) + } + client.err = errors.New("provider failure") + if result, err := engine.Run(ctx, request); result != nil || !errors.Is(err, promptkit.ErrLLMGenerate) { + t.Fatalf("failure = %#v, %v", result, err) + } + client.err = nil + result, err := engine.Run(ctx, request) + if err != nil || result.Exchange == nil || !reflect.DeepEqual(initial.Exchange.Details(), original) { + t.Fatalf("reused source = %#v, %v", result, err) + } +} diff --git a/internal/domain/domain.go b/internal/domain/domain.go index e3a01fb..728cf25 100644 --- a/internal/domain/domain.go +++ b/internal/domain/domain.go @@ -77,6 +77,8 @@ type RunRequest struct { // RunResult represents the complete result of a prompt execution run. type RunResult struct { + Exchange *Exchange + ParentRunID string RunID string Artifact Artifact RawOutput string diff --git a/internal/usecase/prepared_execution.go b/internal/usecase/prepared_execution.go index b975068..7ac526c 100644 --- a/internal/usecase/prepared_execution.go +++ b/internal/usecase/prepared_execution.go @@ -30,6 +30,8 @@ type PreparedExecution struct { type preparedExecutionPayload struct { prepared *domain.PreparedRun + capture bool + parent *domain.Exchange validation validate.PreparedValidation directKey string transientRetries uint64 @@ -55,6 +57,8 @@ func (r *Runner) PrepareExecution(ctx context.Context, req domain.RunRequest) (* details: operation.run, payload: &preparedExecutionPayload{ prepared: operation.run, + capture: operation.capture, + parent: operation.parent, validation: operation.validation, directKey: state.effectiveModel.APIKey, transientRetries: state.transientRetries, @@ -139,7 +143,7 @@ func (r *Runner) RunPrepared(ctx context.Context, prepared *PreparedExecution) ( } defer release() - return r.executePreparedRun(ctx, payload.prepared, payload.directKey, payload.transientRetries, runID, start, func( + return r.executePreparedRun(ctx, payload.prepared, payload.capture, payload.parent, payload.directKey, payload.transientRetries, runID, start, func( ctx context.Context, artifact *domain.Artifact, attemptsUsed int, @@ -174,6 +178,8 @@ func (p *preparedExecutionPayload) clear() { return } p.prepared = nil + p.capture = false + p.parent = nil p.validation = nil p.directKey = "" p.transientRetries = 0 diff --git a/internal/usecase/runner.go b/internal/usecase/runner.go index eab4ce9..c262071 100644 --- a/internal/usecase/runner.go +++ b/internal/usecase/runner.go @@ -165,7 +165,7 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes } directAPIKey := state.effectiveModel.APIKey - return r.executePreparedRun(ctx, operation.run, directAPIKey, state.transientRetries, runID, start, func( + return r.executePreparedRun(ctx, operation.run, operation.capture, operation.parent, directAPIKey, state.transientRetries, runID, start, func( ctx context.Context, artifact *domain.Artifact, attemptsUsed int, @@ -185,6 +185,8 @@ type preparedValidationFunc func( func (r *Runner) executePreparedRun( ctx context.Context, prepared *domain.PreparedRun, + capture bool, + parent *domain.Exchange, directAPIKey string, transientRetries uint64, runID string, @@ -254,8 +256,14 @@ func (r *Runner) executePreparedRun( end := time.Now().UTC() executionTarget.APIKey = "" + var exchange *domain.Exchange + if capture { + exchange = domain.CaptureExchange(parent, runID, prepared.Messages[parent.MessageCount():], genResp.Content) + } return &domain.RunResult{ + Exchange: exchange, + ParentRunID: prepared.ParentRunID, RunID: runID, Artifact: outputArtifact, RawOutput: genResp.Content, diff --git a/prepared_execution_contract_test.go b/prepared_execution_contract_test.go index 3de1299..adb8249 100644 --- a/prepared_execution_contract_test.go +++ b/prepared_execution_contract_test.go @@ -276,7 +276,7 @@ func TestPreparedExecutionRepairsEmptyBasicOutput(t *testing.T) { }} engine := newPreparedContractEngine(t, client, "Summarize the source.") prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{ - PromptID: "prepared", + PromptID: "prepared", CaptureExchange: true, Validation: &promptkit.OutputContract{ Format: promptkit.FormatMarkdown, ValidationMode: promptkit.ValidationBasic, @@ -296,6 +296,9 @@ func TestPreparedExecutionRepairsEmptyBasicOutput(t *testing.T) { result.Validation.RepairAttempts != 1 || result.Usage != (promptkit.TokenUsage{PromptTokens: 10, CompletionTokens: 16, TotalTokens: 26}) { t.Fatalf("repaired result = %+v", result) } + if result.Exchange == nil || len(result.Exchange.Details().Messages) != len(details.Messages)+1 || result.Exchange.Details().Messages[len(details.Messages)].Content != "Corrected summary." { + t.Fatalf("repaired exchange = %#v", result.Exchange) + } requests := client.snapshot() if len(requests) != 2 || len(requests[1].Prompt.Messages) != len(details.Messages)+1 || !reflect.DeepEqual(requests[1].Prompt.Messages[:len(details.Messages)], details.Messages) || @@ -315,7 +318,8 @@ func TestPreparedExecutionIsolatesMutatingClientAcrossRepair(t *testing.T) { t.Fatal(err) } prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{ - PromptID: "prepared", + PromptID: "prepared", + CaptureExchange: true, Execution: &promptkit.ExecutionTargetOverride{ExtraParams: map[string]any{ "nested": map[string]any{"value": "original"}, }}, @@ -333,6 +337,9 @@ func TestPreparedExecutionIsolatesMutatingClientAcrossRepair(t *testing.T) { if !reflect.DeepEqual(before, prepared.Details()) { t.Fatal("client mutation changed retained prepared metadata") } + if result.Exchange == nil || result.Exchange.Details().Messages[0].Content != "original message" || result.Exchange.Details().Messages[1].CacheControl.TTL != "1h" || result.Exchange.Details().Messages[2].Content != `{"value":3}` { + t.Fatalf("mutating client changed exchange history: %#v", result.Exchange.Details()) + } result.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["value"] = "result mutation" if prepared.Details().EffectiveModelParams.ExtraParams["nested"].(map[string]any)["value"] != "original" { t.Fatal("result mutation changed retained prepared metadata")