diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index cfdf3fbc..c47b1da8 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -448,6 +448,22 @@ unavailable assessment, guidance contract errors, and cancellation. ## Stage 10: Implement The Small Native Producer Helper +**Status:** Complete. Added one typed helper using native Run/Coordinate and +module-owned interpretation/correction/fallback functions. It checks native +structural status before interpretation, reuses the initial assessment on the +first coordination callback, and assesses every delivered revision once, +including the final allowed turn. Native coordination owns history and budget. +Execution/profile/session selections and existing observers/preparation callbacks +are preserved without duplicate enrollment. Failed continuations cannot recover +earlier candidates; caller cancellation and undelivered responses never become +acceptance or fallback. Real-engine offline fake-provider tests cover acceptance, +zero turns, final-turn acceptance, exhaustion, nil-error structural failure, +operational/unavailable failures, interpretation guidance, cancellation, and +typed native operation timeouts. File-scoped helper tests/vet/race checks and +contracts tests pass. Full pipeline tests remain blocked by the recorded legacy +ConfigKit YAML import and unmigrated consumers; no whole-package success is +claimed. Runner integration, caching, and capture remain in their assigned stages. + **Goal:** delegate turn iteration while retaining typed Notarius acceptance. **Inspect/change:** a focused helper alongside pipeline assessment; upstream diff --git a/internal/framework/pipeline/producer.go b/internal/framework/pipeline/producer.go new file mode 100644 index 00000000..106c96cf --- /dev/null +++ b/internal/framework/pipeline/producer.go @@ -0,0 +1,184 @@ +package pipeline + +import ( + "context" + "errors" + "fmt" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "go.maximumdirect.net/promptkit" +) + +// RunProducer connects native execution to typed interpretation and application +// assessment. Interpretation returns nil when no typed candidate can be built; +// actionable findings remain separate from operational errors. Correction builds +// only the next assignment, never conversation history. Fallback receives only +// the final usable candidate, or nil after an unusable/failed execution. +func RunProducer[T any]( + ctx context.Context, + engine *promptkit.Engine, + operation contracts.ProducerOperation[T], + initial promptkit.RunRequest, + interpret func(context.Context, *promptkit.RunResult) (*T, []string, error), + correct func(context.Context, []string) (promptkit.RunRequest, error), + fallback func(context.Context, *T) (T, error), +) (T, contracts.ProducerOutcome, error) { + var zero T + if ctx == nil { + return zero, contracts.ProducerOutcome{}, errors.New("producer context must not be nil") + } + if err := ctx.Err(); err != nil { + return zero, contracts.ProducerOutcome{}, err + } + if interpret == nil || operation.Assess == nil { + return zero, contracts.ProducerOutcome{}, errors.New("producer interpretation and assessment must be supplied") + } + if operation.Controls.MaxAdditionalTurns < 0 { + return zero, contracts.ProducerOutcome{}, fmt.Errorf("%w: additional turns must not be negative", promptkit.ErrInvalidRequest) + } + + var candidate *T + var guidance []string + var failure error + var assessmentError error + accepted := false + evaluate := func(result *promptkit.RunResult) { + candidate, guidance, failure, accepted = nil, nil, nil, false + if canceled := ctx.Err(); canceled != nil { + failure = canceled + return + } + if result == nil || result.Validation.Status == promptkit.ValidationFailed || !result.Validation.IsValid { + failure = fmt.Errorf("%w: producer output did not satisfy its native contract", promptkit.ErrValidation) + return + } + candidate, guidance, failure = interpret(ctx, result) + if canceled := ctx.Err(); canceled != nil { + failure = canceled + return + } + if failure != nil { + candidate, guidance = nil, nil + return + } + if candidate == nil { + failure = errors.New("producer response could not be interpreted as a usable candidate") + return + } + assessment, err := operation.Assess(ctx, *candidate) + if err != nil { + assessmentError = err + return + } + validatorGuidance, err := assessment.Guidance() + if err != nil { + assessmentError = err + return + } + guidance = append(guidance, validatorGuidance...) + accepted = assessment.Accepted() && len(guidance) == 0 + failure = assessment.Err() + if !accepted && failure == nil { + failure = errors.New("producer candidate requires correction") + } + } + + request := producerRequest(initial, operation.Execution, true) + request.CaptureExchange = request.CaptureExchange || (correct != nil && operation.Controls.MaxAdditionalTurns > 0) + result, err := engine.Run(ctx, request) + if canceled := ctx.Err(); canceled != nil { + return zero, contracts.ProducerOutcome{}, canceled + } + if err != nil { + failure = err + } else { + evaluate(result) + } + if canceled := ctx.Err(); canceled != nil { + return zero, contracts.ProducerOutcome{}, canceled + } + if assessmentError != nil { + return zero, contracts.ProducerOutcome{}, assessmentError + } + + if !accepted && err == nil && len(guidance) > 0 && correct != nil && operation.Controls.MaxAdditionalTurns > 0 { + coordination, coordinateErr := engine.Coordinate(ctx, promptkit.CoordinateRequest{ + Start: result.Exchange, AdditionalTurns: operation.Controls.MaxAdditionalTurns, Observer: operation.Execution.Observer, + Next: func(_ context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) { + // The first callback has no new result. Its findings already exist. + if turn.Result != nil { + evaluate(turn.Result) + } + if assessmentError != nil { + return nil, assessmentError + } + if accepted || len(guidance) == 0 || turn.RemainingTurns == 0 { + return nil, nil + } + // Once another execution is selected, an earlier candidate is not a + // terminal fallback source if construction or native execution fails. + candidate = nil + next, buildErr := correct(ctx, guidance) + if buildErr != nil { + return nil, buildErr + } + if initial.ProfileID != "" { + next.ProfileID = initial.ProfileID + } + if initial.SessionID != "" { + next.SessionID = initial.SessionID + } + next = producerRequest(next, operation.Execution, false) + return &next, nil + }, + }) + if canceled := ctx.Err(); canceled != nil { + return zero, contracts.ProducerOutcome{}, canceled + } + if coordination.Undelivered != nil { + if coordinateErr == nil { + coordinateErr = errors.New("native continuation response was not delivered") + } + return zero, contracts.ProducerOutcome{}, coordinateErr + } + if assessmentError != nil { + return zero, contracts.ProducerOutcome{}, assessmentError + } + if coordinateErr != nil { + candidate, failure, accepted = nil, coordinateErr, false + } + } + if accepted { + return *candidate, contracts.ProducerOutcome{Disposition: contracts.ProducerAccepted, ReuseEligible: operation.ReuseEligible}, nil + } + var construct func(context.Context) (T, error) + if fallback != nil { + construct = func(ctx context.Context) (T, error) { return fallback(ctx, candidate) } + } + return finishProducerFailure(ctx, operation.Controls.Mode, failure, construct) +} + +// producerRequest applies native options to a value copy. The invocation +// observer is enrolled here for the initial run and on CoordinateRequest for +// continuations, never twice on the same continuation. +func producerRequest(request promptkit.RunRequest, options contracts.ExecutionOptions, initial bool) promptkit.RunRequest { + if options.Execution != nil { + request.Execution = options.Execution + } + if options.TransientRetries != nil { + request.TransientRetries = options.TransientRetries + } + if initial { + request.Observer = promptkit.ComposeObservers(request.Observer, options.Observer) + } + if options.OnPrepared != nil { + existing := request.OnPrepared + request.OnPrepared = func(ctx context.Context, details promptkit.PreparedRun) { + if existing != nil { + existing(ctx, details) + } + options.OnPrepared(ctx, details) + } + } + return request +} diff --git a/internal/framework/pipeline/producer_test.go b/internal/framework/pipeline/producer_test.go new file mode 100644 index 00000000..8a7b71ea --- /dev/null +++ b/internal/framework/pipeline/producer_test.go @@ -0,0 +1,518 @@ +package pipeline + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "reflect" + "strings" + "testing" + "testing/synctest" + "time" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "go.maximumdirect.net/configkit" + "go.maximumdirect.net/promptkit" +) + +type producerClientFunc func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) + +func (f producerClientFunc) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + return f(ctx, request) +} + +func nativeProducerEngine(t *testing.T, client promptkit.LLMClient) *promptkit.Engine { + t.Helper() + settings := `profiles: + - id: base + endpoint: https://example.invalid/v1 + model: initial-model + api_key_required: false + - id: correction-default + endpoint: https://example.invalid/v1 + model: correction-model + api_key_required: false + - id: chosen + endpoint: https://example.invalid/v1 + model: chosen-model + api_key_required: false +prompts: + - id: initial + default_profile: base + inputs: [{name: source, required: true}] + messages: + - {role: system, content: 'Keep enduring constraints.'} + - {role: user, content: 'Source: {{input "source"}}'} + output: {format: json, validation_mode: json, repair_attempts: 0} + - id: correction + default_profile: correction-default + inputs: [{name: guidance, required: true}] + messages: + - {role: user, content: 'Correct: {{input "guidance"}}'} + output: {format: json, validation_mode: json, repair_attempts: 0} +` + engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ + Underlays: []configkit.Underlay{{Inputs: []configkit.Input{promptkit.SettingsInput(configkit.Bytes("producer fixture", []byte(settings)))}}}, + }}, promptkit.WithBundledOpenRouter(false), promptkit.WithLLMClient(client)) + if err != nil { + t.Fatal(err) + } + return engine +} + +func nativeProducerInitial() promptkit.RunRequest { + return promptkit.RunRequest{PromptID: "initial", Inputs: map[string]promptkit.ArtifactRef{"source": promptkit.Inline("fixed source")}} +} + +func interpretProducerNumber(_ context.Context, result *promptkit.RunResult) (*int, []string, error) { + var candidate struct { + Value int `json:"value"` + } + if err := json.Unmarshal([]byte(result.RawOutput), &candidate); err != nil { + return nil, nil, err + } + return &candidate.Value, nil, nil +} + +func producerCorrection(_ context.Context, guidance []string) (promptkit.RunRequest, error) { + return promptkit.RunRequest{PromptID: "correction", Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline(strings.Join(guidance, "\n"))}}, nil +} + +func acceptedProducerAssessment(context.Context, int) (contracts.CandidateAssessment, error) { + return contracts.CandidateAssessment{}, nil +} + +func rejectedProducerAssessment(context.Context, int) (contracts.CandidateAssessment, error) { + return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{ValidatorName: "domain", Decision: contracts.AssessmentRejected, Result: contracts.ValidationResult{ReasonCode: "not_one", Message: "operator diagnostic", CorrectionGuidance: "Return value one."}}}}, nil +} + +func TestNativeProducerAcceptsOnlyAssessedCandidates(t *testing.T) { + for _, test := range []struct { + name string + values []int + turns int + mode contracts.ExecutionMode + wantAccepted bool + }{ + {"initial acceptance", []int{1}, 3, contracts.ModeStrict, true}, + {"zero turns", []int{0}, 0, contracts.ModeStrict, false}, + {"correction acceptance", []int{0, 1}, 3, contracts.ModeStrict, true}, + {"final turn acceptance", []int{0, 0, 1}, 2, contracts.ModeStrict, true}, + {"exhaustion", []int{0, 0, 0}, 2, contracts.ModeStrict, false}, + {"permissive exhaustion", []int{0, 0, 0}, 2, contracts.ModePermissive, false}, + } { + t.Run(test.name, func(t *testing.T) { + calls, assessments, interpreted, fallbacks := 0, 0, 0, 0 + engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + if calls >= len(test.values) { + return nil, errors.New("unexpected extra provider call") + } + value := test.values[calls] + calls++ + return &promptkit.GenerateResponse{Content: fmt.Sprintf(`{"value":%d}`, value)}, nil + })) + operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: test.mode, MaxAdditionalTurns: test.turns}, ReuseEligible: true, + Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) { + assessments++ + if value == 1 { + return acceptedProducerAssessment(ctx, value) + } + return rejectedProducerAssessment(ctx, value) + }, + } + value, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), + func(ctx context.Context, result *promptkit.RunResult) (*int, []string, error) { + interpreted++ + return interpretProducerNumber(ctx, result) + }, + producerCorrection, + func(_ context.Context, final *int) (int, error) { + fallbacks++ + if final == nil || *final != test.values[len(test.values)-1] { + t.Fatalf("fallback did not receive the final usable candidate: %v", final) + } + return -1, nil + }) + if calls != len(test.values) || assessments != calls || interpreted != calls { + t.Fatalf("provider=%d interpretation=%d assessment=%d; want %d each", calls, interpreted, assessments, len(test.values)) + } + switch { + case test.wantAccepted: + if err != nil || value != 1 || outcome.Disposition != contracts.ProducerAccepted || !outcome.ReuseEligible || fallbacks != 0 { + t.Fatalf("accepted = %d %#v %v, fallback calls %d", value, outcome, err, fallbacks) + } + case test.mode == contracts.ModePermissive: + if err != nil || value != -1 || outcome.Disposition != contracts.ProducerFallback || outcome.ReuseEligible || fallbacks != 1 { + t.Fatalf("fallback = %d %#v %v, calls %d", value, outcome, err, fallbacks) + } + default: + if err == nil || outcome.Disposition != "" || fallbacks != 0 { + t.Fatalf("strict rejection = %d %#v %v, fallback calls %d", value, outcome, err, fallbacks) + } + } + }) + } +} + +func TestNativeProducerDoesNotInterpretFailedNativeValidation(t *testing.T) { + for _, failCorrection := range []bool{false, true} { + calls, interpretations, assessments, corrections, fallbacks := 0, 0, 0, 0, 0 + engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + calls++ + if failCorrection && calls == 1 { + return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil + } + return &promptkit.GenerateResponse{Content: "not JSON"}, nil + })) + operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 2}, + Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) { + assessments++ + return rejectedProducerAssessment(ctx, value) + }, + } + _, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), + func(ctx context.Context, result *promptkit.RunResult) (*int, []string, error) { + interpretations++ + return interpretProducerNumber(ctx, result) + }, + func(ctx context.Context, guidance []string) (promptkit.RunRequest, error) { + corrections++ + return producerCorrection(ctx, guidance) + }, + func(_ context.Context, candidate *int) (int, error) { + fallbacks++ + if candidate != nil { + t.Fatal("structurally failed response recovered a prior candidate") + } + return -1, nil + }) + want := 0 + if failCorrection { + want = 1 + } + if err != nil || outcome.Disposition != contracts.ProducerFallback || calls != want+1 || interpretations != want || assessments != want || corrections != want || fallbacks != 1 { + t.Fatalf("failed native status: %#v %v, calls %d/%d/%d/%d/%d", outcome, err, calls, interpretations, assessments, corrections, fallbacks) + } + } +} + +func TestNativeProducerFailureDoesNotRecoverEarlierCandidate(t *testing.T) { + failure := errors.New("provider unavailable") + for _, failCorrection := range []bool{false, true} { + calls, interpreted, assessed := 0, 0, 0 + engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + calls++ + if failCorrection && calls == 1 { + return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil + } + return nil, failure + })) + operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 2}, + Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) { + assessed++ + return rejectedProducerAssessment(ctx, value) + }, + } + _, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), + func(ctx context.Context, result *promptkit.RunResult) (*int, []string, error) { + interpreted++ + return interpretProducerNumber(ctx, result) + }, + producerCorrection, + func(_ context.Context, candidate *int) (int, error) { + if candidate != nil { + t.Fatal("failed execution recovered an earlier candidate") + } + return -1, nil + }) + want := 0 + if failCorrection { + want = 1 + } + if err != nil || outcome.Disposition != contracts.ProducerFallback || calls != want+1 || interpreted != want || assessed != want { + t.Fatalf("provider failure = %#v %v, calls %d/%d/%d", outcome, err, calls, interpreted, assessed) + } + } + engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + return nil, failure + })) + _, _, err := RunProducer(context.Background(), engine, contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict}, Assess: acceptedProducerAssessment}, nativeProducerInitial(), interpretProducerNumber, nil, nil) + if !errors.Is(err, failure) { + t.Fatalf("native failure cause was lost: %v", err) + } +} + +func TestNativeProducerUnavailableAssessmentSuppliesNoCorrection(t *testing.T) { + calls, corrections := 0, 0 + engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + calls++ + return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil + })) + operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 5}, + Assess: func(context.Context, int) (contracts.CandidateAssessment, error) { + return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{Decision: contracts.AssessmentUnavailable, Failure: errors.New("provider unavailable")}}}, nil + }, + } + _, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, + func(ctx context.Context, guidance []string) (promptkit.RunRequest, error) { + corrections++ + return producerCorrection(ctx, guidance) + }, + func(context.Context, *int) (int, error) { return -1, nil }) + if err != nil || outcome.Disposition != contracts.ProducerFallback || calls != 1 || corrections != 0 { + t.Fatalf("unavailable assessment = %#v %v, calls %d/%d", outcome, err, calls, corrections) + } +} + +func TestNativeProducerPreservesConversationAndExplicitSelections(t *testing.T) { + for _, explicit := range []bool{false, true} { + var requests []promptkit.GenerateRequest + var profiles []string + initialPrepared, correctionPrepared, observed := 0, 0, 0 + initialObserved, correctionObserved := 0, 0 + engine := nativeProducerEngine(t, producerClientFunc(func(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + requests = append(requests, request) + return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil + })) + initial := nativeProducerInitial() + initial.SessionID = "source-session" + if explicit { + initial.ProfileID = "chosen" + } + initial.OnPrepared = func(context.Context, promptkit.PreparedRun) { initialPrepared++ } + initial.Observer = func(event promptkit.ExecutionEvent) { + if event.Kind == promptkit.ExecutionFinished { + initialObserved++ + } + } + overrideModel := "override-model" + operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 2}, Assess: rejectedProducerAssessment, + Execution: contracts.ExecutionOptions{Execution: &promptkit.ExecutionTargetOverride{Model: overrideModel}, + Observer: func(event promptkit.ExecutionEvent) { + if event.Kind == promptkit.ExecutionFinished { + observed++ + } + }, + OnPrepared: func(_ context.Context, details promptkit.PreparedRun) { + profiles = append(profiles, details.SelectedProfileID) + }, + }, + } + _, outcome, err := RunProducer(context.Background(), engine, operation, initial, interpretProducerNumber, + func(ctx context.Context, guidance []string) (promptkit.RunRequest, error) { + if !reflect.DeepEqual(guidance, []string{"Return value one."}) { + t.Fatalf("unexpected correction guidance: %#v", guidance) + } + request, err := producerCorrection(ctx, guidance) + request.OnPrepared = func(context.Context, promptkit.PreparedRun) { correctionPrepared++ } + request.Observer = func(event promptkit.ExecutionEvent) { + if event.Kind == promptkit.ExecutionFinished { + correctionObserved++ + } + } + return request, err + }, func(context.Context, *int) (int, error) { return -1, nil }) + if err != nil || outcome.Disposition != contracts.ProducerFallback || len(requests) != 3 || observed != 3 || initialPrepared != 1 || correctionPrepared != 2 { + t.Fatalf("native continuation = %#v %v, counts %d/%d/%d/%d", outcome, err, len(requests), observed, initialPrepared, correctionPrepared) + } + if initialObserved != 1 || correctionObserved != 2 { + t.Fatalf("existing request observers were lost or duplicated: %d/%d", initialObserved, correctionObserved) + } + wantProfiles := []string{"base", "correction-default", "correction-default"} + if explicit { + wantProfiles = []string{"chosen", "chosen", "chosen"} + } + if !reflect.DeepEqual(profiles, wantProfiles) { + t.Fatalf("profiles = %#v, want %#v", profiles, wantProfiles) + } + for i, request := range requests { + if request.Target.Model != overrideModel || request.Prompt.SessionID != initial.SessionID { + t.Fatalf("selection lost on turn %d: %#v", i, request) + } + var content []string + for _, message := range request.Prompt.Messages { + content = append(content, message.Content) + } + joined := strings.Join(content, "\n") + if strings.Count(joined, "fixed source") != 1 || strings.Count(joined, "Keep enduring constraints.") != 1 || len(request.Prompt.Messages) != 2+2*i { + t.Fatalf("conversation replayed or lost history on turn %d: %#v", i, content) + } + } + if initial.CaptureExchange || initial.Execution != nil || initial.ProfileID == "base" { + t.Fatal("helper mutated the caller's initial request") + } + } +} + +func TestNativeProducerCancellationDoesNotDeliverOrFallback(t *testing.T) { + for _, moment := range []string{"before initial", "after initial execution", "inside interpretation", "inside assessment", "before continuation delivery"} { + t.Run(moment, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + calls, interpreted, assessed, finished, fallbacks := 0, 0, 0, 0, 0 + engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + calls++ + return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil + })) + operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 2}, + Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) { + assessed++ + if moment == "inside assessment" { + cancel() + return acceptedProducerAssessment(ctx, value) + } + return rejectedProducerAssessment(ctx, value) + }, + Execution: contracts.ExecutionOptions{Observer: func(event promptkit.ExecutionEvent) { + if event.Kind == promptkit.ExecutionFinished { + finished++ + if (moment == "before continuation delivery" && finished == 2) || moment == "after initial execution" { + cancel() + } + } + }}, + } + if moment == "before initial" { + cancel() + } + _, outcome, err := RunProducer(ctx, engine, operation, nativeProducerInitial(), + func(ctx context.Context, result *promptkit.RunResult) (*int, []string, error) { + interpreted++ + if moment == "inside interpretation" { + cancel() + } + return interpretProducerNumber(ctx, result) + }, + producerCorrection, + func(context.Context, *int) (int, error) { fallbacks++; return -1, nil }) + if !errors.Is(err, context.Canceled) || outcome.Disposition != "" || fallbacks != 0 { + t.Fatalf("cancellation = %#v %v, fallback calls %d", outcome, err, fallbacks) + } + switch moment { + case "before initial": + if calls != 0 || interpreted != 0 || assessed != 0 { + t.Fatalf("work started after cancellation: %d/%d/%d", calls, interpreted, assessed) + } + case "inside assessment": + if calls != 1 || interpreted != 1 || assessed != 1 { + t.Fatalf("canceled assessment repeated: %d/%d/%d", calls, interpreted, assessed) + } + case "after initial execution": + if calls != 1 || interpreted != 0 || assessed != 0 { + t.Fatalf("canceled native result was interpreted: %d/%d/%d", calls, interpreted, assessed) + } + case "inside interpretation": + if calls != 1 || interpreted != 1 || assessed != 0 { + t.Fatalf("assessment started after cancellation: %d/%d/%d", calls, interpreted, assessed) + } + default: + if calls != 2 || interpreted != 1 || assessed != 1 { + t.Fatalf("undelivered response assessed: %d/%d/%d", calls, interpreted, assessed) + } + } + }) + } +} + +func TestNativeProducerOperationTimeoutKeepsCallerActive(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + for _, continuation := range []bool{false, true} { + for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} { + calls, fallbacks := 0, 0 + engine := nativeProducerEngine(t, producerClientFunc(func(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + calls++ + if continuation && calls == 1 { + return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil + } + <-ctx.Done() + return nil, ctx.Err() + })) + initial := nativeProducerInitial() + initial.OperationTimeout = time.Second + parent := context.Background() + _, outcome, err := RunProducer(parent, engine, contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: mode, MaxAdditionalTurns: 2}, Assess: rejectedProducerAssessment}, + initial, interpretProducerNumber, + func(ctx context.Context, guidance []string) (promptkit.RunRequest, error) { + request, err := producerCorrection(ctx, guidance) + request.OperationTimeout = time.Second + return request, err + }, + func(_ context.Context, final *int) (int, error) { + fallbacks++ + if final != nil { + t.Fatal("timeout recovered an earlier candidate") + } + return -1, nil + }) + if parent.Err() != nil { + t.Fatalf("native timeout canceled caller: %v", parent.Err()) + } + if mode == contracts.ModeStrict { + if !errors.Is(err, promptkit.ErrOperationTimeout) || !errors.Is(err, context.DeadlineExceeded) || outcome.Disposition != "" || fallbacks != 0 { + t.Fatalf("strict timeout = %#v %v, fallback calls %d", outcome, err, fallbacks) + } + } else if err != nil || outcome.Disposition != contracts.ProducerFallback || fallbacks != 1 { + t.Fatalf("permissive timeout = %#v %v, fallback calls %d", outcome, err, fallbacks) + } + } + } + }) +} + +func TestNativeProducerInterpretationFindingsUseNativeCorrection(t *testing.T) { + for _, usable := range []bool{false, true} { + calls, interpreted, assessed := 0, 0, 0 + engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + calls++ + return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil + })) + operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, MaxAdditionalTurns: 1}, ReuseEligible: true, + Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) { + assessed++ + return acceptedProducerAssessment(ctx, value) + }, + } + value, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), + func(ctx context.Context, result *promptkit.RunResult) (*int, []string, error) { + interpreted++ + candidate, _, err := interpretProducerNumber(ctx, result) + if interpreted == 1 { + if !usable { + candidate = nil + } + return candidate, []string{"Use the supplied candidate handles."}, err + } + return candidate, nil, err + }, + func(ctx context.Context, guidance []string) (promptkit.RunRequest, error) { + if !reflect.DeepEqual(guidance, []string{"Use the supplied candidate handles."}) { + t.Fatalf("module findings lost: %#v", guidance) + } + return producerCorrection(ctx, guidance) + }, nil) + wantAssessed := 1 + if usable { + wantAssessed = 2 + } + if err != nil || outcome.Disposition != contracts.ProducerAccepted || value != 1 || calls != 2 || interpreted != 2 || assessed != wantAssessed { + t.Fatalf("interpretation correction = %d %#v %v, calls %d/%d/%d", value, outcome, err, calls, interpreted, assessed) + } + } +} + +func TestNativeProducerAssessmentContractFailureDoesNotFallback(t *testing.T) { + failure := errors.New("assessment framework failure") + fallbacks, calls := 0, 0 + engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + calls++ + return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil + })) + operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 2}, Assess: func(context.Context, int) (contracts.CandidateAssessment, error) { + return contracts.CandidateAssessment{}, failure + }} + _, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, producerCorrection, func(context.Context, *int) (int, error) { fallbacks++; return -1, nil }) + if !errors.Is(err, failure) || outcome.Disposition != "" || calls != 1 || fallbacks != 0 { + t.Fatalf("framework failure = %#v %v, calls %d/%d", outcome, err, calls, fallbacks) + } +}