Run typed producers through native continuation and assessment

This commit is contained in:
2026-11-21 23:30:45 +00:00
parent c56c7d7ee9
commit 44e4fb92c2
3 changed files with 718 additions and 0 deletions

View File

@@ -448,6 +448,22 @@ unavailable assessment, guidance contract errors, and cancellation.
## Stage 10: Implement The Small Native Producer Helper ## Stage 10: Implement The Small Native Producer Helper
**Status:** Complete. Added one typed helper using native Run/Coordinate and
module-owned interpretation/correction/fallback functions. It checks native
structural status before interpretation, reuses the initial assessment on the
first coordination callback, and assesses every delivered revision once,
including the final allowed turn. Native coordination owns history and budget.
Execution/profile/session selections and existing observers/preparation callbacks
are preserved without duplicate enrollment. Failed continuations cannot recover
earlier candidates; caller cancellation and undelivered responses never become
acceptance or fallback. Real-engine offline fake-provider tests cover acceptance,
zero turns, final-turn acceptance, exhaustion, nil-error structural failure,
operational/unavailable failures, interpretation guidance, cancellation, and
typed native operation timeouts. File-scoped helper tests/vet/race checks and
contracts tests pass. Full pipeline tests remain blocked by the recorded legacy
ConfigKit YAML import and unmigrated consumers; no whole-package success is
claimed. Runner integration, caching, and capture remain in their assigned stages.
**Goal:** delegate turn iteration while retaining typed Notarius acceptance. **Goal:** delegate turn iteration while retaining typed Notarius acceptance.
**Inspect/change:** a focused helper alongside pipeline assessment; upstream **Inspect/change:** a focused helper alongside pipeline assessment; upstream

View File

@@ -0,0 +1,184 @@
package pipeline
import (
"context"
"errors"
"fmt"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"go.maximumdirect.net/promptkit"
)
// RunProducer connects native execution to typed interpretation and application
// assessment. Interpretation returns nil when no typed candidate can be built;
// actionable findings remain separate from operational errors. Correction builds
// only the next assignment, never conversation history. Fallback receives only
// the final usable candidate, or nil after an unusable/failed execution.
func RunProducer[T any](
ctx context.Context,
engine *promptkit.Engine,
operation contracts.ProducerOperation[T],
initial promptkit.RunRequest,
interpret func(context.Context, *promptkit.RunResult) (*T, []string, error),
correct func(context.Context, []string) (promptkit.RunRequest, error),
fallback func(context.Context, *T) (T, error),
) (T, contracts.ProducerOutcome, error) {
var zero T
if ctx == nil {
return zero, contracts.ProducerOutcome{}, errors.New("producer context must not be nil")
}
if err := ctx.Err(); err != nil {
return zero, contracts.ProducerOutcome{}, err
}
if interpret == nil || operation.Assess == nil {
return zero, contracts.ProducerOutcome{}, errors.New("producer interpretation and assessment must be supplied")
}
if operation.Controls.MaxAdditionalTurns < 0 {
return zero, contracts.ProducerOutcome{}, fmt.Errorf("%w: additional turns must not be negative", promptkit.ErrInvalidRequest)
}
var candidate *T
var guidance []string
var failure error
var assessmentError error
accepted := false
evaluate := func(result *promptkit.RunResult) {
candidate, guidance, failure, accepted = nil, nil, nil, false
if canceled := ctx.Err(); canceled != nil {
failure = canceled
return
}
if result == nil || result.Validation.Status == promptkit.ValidationFailed || !result.Validation.IsValid {
failure = fmt.Errorf("%w: producer output did not satisfy its native contract", promptkit.ErrValidation)
return
}
candidate, guidance, failure = interpret(ctx, result)
if canceled := ctx.Err(); canceled != nil {
failure = canceled
return
}
if failure != nil {
candidate, guidance = nil, nil
return
}
if candidate == nil {
failure = errors.New("producer response could not be interpreted as a usable candidate")
return
}
assessment, err := operation.Assess(ctx, *candidate)
if err != nil {
assessmentError = err
return
}
validatorGuidance, err := assessment.Guidance()
if err != nil {
assessmentError = err
return
}
guidance = append(guidance, validatorGuidance...)
accepted = assessment.Accepted() && len(guidance) == 0
failure = assessment.Err()
if !accepted && failure == nil {
failure = errors.New("producer candidate requires correction")
}
}
request := producerRequest(initial, operation.Execution, true)
request.CaptureExchange = request.CaptureExchange || (correct != nil && operation.Controls.MaxAdditionalTurns > 0)
result, err := engine.Run(ctx, request)
if canceled := ctx.Err(); canceled != nil {
return zero, contracts.ProducerOutcome{}, canceled
}
if err != nil {
failure = err
} else {
evaluate(result)
}
if canceled := ctx.Err(); canceled != nil {
return zero, contracts.ProducerOutcome{}, canceled
}
if assessmentError != nil {
return zero, contracts.ProducerOutcome{}, assessmentError
}
if !accepted && err == nil && len(guidance) > 0 && correct != nil && operation.Controls.MaxAdditionalTurns > 0 {
coordination, coordinateErr := engine.Coordinate(ctx, promptkit.CoordinateRequest{
Start: result.Exchange, AdditionalTurns: operation.Controls.MaxAdditionalTurns, Observer: operation.Execution.Observer,
Next: func(_ context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
// The first callback has no new result. Its findings already exist.
if turn.Result != nil {
evaluate(turn.Result)
}
if assessmentError != nil {
return nil, assessmentError
}
if accepted || len(guidance) == 0 || turn.RemainingTurns == 0 {
return nil, nil
}
// Once another execution is selected, an earlier candidate is not a
// terminal fallback source if construction or native execution fails.
candidate = nil
next, buildErr := correct(ctx, guidance)
if buildErr != nil {
return nil, buildErr
}
if initial.ProfileID != "" {
next.ProfileID = initial.ProfileID
}
if initial.SessionID != "" {
next.SessionID = initial.SessionID
}
next = producerRequest(next, operation.Execution, false)
return &next, nil
},
})
if canceled := ctx.Err(); canceled != nil {
return zero, contracts.ProducerOutcome{}, canceled
}
if coordination.Undelivered != nil {
if coordinateErr == nil {
coordinateErr = errors.New("native continuation response was not delivered")
}
return zero, contracts.ProducerOutcome{}, coordinateErr
}
if assessmentError != nil {
return zero, contracts.ProducerOutcome{}, assessmentError
}
if coordinateErr != nil {
candidate, failure, accepted = nil, coordinateErr, false
}
}
if accepted {
return *candidate, contracts.ProducerOutcome{Disposition: contracts.ProducerAccepted, ReuseEligible: operation.ReuseEligible}, nil
}
var construct func(context.Context) (T, error)
if fallback != nil {
construct = func(ctx context.Context) (T, error) { return fallback(ctx, candidate) }
}
return finishProducerFailure(ctx, operation.Controls.Mode, failure, construct)
}
// producerRequest applies native options to a value copy. The invocation
// observer is enrolled here for the initial run and on CoordinateRequest for
// continuations, never twice on the same continuation.
func producerRequest(request promptkit.RunRequest, options contracts.ExecutionOptions, initial bool) promptkit.RunRequest {
if options.Execution != nil {
request.Execution = options.Execution
}
if options.TransientRetries != nil {
request.TransientRetries = options.TransientRetries
}
if initial {
request.Observer = promptkit.ComposeObservers(request.Observer, options.Observer)
}
if options.OnPrepared != nil {
existing := request.OnPrepared
request.OnPrepared = func(ctx context.Context, details promptkit.PreparedRun) {
if existing != nil {
existing(ctx, details)
}
options.OnPrepared(ctx, details)
}
}
return request
}

View File

@@ -0,0 +1,518 @@
package pipeline
import (
"context"
"encoding/json"
"errors"
"fmt"
"reflect"
"strings"
"testing"
"testing/synctest"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
type producerClientFunc func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error)
func (f producerClientFunc) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
return f(ctx, request)
}
func nativeProducerEngine(t *testing.T, client promptkit.LLMClient) *promptkit.Engine {
t.Helper()
settings := `profiles:
- id: base
endpoint: https://example.invalid/v1
model: initial-model
api_key_required: false
- id: correction-default
endpoint: https://example.invalid/v1
model: correction-model
api_key_required: false
- id: chosen
endpoint: https://example.invalid/v1
model: chosen-model
api_key_required: false
prompts:
- id: initial
default_profile: base
inputs: [{name: source, required: true}]
messages:
- {role: system, content: 'Keep enduring constraints.'}
- {role: user, content: 'Source: {{input "source"}}'}
output: {format: json, validation_mode: json, repair_attempts: 0}
- id: correction
default_profile: correction-default
inputs: [{name: guidance, required: true}]
messages:
- {role: user, content: 'Correct: {{input "guidance"}}'}
output: {format: json, validation_mode: json, repair_attempts: 0}
`
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{promptkit.SettingsInput(configkit.Bytes("producer fixture", []byte(settings)))}}},
}}, promptkit.WithBundledOpenRouter(false), promptkit.WithLLMClient(client))
if err != nil {
t.Fatal(err)
}
return engine
}
func nativeProducerInitial() promptkit.RunRequest {
return promptkit.RunRequest{PromptID: "initial", Inputs: map[string]promptkit.ArtifactRef{"source": promptkit.Inline("fixed source")}}
}
func interpretProducerNumber(_ context.Context, result *promptkit.RunResult) (*int, []string, error) {
var candidate struct {
Value int `json:"value"`
}
if err := json.Unmarshal([]byte(result.RawOutput), &candidate); err != nil {
return nil, nil, err
}
return &candidate.Value, nil, nil
}
func producerCorrection(_ context.Context, guidance []string) (promptkit.RunRequest, error) {
return promptkit.RunRequest{PromptID: "correction", Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline(strings.Join(guidance, "\n"))}}, nil
}
func acceptedProducerAssessment(context.Context, int) (contracts.CandidateAssessment, error) {
return contracts.CandidateAssessment{}, nil
}
func rejectedProducerAssessment(context.Context, int) (contracts.CandidateAssessment, error) {
return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{ValidatorName: "domain", Decision: contracts.AssessmentRejected, Result: contracts.ValidationResult{ReasonCode: "not_one", Message: "operator diagnostic", CorrectionGuidance: "Return value one."}}}}, nil
}
func TestNativeProducerAcceptsOnlyAssessedCandidates(t *testing.T) {
for _, test := range []struct {
name string
values []int
turns int
mode contracts.ExecutionMode
wantAccepted bool
}{
{"initial acceptance", []int{1}, 3, contracts.ModeStrict, true},
{"zero turns", []int{0}, 0, contracts.ModeStrict, false},
{"correction acceptance", []int{0, 1}, 3, contracts.ModeStrict, true},
{"final turn acceptance", []int{0, 0, 1}, 2, contracts.ModeStrict, true},
{"exhaustion", []int{0, 0, 0}, 2, contracts.ModeStrict, false},
{"permissive exhaustion", []int{0, 0, 0}, 2, contracts.ModePermissive, false},
} {
t.Run(test.name, func(t *testing.T) {
calls, assessments, interpreted, fallbacks := 0, 0, 0, 0
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
if calls >= len(test.values) {
return nil, errors.New("unexpected extra provider call")
}
value := test.values[calls]
calls++
return &promptkit.GenerateResponse{Content: fmt.Sprintf(`{"value":%d}`, value)}, nil
}))
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: test.mode, MaxAdditionalTurns: test.turns}, ReuseEligible: true,
Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) {
assessments++
if value == 1 {
return acceptedProducerAssessment(ctx, value)
}
return rejectedProducerAssessment(ctx, value)
},
}
value, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(),
func(ctx context.Context, result *promptkit.RunResult) (*int, []string, error) {
interpreted++
return interpretProducerNumber(ctx, result)
},
producerCorrection,
func(_ context.Context, final *int) (int, error) {
fallbacks++
if final == nil || *final != test.values[len(test.values)-1] {
t.Fatalf("fallback did not receive the final usable candidate: %v", final)
}
return -1, nil
})
if calls != len(test.values) || assessments != calls || interpreted != calls {
t.Fatalf("provider=%d interpretation=%d assessment=%d; want %d each", calls, interpreted, assessments, len(test.values))
}
switch {
case test.wantAccepted:
if err != nil || value != 1 || outcome.Disposition != contracts.ProducerAccepted || !outcome.ReuseEligible || fallbacks != 0 {
t.Fatalf("accepted = %d %#v %v, fallback calls %d", value, outcome, err, fallbacks)
}
case test.mode == contracts.ModePermissive:
if err != nil || value != -1 || outcome.Disposition != contracts.ProducerFallback || outcome.ReuseEligible || fallbacks != 1 {
t.Fatalf("fallback = %d %#v %v, calls %d", value, outcome, err, fallbacks)
}
default:
if err == nil || outcome.Disposition != "" || fallbacks != 0 {
t.Fatalf("strict rejection = %d %#v %v, fallback calls %d", value, outcome, err, fallbacks)
}
}
})
}
}
func TestNativeProducerDoesNotInterpretFailedNativeValidation(t *testing.T) {
for _, failCorrection := range []bool{false, true} {
calls, interpretations, assessments, corrections, fallbacks := 0, 0, 0, 0, 0
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
if failCorrection && calls == 1 {
return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil
}
return &promptkit.GenerateResponse{Content: "not JSON"}, nil
}))
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 2},
Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) {
assessments++
return rejectedProducerAssessment(ctx, value)
},
}
_, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(),
func(ctx context.Context, result *promptkit.RunResult) (*int, []string, error) {
interpretations++
return interpretProducerNumber(ctx, result)
},
func(ctx context.Context, guidance []string) (promptkit.RunRequest, error) {
corrections++
return producerCorrection(ctx, guidance)
},
func(_ context.Context, candidate *int) (int, error) {
fallbacks++
if candidate != nil {
t.Fatal("structurally failed response recovered a prior candidate")
}
return -1, nil
})
want := 0
if failCorrection {
want = 1
}
if err != nil || outcome.Disposition != contracts.ProducerFallback || calls != want+1 || interpretations != want || assessments != want || corrections != want || fallbacks != 1 {
t.Fatalf("failed native status: %#v %v, calls %d/%d/%d/%d/%d", outcome, err, calls, interpretations, assessments, corrections, fallbacks)
}
}
}
func TestNativeProducerFailureDoesNotRecoverEarlierCandidate(t *testing.T) {
failure := errors.New("provider unavailable")
for _, failCorrection := range []bool{false, true} {
calls, interpreted, assessed := 0, 0, 0
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
if failCorrection && calls == 1 {
return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil
}
return nil, failure
}))
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 2},
Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) {
assessed++
return rejectedProducerAssessment(ctx, value)
},
}
_, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(),
func(ctx context.Context, result *promptkit.RunResult) (*int, []string, error) {
interpreted++
return interpretProducerNumber(ctx, result)
},
producerCorrection,
func(_ context.Context, candidate *int) (int, error) {
if candidate != nil {
t.Fatal("failed execution recovered an earlier candidate")
}
return -1, nil
})
want := 0
if failCorrection {
want = 1
}
if err != nil || outcome.Disposition != contracts.ProducerFallback || calls != want+1 || interpreted != want || assessed != want {
t.Fatalf("provider failure = %#v %v, calls %d/%d/%d", outcome, err, calls, interpreted, assessed)
}
}
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
return nil, failure
}))
_, _, err := RunProducer(context.Background(), engine, contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict}, Assess: acceptedProducerAssessment}, nativeProducerInitial(), interpretProducerNumber, nil, nil)
if !errors.Is(err, failure) {
t.Fatalf("native failure cause was lost: %v", err)
}
}
func TestNativeProducerUnavailableAssessmentSuppliesNoCorrection(t *testing.T) {
calls, corrections := 0, 0
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil
}))
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 5},
Assess: func(context.Context, int) (contracts.CandidateAssessment, error) {
return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{Decision: contracts.AssessmentUnavailable, Failure: errors.New("provider unavailable")}}}, nil
},
}
_, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber,
func(ctx context.Context, guidance []string) (promptkit.RunRequest, error) {
corrections++
return producerCorrection(ctx, guidance)
},
func(context.Context, *int) (int, error) { return -1, nil })
if err != nil || outcome.Disposition != contracts.ProducerFallback || calls != 1 || corrections != 0 {
t.Fatalf("unavailable assessment = %#v %v, calls %d/%d", outcome, err, calls, corrections)
}
}
func TestNativeProducerPreservesConversationAndExplicitSelections(t *testing.T) {
for _, explicit := range []bool{false, true} {
var requests []promptkit.GenerateRequest
var profiles []string
initialPrepared, correctionPrepared, observed := 0, 0, 0
initialObserved, correctionObserved := 0, 0
engine := nativeProducerEngine(t, producerClientFunc(func(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
requests = append(requests, request)
return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil
}))
initial := nativeProducerInitial()
initial.SessionID = "source-session"
if explicit {
initial.ProfileID = "chosen"
}
initial.OnPrepared = func(context.Context, promptkit.PreparedRun) { initialPrepared++ }
initial.Observer = func(event promptkit.ExecutionEvent) {
if event.Kind == promptkit.ExecutionFinished {
initialObserved++
}
}
overrideModel := "override-model"
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 2}, Assess: rejectedProducerAssessment,
Execution: contracts.ExecutionOptions{Execution: &promptkit.ExecutionTargetOverride{Model: overrideModel},
Observer: func(event promptkit.ExecutionEvent) {
if event.Kind == promptkit.ExecutionFinished {
observed++
}
},
OnPrepared: func(_ context.Context, details promptkit.PreparedRun) {
profiles = append(profiles, details.SelectedProfileID)
},
},
}
_, outcome, err := RunProducer(context.Background(), engine, operation, initial, interpretProducerNumber,
func(ctx context.Context, guidance []string) (promptkit.RunRequest, error) {
if !reflect.DeepEqual(guidance, []string{"Return value one."}) {
t.Fatalf("unexpected correction guidance: %#v", guidance)
}
request, err := producerCorrection(ctx, guidance)
request.OnPrepared = func(context.Context, promptkit.PreparedRun) { correctionPrepared++ }
request.Observer = func(event promptkit.ExecutionEvent) {
if event.Kind == promptkit.ExecutionFinished {
correctionObserved++
}
}
return request, err
}, func(context.Context, *int) (int, error) { return -1, nil })
if err != nil || outcome.Disposition != contracts.ProducerFallback || len(requests) != 3 || observed != 3 || initialPrepared != 1 || correctionPrepared != 2 {
t.Fatalf("native continuation = %#v %v, counts %d/%d/%d/%d", outcome, err, len(requests), observed, initialPrepared, correctionPrepared)
}
if initialObserved != 1 || correctionObserved != 2 {
t.Fatalf("existing request observers were lost or duplicated: %d/%d", initialObserved, correctionObserved)
}
wantProfiles := []string{"base", "correction-default", "correction-default"}
if explicit {
wantProfiles = []string{"chosen", "chosen", "chosen"}
}
if !reflect.DeepEqual(profiles, wantProfiles) {
t.Fatalf("profiles = %#v, want %#v", profiles, wantProfiles)
}
for i, request := range requests {
if request.Target.Model != overrideModel || request.Prompt.SessionID != initial.SessionID {
t.Fatalf("selection lost on turn %d: %#v", i, request)
}
var content []string
for _, message := range request.Prompt.Messages {
content = append(content, message.Content)
}
joined := strings.Join(content, "\n")
if strings.Count(joined, "fixed source") != 1 || strings.Count(joined, "Keep enduring constraints.") != 1 || len(request.Prompt.Messages) != 2+2*i {
t.Fatalf("conversation replayed or lost history on turn %d: %#v", i, content)
}
}
if initial.CaptureExchange || initial.Execution != nil || initial.ProfileID == "base" {
t.Fatal("helper mutated the caller's initial request")
}
}
}
func TestNativeProducerCancellationDoesNotDeliverOrFallback(t *testing.T) {
for _, moment := range []string{"before initial", "after initial execution", "inside interpretation", "inside assessment", "before continuation delivery"} {
t.Run(moment, func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
calls, interpreted, assessed, finished, fallbacks := 0, 0, 0, 0, 0
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil
}))
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 2},
Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) {
assessed++
if moment == "inside assessment" {
cancel()
return acceptedProducerAssessment(ctx, value)
}
return rejectedProducerAssessment(ctx, value)
},
Execution: contracts.ExecutionOptions{Observer: func(event promptkit.ExecutionEvent) {
if event.Kind == promptkit.ExecutionFinished {
finished++
if (moment == "before continuation delivery" && finished == 2) || moment == "after initial execution" {
cancel()
}
}
}},
}
if moment == "before initial" {
cancel()
}
_, outcome, err := RunProducer(ctx, engine, operation, nativeProducerInitial(),
func(ctx context.Context, result *promptkit.RunResult) (*int, []string, error) {
interpreted++
if moment == "inside interpretation" {
cancel()
}
return interpretProducerNumber(ctx, result)
},
producerCorrection,
func(context.Context, *int) (int, error) { fallbacks++; return -1, nil })
if !errors.Is(err, context.Canceled) || outcome.Disposition != "" || fallbacks != 0 {
t.Fatalf("cancellation = %#v %v, fallback calls %d", outcome, err, fallbacks)
}
switch moment {
case "before initial":
if calls != 0 || interpreted != 0 || assessed != 0 {
t.Fatalf("work started after cancellation: %d/%d/%d", calls, interpreted, assessed)
}
case "inside assessment":
if calls != 1 || interpreted != 1 || assessed != 1 {
t.Fatalf("canceled assessment repeated: %d/%d/%d", calls, interpreted, assessed)
}
case "after initial execution":
if calls != 1 || interpreted != 0 || assessed != 0 {
t.Fatalf("canceled native result was interpreted: %d/%d/%d", calls, interpreted, assessed)
}
case "inside interpretation":
if calls != 1 || interpreted != 1 || assessed != 0 {
t.Fatalf("assessment started after cancellation: %d/%d/%d", calls, interpreted, assessed)
}
default:
if calls != 2 || interpreted != 1 || assessed != 1 {
t.Fatalf("undelivered response assessed: %d/%d/%d", calls, interpreted, assessed)
}
}
})
}
}
func TestNativeProducerOperationTimeoutKeepsCallerActive(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
for _, continuation := range []bool{false, true} {
for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} {
calls, fallbacks := 0, 0
engine := nativeProducerEngine(t, producerClientFunc(func(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
if continuation && calls == 1 {
return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil
}
<-ctx.Done()
return nil, ctx.Err()
}))
initial := nativeProducerInitial()
initial.OperationTimeout = time.Second
parent := context.Background()
_, outcome, err := RunProducer(parent, engine, contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: mode, MaxAdditionalTurns: 2}, Assess: rejectedProducerAssessment},
initial, interpretProducerNumber,
func(ctx context.Context, guidance []string) (promptkit.RunRequest, error) {
request, err := producerCorrection(ctx, guidance)
request.OperationTimeout = time.Second
return request, err
},
func(_ context.Context, final *int) (int, error) {
fallbacks++
if final != nil {
t.Fatal("timeout recovered an earlier candidate")
}
return -1, nil
})
if parent.Err() != nil {
t.Fatalf("native timeout canceled caller: %v", parent.Err())
}
if mode == contracts.ModeStrict {
if !errors.Is(err, promptkit.ErrOperationTimeout) || !errors.Is(err, context.DeadlineExceeded) || outcome.Disposition != "" || fallbacks != 0 {
t.Fatalf("strict timeout = %#v %v, fallback calls %d", outcome, err, fallbacks)
}
} else if err != nil || outcome.Disposition != contracts.ProducerFallback || fallbacks != 1 {
t.Fatalf("permissive timeout = %#v %v, fallback calls %d", outcome, err, fallbacks)
}
}
}
})
}
func TestNativeProducerInterpretationFindingsUseNativeCorrection(t *testing.T) {
for _, usable := range []bool{false, true} {
calls, interpreted, assessed := 0, 0, 0
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
}))
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, MaxAdditionalTurns: 1}, ReuseEligible: true,
Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) {
assessed++
return acceptedProducerAssessment(ctx, value)
},
}
value, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(),
func(ctx context.Context, result *promptkit.RunResult) (*int, []string, error) {
interpreted++
candidate, _, err := interpretProducerNumber(ctx, result)
if interpreted == 1 {
if !usable {
candidate = nil
}
return candidate, []string{"Use the supplied candidate handles."}, err
}
return candidate, nil, err
},
func(ctx context.Context, guidance []string) (promptkit.RunRequest, error) {
if !reflect.DeepEqual(guidance, []string{"Use the supplied candidate handles."}) {
t.Fatalf("module findings lost: %#v", guidance)
}
return producerCorrection(ctx, guidance)
}, nil)
wantAssessed := 1
if usable {
wantAssessed = 2
}
if err != nil || outcome.Disposition != contracts.ProducerAccepted || value != 1 || calls != 2 || interpreted != 2 || assessed != wantAssessed {
t.Fatalf("interpretation correction = %d %#v %v, calls %d/%d/%d", value, outcome, err, calls, interpreted, assessed)
}
}
}
func TestNativeProducerAssessmentContractFailureDoesNotFallback(t *testing.T) {
failure := errors.New("assessment framework failure")
fallbacks, calls := 0, 0
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
}))
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 2}, Assess: func(context.Context, int) (contracts.CandidateAssessment, error) {
return contracts.CandidateAssessment{}, failure
}}
_, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, producerCorrection, func(context.Context, *int) (int, error) { fallbacks++; return -1, nil })
if !errors.Is(err, failure) || outcome.Disposition != "" || calls != 1 || fallbacks != 0 {
t.Fatalf("framework failure = %#v %v, calls %d/%d", outcome, err, calls, fallbacks)
}
}