519 lines
22 KiB
Go
519 lines
22 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
"testing/synctest"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
"go.maximumdirect.net/configkit"
|
|
"go.maximumdirect.net/promptkit"
|
|
)
|
|
|
|
type producerClientFunc func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error)
|
|
|
|
func (f producerClientFunc) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
return f(ctx, request)
|
|
}
|
|
|
|
func nativeProducerEngine(t *testing.T, client promptkit.LLMClient) *promptkit.Engine {
|
|
t.Helper()
|
|
settings := `profiles:
|
|
- id: base
|
|
endpoint: https://example.invalid/v1
|
|
model: initial-model
|
|
api_key_required: false
|
|
- id: correction-default
|
|
endpoint: https://example.invalid/v1
|
|
model: correction-model
|
|
api_key_required: false
|
|
- id: chosen
|
|
endpoint: https://example.invalid/v1
|
|
model: chosen-model
|
|
api_key_required: false
|
|
prompts:
|
|
- id: initial
|
|
default_profile: base
|
|
inputs: [{name: source, required: true}]
|
|
messages:
|
|
- {role: system, content: 'Keep enduring constraints.'}
|
|
- {role: user, content: 'Source: {{input "source"}}'}
|
|
output: {format: json, validation_mode: json, repair_attempts: 0}
|
|
- id: correction
|
|
default_profile: correction-default
|
|
inputs: [{name: guidance, required: true}]
|
|
messages:
|
|
- {role: user, content: 'Correct: {{input "guidance"}}'}
|
|
output: {format: json, validation_mode: json, repair_attempts: 0}
|
|
`
|
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{promptkit.SettingsInput(configkit.Bytes("producer fixture", []byte(settings)))}}},
|
|
}}, promptkit.WithBundledOpenRouter(false), promptkit.WithLLMClient(client))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return engine
|
|
}
|
|
|
|
func nativeProducerInitial() promptkit.RunRequest {
|
|
return promptkit.RunRequest{PromptID: "initial", Inputs: map[string]promptkit.ArtifactRef{"source": promptkit.Inline("fixed source")}}
|
|
}
|
|
|
|
func interpretProducerNumber(_ context.Context, result *promptkit.RunResult) (*int, []string, error) {
|
|
var candidate struct {
|
|
Value int `json:"value"`
|
|
}
|
|
if err := json.Unmarshal([]byte(result.RawOutput), &candidate); err != nil {
|
|
return nil, nil, err
|
|
}
|
|
return &candidate.Value, nil, nil
|
|
}
|
|
|
|
func producerCorrection(_ context.Context, guidance []string) (promptkit.RunRequest, error) {
|
|
return promptkit.RunRequest{PromptID: "correction", Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline(strings.Join(guidance, "\n"))}}, nil
|
|
}
|
|
|
|
func acceptedProducerAssessment(context.Context, int) (contracts.CandidateAssessment, error) {
|
|
return contracts.CandidateAssessment{}, nil
|
|
}
|
|
|
|
func rejectedProducerAssessment(context.Context, int) (contracts.CandidateAssessment, error) {
|
|
return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{ValidatorName: "domain", Decision: contracts.AssessmentRejected, Result: contracts.ValidationResult{ReasonCode: "not_one", Message: "operator diagnostic", CorrectionGuidance: "Return value one."}}}}, nil
|
|
}
|
|
|
|
func TestNativeProducerAcceptsOnlyAssessedCandidates(t *testing.T) {
|
|
for _, test := range []struct {
|
|
name string
|
|
values []int
|
|
turns int
|
|
mode contracts.ExecutionMode
|
|
wantAccepted bool
|
|
}{
|
|
{"initial acceptance", []int{1}, 3, contracts.ModeStrict, true},
|
|
{"zero turns", []int{0}, 0, contracts.ModeStrict, false},
|
|
{"correction acceptance", []int{0, 1}, 3, contracts.ModeStrict, true},
|
|
{"final turn acceptance", []int{0, 0, 1}, 2, contracts.ModeStrict, true},
|
|
{"exhaustion", []int{0, 0, 0}, 2, contracts.ModeStrict, false},
|
|
{"permissive exhaustion", []int{0, 0, 0}, 2, contracts.ModePermissive, false},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
calls, assessments, interpreted, fallbacks := 0, 0, 0, 0
|
|
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
if calls >= len(test.values) {
|
|
return nil, errors.New("unexpected extra provider call")
|
|
}
|
|
value := test.values[calls]
|
|
calls++
|
|
return &promptkit.GenerateResponse{Content: fmt.Sprintf(`{"value":%d}`, value)}, nil
|
|
}))
|
|
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: test.mode, MaxAdditionalTurns: test.turns}, ReuseEligible: true,
|
|
Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) {
|
|
assessments++
|
|
if value == 1 {
|
|
return acceptedProducerAssessment(ctx, value)
|
|
}
|
|
return rejectedProducerAssessment(ctx, value)
|
|
},
|
|
}
|
|
value, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(),
|
|
func(ctx context.Context, result *promptkit.RunResult) (*int, []string, error) {
|
|
interpreted++
|
|
return interpretProducerNumber(ctx, result)
|
|
},
|
|
producerCorrection,
|
|
func(_ context.Context, final *int) (int, error) {
|
|
fallbacks++
|
|
if final == nil || *final != test.values[len(test.values)-1] {
|
|
t.Fatalf("fallback did not receive the final usable candidate: %v", final)
|
|
}
|
|
return -1, nil
|
|
})
|
|
if calls != len(test.values) || assessments != calls || interpreted != calls {
|
|
t.Fatalf("provider=%d interpretation=%d assessment=%d; want %d each", calls, interpreted, assessments, len(test.values))
|
|
}
|
|
switch {
|
|
case test.wantAccepted:
|
|
if err != nil || value != 1 || outcome.Disposition != contracts.ProducerAccepted || !outcome.ReuseEligible || fallbacks != 0 {
|
|
t.Fatalf("accepted = %d %#v %v, fallback calls %d", value, outcome, err, fallbacks)
|
|
}
|
|
case test.mode == contracts.ModePermissive:
|
|
if err != nil || value != -1 || outcome.Disposition != contracts.ProducerFallback || outcome.ReuseEligible || fallbacks != 1 {
|
|
t.Fatalf("fallback = %d %#v %v, calls %d", value, outcome, err, fallbacks)
|
|
}
|
|
default:
|
|
if err == nil || outcome.Disposition != "" || fallbacks != 0 {
|
|
t.Fatalf("strict rejection = %d %#v %v, fallback calls %d", value, outcome, err, fallbacks)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestNativeProducerDoesNotInterpretFailedNativeValidation(t *testing.T) {
|
|
for _, failCorrection := range []bool{false, true} {
|
|
calls, interpretations, assessments, corrections, fallbacks := 0, 0, 0, 0, 0
|
|
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
calls++
|
|
if failCorrection && calls == 1 {
|
|
return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil
|
|
}
|
|
return &promptkit.GenerateResponse{Content: "not JSON"}, nil
|
|
}))
|
|
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 2},
|
|
Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) {
|
|
assessments++
|
|
return rejectedProducerAssessment(ctx, value)
|
|
},
|
|
}
|
|
_, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(),
|
|
func(ctx context.Context, result *promptkit.RunResult) (*int, []string, error) {
|
|
interpretations++
|
|
return interpretProducerNumber(ctx, result)
|
|
},
|
|
func(ctx context.Context, guidance []string) (promptkit.RunRequest, error) {
|
|
corrections++
|
|
return producerCorrection(ctx, guidance)
|
|
},
|
|
func(_ context.Context, candidate *int) (int, error) {
|
|
fallbacks++
|
|
if candidate != nil {
|
|
t.Fatal("structurally failed response recovered a prior candidate")
|
|
}
|
|
return -1, nil
|
|
})
|
|
want := 0
|
|
if failCorrection {
|
|
want = 1
|
|
}
|
|
if err != nil || outcome.Disposition != contracts.ProducerFallback || calls != want+1 || interpretations != want || assessments != want || corrections != want || fallbacks != 1 {
|
|
t.Fatalf("failed native status: %#v %v, calls %d/%d/%d/%d/%d", outcome, err, calls, interpretations, assessments, corrections, fallbacks)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestNativeProducerFailureDoesNotRecoverEarlierCandidate(t *testing.T) {
|
|
failure := errors.New("provider unavailable")
|
|
for _, failCorrection := range []bool{false, true} {
|
|
calls, interpreted, assessed := 0, 0, 0
|
|
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
calls++
|
|
if failCorrection && calls == 1 {
|
|
return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil
|
|
}
|
|
return nil, failure
|
|
}))
|
|
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 2},
|
|
Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) {
|
|
assessed++
|
|
return rejectedProducerAssessment(ctx, value)
|
|
},
|
|
}
|
|
_, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(),
|
|
func(ctx context.Context, result *promptkit.RunResult) (*int, []string, error) {
|
|
interpreted++
|
|
return interpretProducerNumber(ctx, result)
|
|
},
|
|
producerCorrection,
|
|
func(_ context.Context, candidate *int) (int, error) {
|
|
if candidate != nil {
|
|
t.Fatal("failed execution recovered an earlier candidate")
|
|
}
|
|
return -1, nil
|
|
})
|
|
want := 0
|
|
if failCorrection {
|
|
want = 1
|
|
}
|
|
if err != nil || outcome.Disposition != contracts.ProducerFallback || calls != want+1 || interpreted != want || assessed != want {
|
|
t.Fatalf("provider failure = %#v %v, calls %d/%d/%d", outcome, err, calls, interpreted, assessed)
|
|
}
|
|
}
|
|
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
return nil, failure
|
|
}))
|
|
_, _, err := RunProducer(context.Background(), engine, contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict}, Assess: acceptedProducerAssessment}, nativeProducerInitial(), interpretProducerNumber, nil, nil)
|
|
if !errors.Is(err, failure) {
|
|
t.Fatalf("native failure cause was lost: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestNativeProducerUnavailableAssessmentSuppliesNoCorrection(t *testing.T) {
|
|
calls, corrections := 0, 0
|
|
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
calls++
|
|
return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil
|
|
}))
|
|
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 5},
|
|
Assess: func(context.Context, int) (contracts.CandidateAssessment, error) {
|
|
return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{Decision: contracts.AssessmentUnavailable, Failure: errors.New("provider unavailable")}}}, nil
|
|
},
|
|
}
|
|
_, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber,
|
|
func(ctx context.Context, guidance []string) (promptkit.RunRequest, error) {
|
|
corrections++
|
|
return producerCorrection(ctx, guidance)
|
|
},
|
|
func(context.Context, *int) (int, error) { return -1, nil })
|
|
if err != nil || outcome.Disposition != contracts.ProducerFallback || calls != 1 || corrections != 0 {
|
|
t.Fatalf("unavailable assessment = %#v %v, calls %d/%d", outcome, err, calls, corrections)
|
|
}
|
|
}
|
|
|
|
func TestNativeProducerPreservesConversationAndExplicitSelections(t *testing.T) {
|
|
for _, explicit := range []bool{false, true} {
|
|
var requests []promptkit.GenerateRequest
|
|
var profiles []string
|
|
initialPrepared, correctionPrepared, observed := 0, 0, 0
|
|
initialObserved, correctionObserved := 0, 0
|
|
engine := nativeProducerEngine(t, producerClientFunc(func(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
requests = append(requests, request)
|
|
return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil
|
|
}))
|
|
initial := nativeProducerInitial()
|
|
initial.SessionID = "source-session"
|
|
if explicit {
|
|
initial.ProfileID = "chosen"
|
|
}
|
|
initial.OnPrepared = func(context.Context, promptkit.PreparedRun) { initialPrepared++ }
|
|
initial.Observer = func(event promptkit.ExecutionEvent) {
|
|
if event.Kind == promptkit.ExecutionFinished {
|
|
initialObserved++
|
|
}
|
|
}
|
|
overrideModel := "override-model"
|
|
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 2}, Assess: rejectedProducerAssessment,
|
|
Execution: contracts.ExecutionOptions{Execution: &promptkit.ExecutionTargetOverride{Model: overrideModel},
|
|
Observer: func(event promptkit.ExecutionEvent) {
|
|
if event.Kind == promptkit.ExecutionFinished {
|
|
observed++
|
|
}
|
|
},
|
|
OnPrepared: func(_ context.Context, details promptkit.PreparedRun) {
|
|
profiles = append(profiles, details.SelectedProfileID)
|
|
},
|
|
},
|
|
}
|
|
_, outcome, err := RunProducer(context.Background(), engine, operation, initial, interpretProducerNumber,
|
|
func(ctx context.Context, guidance []string) (promptkit.RunRequest, error) {
|
|
if !reflect.DeepEqual(guidance, []string{"Return value one."}) {
|
|
t.Fatalf("unexpected correction guidance: %#v", guidance)
|
|
}
|
|
request, err := producerCorrection(ctx, guidance)
|
|
request.OnPrepared = func(context.Context, promptkit.PreparedRun) { correctionPrepared++ }
|
|
request.Observer = func(event promptkit.ExecutionEvent) {
|
|
if event.Kind == promptkit.ExecutionFinished {
|
|
correctionObserved++
|
|
}
|
|
}
|
|
return request, err
|
|
}, func(context.Context, *int) (int, error) { return -1, nil })
|
|
if err != nil || outcome.Disposition != contracts.ProducerFallback || len(requests) != 3 || observed != 3 || initialPrepared != 1 || correctionPrepared != 2 {
|
|
t.Fatalf("native continuation = %#v %v, counts %d/%d/%d/%d", outcome, err, len(requests), observed, initialPrepared, correctionPrepared)
|
|
}
|
|
if initialObserved != 1 || correctionObserved != 2 {
|
|
t.Fatalf("existing request observers were lost or duplicated: %d/%d", initialObserved, correctionObserved)
|
|
}
|
|
wantProfiles := []string{"base", "correction-default", "correction-default"}
|
|
if explicit {
|
|
wantProfiles = []string{"chosen", "chosen", "chosen"}
|
|
}
|
|
if !reflect.DeepEqual(profiles, wantProfiles) {
|
|
t.Fatalf("profiles = %#v, want %#v", profiles, wantProfiles)
|
|
}
|
|
for i, request := range requests {
|
|
if request.Target.Model != overrideModel || request.Prompt.SessionID != initial.SessionID {
|
|
t.Fatalf("selection lost on turn %d: %#v", i, request)
|
|
}
|
|
var content []string
|
|
for _, message := range request.Prompt.Messages {
|
|
content = append(content, message.Content)
|
|
}
|
|
joined := strings.Join(content, "\n")
|
|
if strings.Count(joined, "fixed source") != 1 || strings.Count(joined, "Keep enduring constraints.") != 1 || len(request.Prompt.Messages) != 2+2*i {
|
|
t.Fatalf("conversation replayed or lost history on turn %d: %#v", i, content)
|
|
}
|
|
}
|
|
if initial.CaptureExchange || initial.Execution != nil || initial.ProfileID == "base" {
|
|
t.Fatal("helper mutated the caller's initial request")
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestNativeProducerCancellationDoesNotDeliverOrFallback(t *testing.T) {
|
|
for _, moment := range []string{"before initial", "after initial execution", "inside interpretation", "inside assessment", "before continuation delivery"} {
|
|
t.Run(moment, func(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
calls, interpreted, assessed, finished, fallbacks := 0, 0, 0, 0, 0
|
|
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
calls++
|
|
return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil
|
|
}))
|
|
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 2},
|
|
Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) {
|
|
assessed++
|
|
if moment == "inside assessment" {
|
|
cancel()
|
|
return acceptedProducerAssessment(ctx, value)
|
|
}
|
|
return rejectedProducerAssessment(ctx, value)
|
|
},
|
|
Execution: contracts.ExecutionOptions{Observer: func(event promptkit.ExecutionEvent) {
|
|
if event.Kind == promptkit.ExecutionFinished {
|
|
finished++
|
|
if (moment == "before continuation delivery" && finished == 2) || moment == "after initial execution" {
|
|
cancel()
|
|
}
|
|
}
|
|
}},
|
|
}
|
|
if moment == "before initial" {
|
|
cancel()
|
|
}
|
|
_, outcome, err := RunProducer(ctx, engine, operation, nativeProducerInitial(),
|
|
func(ctx context.Context, result *promptkit.RunResult) (*int, []string, error) {
|
|
interpreted++
|
|
if moment == "inside interpretation" {
|
|
cancel()
|
|
}
|
|
return interpretProducerNumber(ctx, result)
|
|
},
|
|
producerCorrection,
|
|
func(context.Context, *int) (int, error) { fallbacks++; return -1, nil })
|
|
if !errors.Is(err, context.Canceled) || outcome.Disposition != "" || fallbacks != 0 {
|
|
t.Fatalf("cancellation = %#v %v, fallback calls %d", outcome, err, fallbacks)
|
|
}
|
|
switch moment {
|
|
case "before initial":
|
|
if calls != 0 || interpreted != 0 || assessed != 0 {
|
|
t.Fatalf("work started after cancellation: %d/%d/%d", calls, interpreted, assessed)
|
|
}
|
|
case "inside assessment":
|
|
if calls != 1 || interpreted != 1 || assessed != 1 {
|
|
t.Fatalf("canceled assessment repeated: %d/%d/%d", calls, interpreted, assessed)
|
|
}
|
|
case "after initial execution":
|
|
if calls != 1 || interpreted != 0 || assessed != 0 {
|
|
t.Fatalf("canceled native result was interpreted: %d/%d/%d", calls, interpreted, assessed)
|
|
}
|
|
case "inside interpretation":
|
|
if calls != 1 || interpreted != 1 || assessed != 0 {
|
|
t.Fatalf("assessment started after cancellation: %d/%d/%d", calls, interpreted, assessed)
|
|
}
|
|
default:
|
|
if calls != 2 || interpreted != 1 || assessed != 1 {
|
|
t.Fatalf("undelivered response assessed: %d/%d/%d", calls, interpreted, assessed)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestNativeProducerOperationTimeoutKeepsCallerActive(t *testing.T) {
|
|
synctest.Test(t, func(t *testing.T) {
|
|
for _, continuation := range []bool{false, true} {
|
|
for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} {
|
|
calls, fallbacks := 0, 0
|
|
engine := nativeProducerEngine(t, producerClientFunc(func(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
calls++
|
|
if continuation && calls == 1 {
|
|
return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil
|
|
}
|
|
<-ctx.Done()
|
|
return nil, ctx.Err()
|
|
}))
|
|
initial := nativeProducerInitial()
|
|
initial.OperationTimeout = time.Second
|
|
parent := context.Background()
|
|
_, outcome, err := RunProducer(parent, engine, contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: mode, MaxAdditionalTurns: 2}, Assess: rejectedProducerAssessment},
|
|
initial, interpretProducerNumber,
|
|
func(ctx context.Context, guidance []string) (promptkit.RunRequest, error) {
|
|
request, err := producerCorrection(ctx, guidance)
|
|
request.OperationTimeout = time.Second
|
|
return request, err
|
|
},
|
|
func(_ context.Context, final *int) (int, error) {
|
|
fallbacks++
|
|
if final != nil {
|
|
t.Fatal("timeout recovered an earlier candidate")
|
|
}
|
|
return -1, nil
|
|
})
|
|
if parent.Err() != nil {
|
|
t.Fatalf("native timeout canceled caller: %v", parent.Err())
|
|
}
|
|
if mode == contracts.ModeStrict {
|
|
if !errors.Is(err, promptkit.ErrOperationTimeout) || !errors.Is(err, context.DeadlineExceeded) || outcome.Disposition != "" || fallbacks != 0 {
|
|
t.Fatalf("strict timeout = %#v %v, fallback calls %d", outcome, err, fallbacks)
|
|
}
|
|
} else if err != nil || outcome.Disposition != contracts.ProducerFallback || fallbacks != 1 {
|
|
t.Fatalf("permissive timeout = %#v %v, fallback calls %d", outcome, err, fallbacks)
|
|
}
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestNativeProducerInterpretationFindingsUseNativeCorrection(t *testing.T) {
|
|
for _, usable := range []bool{false, true} {
|
|
calls, interpreted, assessed := 0, 0, 0
|
|
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
calls++
|
|
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
|
|
}))
|
|
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, MaxAdditionalTurns: 1}, ReuseEligible: true,
|
|
Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) {
|
|
assessed++
|
|
return acceptedProducerAssessment(ctx, value)
|
|
},
|
|
}
|
|
value, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(),
|
|
func(ctx context.Context, result *promptkit.RunResult) (*int, []string, error) {
|
|
interpreted++
|
|
candidate, _, err := interpretProducerNumber(ctx, result)
|
|
if interpreted == 1 {
|
|
if !usable {
|
|
candidate = nil
|
|
}
|
|
return candidate, []string{"Use the supplied candidate handles."}, err
|
|
}
|
|
return candidate, nil, err
|
|
},
|
|
func(ctx context.Context, guidance []string) (promptkit.RunRequest, error) {
|
|
if !reflect.DeepEqual(guidance, []string{"Use the supplied candidate handles."}) {
|
|
t.Fatalf("module findings lost: %#v", guidance)
|
|
}
|
|
return producerCorrection(ctx, guidance)
|
|
}, nil)
|
|
wantAssessed := 1
|
|
if usable {
|
|
wantAssessed = 2
|
|
}
|
|
if err != nil || outcome.Disposition != contracts.ProducerAccepted || value != 1 || calls != 2 || interpreted != 2 || assessed != wantAssessed {
|
|
t.Fatalf("interpretation correction = %d %#v %v, calls %d/%d/%d", value, outcome, err, calls, interpreted, assessed)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestNativeProducerAssessmentContractFailureDoesNotFallback(t *testing.T) {
|
|
failure := errors.New("assessment framework failure")
|
|
fallbacks, calls := 0, 0
|
|
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
calls++
|
|
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
|
|
}))
|
|
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 2}, Assess: func(context.Context, int) (contracts.CandidateAssessment, error) {
|
|
return contracts.CandidateAssessment{}, failure
|
|
}}
|
|
_, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, producerCorrection, func(context.Context, *int) (int, error) { fallbacks++; return -1, nil })
|
|
if !errors.Is(err, failure) || outcome.Disposition != "" || calls != 1 || fallbacks != 0 {
|
|
t.Fatalf("framework failure = %#v %v, calls %d/%d", outcome, err, calls, fallbacks)
|
|
}
|
|
}
|