From 1c075bd510fd16cf5654b36d77a548f71ee0d91c Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Thu, 5 Nov 2026 13:48:16 +0000 Subject: [PATCH] Identify built-in generation preparation failures --- docs/roadmap/implementation.md | 5 ++++ internal/generation/coordinator.go | 8 +++++- internal/generation/evidence.go | 2 ++ internal/usecase/prepared_execution_test.go | 2 +- internal/usecase/runner.go | 4 +++ internal/usecase/runner_test.go | 9 +++---- internal/usecase/smoke_tester.go | 3 +++ preparation_error_contract_test.go | 29 +++++++++++++++++++++ 8 files changed, 54 insertions(+), 8 deletions(-) create mode 100644 preparation_error_contract_test.go diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index f234e69..003822c 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -198,6 +198,11 @@ fails with the public invalid-request category and zero dispatch. Existing provider rejection, missing-key, capacity, injected, and cancellation tests retain their respective outcomes. +Completion: built-in preparation failures carry direct error and evidence +markers; diagnostics return invalid-request errors before dispatch. Generation, +use-case, and root tests pass on Go 1.27.3, including the previously deferred +injected-client classification cases. + ## Stage 7: Bound Authentication Recognition And Report Overflow **Catalog:** 2. **Files:** current authentication probe, public/domain smoke reasons. diff --git a/internal/generation/coordinator.go b/internal/generation/coordinator.go index e9b9e40..3529872 100644 --- a/internal/generation/coordinator.go +++ b/internal/generation/coordinator.go @@ -85,6 +85,12 @@ func (c *Coordinator) GenerateWithEvidence( return c.generate(ctx, req, true) } +// PreparationError identifies a built-in request preparation failure. +type PreparationError struct{ Cause error } + +func (e *PreparationError) Error() string { return e.Cause.Error() } +func (e *PreparationError) Unwrap() error { return e.Cause } + func (c *Coordinator) generate( ctx context.Context, req domain.GenerateRequest, @@ -95,7 +101,7 @@ func (c *Coordinator) generate( } prepared, err := c.builtIn.PrepareGeneration(req) if err != nil { - return nil, err, GenerationEvidence{} + return nil, &PreparationError{Cause: err}, GenerationEvidence{PreparationFailed: true} } evidence := GenerationEvidence{} diff --git a/internal/generation/evidence.go b/internal/generation/evidence.go index d86d755..ad40bc7 100644 --- a/internal/generation/evidence.go +++ b/internal/generation/evidence.go @@ -10,6 +10,8 @@ import ( // GenerationEvidence contains invocation-local request evidence without raw // provider bodies, headers, credentials, or generated content. type GenerationEvidence struct { + // PreparationFailed identifies a built-in request rejected before attempts. + PreparationFailed bool // InitialEndpoint is the frozen built-in request URL, if prepared. // Injected generation leaves it unknown. InitialEndpoint string diff --git a/internal/usecase/prepared_execution_test.go b/internal/usecase/prepared_execution_test.go index dcd8619..1135564 100644 --- a/internal/usecase/prepared_execution_test.go +++ b/internal/usecase/prepared_execution_test.go @@ -498,7 +498,7 @@ func TestRunnerRunPreparedReleasesAdmissionAcrossExecutionErrors(t *testing.T) { }}, }, repairer: &fakeRepairer{err: repairInvalidRequest}, - wantError: ErrInvalidRequest, + wantError: ErrLLMGenerate, wantSource: repairInvalidRequest, }, { diff --git a/internal/usecase/runner.go b/internal/usecase/runner.go index c31585b..0878797 100644 --- a/internal/usecase/runner.go +++ b/internal/usecase/runner.go @@ -18,6 +18,7 @@ import ( "go.maximumdirect.net/promptkit/internal/capacity" "go.maximumdirect.net/promptkit/internal/defaults" "go.maximumdirect.net/promptkit/internal/domain" + "go.maximumdirect.net/promptkit/internal/generation" "go.maximumdirect.net/promptkit/internal/jsonvalue" "go.maximumdirect.net/promptkit/internal/llm" "go.maximumdirect.net/promptkit/internal/observation" @@ -269,6 +270,9 @@ func (r *Runner) executePreparedRun( } func wrapGenerationError(err error) error { + if _, ok := err.(*generation.PreparationError); ok { + return &OperationError{Category: ErrInvalidRequest, Cause: err} + } return &OperationError{Category: ErrLLMGenerate, Cause: err} } diff --git a/internal/usecase/runner_test.go b/internal/usecase/runner_test.go index 4a8ad3d..a7da313 100644 --- a/internal/usecase/runner_test.go +++ b/internal/usecase/runner_test.go @@ -2074,7 +2074,7 @@ func TestRunnerRunCancellationPreservesGenerationCategory(t *testing.T) { } } -func TestRunnerRunLLMInvalidRequestMapsToUsecaseInvalidRequest(t *testing.T) { +func TestRunnerRunInjectedInvalidRequestRemainsGenerationFailure(t *testing.T) { runner := NewRunner( &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, @@ -2089,11 +2089,8 @@ func TestRunnerRunLLMInvalidRequestMapsToUsecaseInvalidRequest(t *testing.T) { ProfileID: "exec", Inputs: singleInputRef(), }) - if !errors.Is(err, ErrInvalidRequest) { - t.Fatalf("expected ErrInvalidRequest, got %v", err) - } - if errors.Is(err, ErrLLMGenerate) { - t.Fatalf("did not expect ErrLLMGenerate, got %v", err) + if !errors.Is(err, ErrLLMGenerate) || !errors.Is(err, llm.ErrInvalidRequest) { + t.Fatalf("expected generation error retaining cause, got %v", err) } } diff --git a/internal/usecase/smoke_tester.go b/internal/usecase/smoke_tester.go index 90a8206..eb1bd3d 100644 --- a/internal/usecase/smoke_tester.go +++ b/internal/usecase/smoke_tester.go @@ -210,6 +210,9 @@ func (t *SmokeTester) runInference( report.Failure = generation.ClassifyFailure(contextErr) return finish(contextErr) } + if evidence.PreparationFailed && generationErr != nil { + return nil, &OperationError{Category: ErrInvalidRequest, Cause: generationErr} + } if generationErr != nil { report.Failure = generation.ClassifyFailure(generationErr) report.Completed = true diff --git a/preparation_error_contract_test.go b/preparation_error_contract_test.go new file mode 100644 index 0000000..2684c77 --- /dev/null +++ b/preparation_error_contract_test.go @@ -0,0 +1,29 @@ +package promptkit_test + +import ( + "context" + "errors" + "net/http" + "testing" + + "go.maximumdirect.net/promptkit" +) + +func TestBuiltInPreparationFailureIsInvalidRequestWithoutDispatch(t *testing.T) { + calls := 0 + cfg := simpleConfig("hello") + cfg.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { calls++; return nil, errors.New("unexpected dispatch") })} + engine, err := promptkit.NewEngine(cfg) + if err != nil { + t.Fatal(err) + } + override := &promptkit.ExecutionTargetOverride{ExtraParams: map[string]any{"model": "reserved"}} + _, err = engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", Execution: override}) + if !errors.Is(err, promptkit.ErrInvalidRequest) || calls != 0 { + t.Fatalf("run error = %v, calls = %d", err, calls) + } + report, err := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestTextInference, ProfileID: "profile", Execution: override}) + if !errors.Is(err, promptkit.ErrInvalidRequest) || report != nil || calls != 0 { + t.Fatalf("smoke = %+v, %v, calls = %d", report, err, calls) + } +}