From f5c1c69ccbb51d5a2be0d60d1ffb3be4833160ec Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Mon, 12 Oct 2026 22:15:06 +0000 Subject: [PATCH] Verify native PromptKit request validation --- docs/roadmap/implementation.md | 13 +- .../ckprompt/validation_adapters_test.go | 233 ++++++++++++++++++ 2 files changed, 245 insertions(+), 1 deletion(-) diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 56cd00f..9e9c190 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1,6 +1,6 @@ # PromptKit request validation implementation plan -Status: Stages 1 and 2 complete; implementation is ongoing. +Status: Stages 1–3 complete; implementation is ongoing. ## Authority, scope, and baseline @@ -196,6 +196,17 @@ selected builder-before-loader lifecycle. **Complete when:** the adapter's native authority, builder forwarding, failure handling, and cancellation boundaries have meaningful public-boundary evidence. +**Completed:** 2026-10-12. Added real-engine adapter coverage for native success, +missing required input, invalid profile, and an unloadable file reference. The +builder test supplies a required inline artifact and verifies context, selection, +typed input, and state reach both callbacks; validation succeeds without a model +call. Tests cover nil engines, retained `*configkit.LoadError` diagnostics and +identity, cancellation before and after building, cancellation during and after +loading, and PromptKit's native artifact-load category when an artifact reader +returns cancellation. Focused adapter tests and +`GOTOOLCHAIN=local go test -count=1 ./...` passed from +`integrations/ckprompt`. + ## Stage 4: Implement the validate command and requirements group composition **Primary files:** `integrations/ckprompt/command.go`, a new private diff --git a/integrations/ckprompt/validation_adapters_test.go b/integrations/ckprompt/validation_adapters_test.go index ba693f8..754e397 100644 --- a/integrations/ckprompt/validation_adapters_test.go +++ b/integrations/ckprompt/validation_adapters_test.go @@ -5,12 +5,45 @@ import ( "errors" "strings" "testing" + "testing/fstest" commandkit "gitea.maximumdirect.net/eric/CommandKit" "gitea.maximumdirect.net/eric/CommandKit/integrations/ckconfig" + configkit "gitea.maximumdirect.net/eric/ConfigKit" promptkit "gitea.maximumdirect.net/eric/promptkit" ) +type validationFailOnGenerate struct{ calls int } + +func (client *validationFailOnGenerate) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + client.calls++ + return nil, errors.New("request validation must not generate") +} + +type validationCancelArtifactReader struct{ cancel context.CancelFunc } + +func (reader validationCancelArtifactReader) Read(context.Context, promptkit.ArtifactRef) (*promptkit.Artifact, error) { + reader.cancel() + return nil, context.Canceled +} + +func validationTestEngine(t *testing.T, configuration string, options ...promptkit.Option) *promptkit.Engine { + t.Helper() + inputs, err := promptkit.FilesystemInputs("request validation fixture", fstest.MapFS{ + "config/prompt.yml": &fstest.MapFile{Data: []byte(configuration)}, + }, "config") + if err != nil { + t.Fatal(err) + } + engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ + Underlays: []configkit.Underlay{{Inputs: inputs}}, + }}, options...) + if err != nil { + t.Fatalf("construct PromptKit validation fixture: %v", err) + } + return engine +} + func TestWithRequestValidationPreservesSuiteAndValidatesSeedWithoutBuilder(t *testing.T) { type contextKey struct{} type input struct{ value string } @@ -98,3 +131,203 @@ func TestWithRequestValidationRetainsNilLoaderIssueIndependently(t *testing.T) { t.Fatalf("installation did not report retained issue after capabilities were cleared: %v", err) } } + +func TestWithRequestValidationDelegatesNativeOutcomesWithoutModelCalls(t *testing.T) { + client := &validationFailOnGenerate{} + engine := validationTestEngine(t, adapterPrompt, promptkit.WithLLMClient(client)) + selection := ckconfig.Selection{Config: commandkit.Supplied[string]{Value: "selected.yml", Set: true}} + suite := WithRequestValidation(Suite[struct{}, struct{}]{}, func(_ context.Context, gotSelection ckconfig.Selection, _, _ struct{}) (*promptkit.Engine, error) { + if gotSelection != selection { + t.Errorf("loader selection = %#v, want %#v", gotSelection, selection) + } + return engine, nil + }, nil) + + for _, test := range []struct { + name string + request promptkit.RunRequest + want error + }{ + {name: "success", request: promptkit.RunRequest{PromptID: "briefing", ProfileID: "default", Inputs: map[string]promptkit.ArtifactRef{"topic": promptkit.Inline("release planning")}}}, + {name: "missing required input", request: promptkit.RunRequest{PromptID: "briefing", ProfileID: "default"}, want: promptkit.ErrPromptRender}, + {name: "invalid profile selection", request: promptkit.RunRequest{PromptID: "briefing", ProfileID: "missing", Inputs: map[string]promptkit.ArtifactRef{"topic": promptkit.Inline("release planning")}}, want: promptkit.ErrProfileNotFound}, + {name: "unloadable artifact", request: promptkit.RunRequest{PromptID: "briefing", ProfileID: "default", Inputs: map[string]promptkit.ArtifactRef{"topic": promptkit.File("missing-topic.txt")}}, want: promptkit.ErrArtifactLoad}, + } { + t.Run(test.name, func(t *testing.T) { + nativeErr := engine.ValidateRequest(context.Background(), test.request) + acquired, adapterErr := suite.ValidateRequest(context.Background(), ValidationRequest{Selection: selection, Request: test.request}, struct{}{}, struct{}{}) + if test.want == nil { + if nativeErr != nil || adapterErr != nil { + t.Fatalf("native error = %v, adapter error = %v", nativeErr, adapterErr) + } + if acquired.Value != (struct{}{}) { + t.Fatalf("successful acquisition = %#v", acquired) + } + return + } + if !errors.Is(nativeErr, test.want) || !errors.Is(adapterErr, test.want) { + t.Fatalf("native error = %v, adapter error = %v, want both to match %v", nativeErr, adapterErr, test.want) + } + }) + } + if client.calls != 0 { + t.Fatalf("request validation made %d model calls", client.calls) + } +} + +func TestWithRequestValidationBuilderOutputReachesNativeValidation(t *testing.T) { + type contextKey struct{} + type input struct{ campaign string } + type state struct{ token string } + + client := &validationFailOnGenerate{} + engine := validationTestEngine(t, adapterPrompt, promptkit.WithLLMClient(client)) + ctx := context.WithValue(context.Background(), contextKey{}, "validation invocation") + selection := ckconfig.Selection{Overlay: commandkit.Supplied[string]{Value: "campaign.yml", Set: true}} + seed := promptkit.RunRequest{PromptID: "briefing", ProfileID: "default", Vars: map[string]string{"topic": "seed topic"}} + wantInput, wantState := input{campaign: "spring"}, state{token: "shared"} + builderCalls, loaderCalls := 0, 0 + suite := WithRequestValidation(Suite[input, state]{}, func(gotContext context.Context, gotSelection ckconfig.Selection, gotInput input, gotState state) (*promptkit.Engine, error) { + loaderCalls++ + if gotContext != ctx || gotContext.Value(contextKey{}) != "validation invocation" || gotSelection != selection || gotInput != wantInput || gotState != wantState { + t.Errorf("loader args = (%v, %#v, %#v, %#v)", gotContext, gotSelection, gotInput, gotState) + } + return engine, nil + }, func(gotContext context.Context, request ValidationRequest, gotInput input, gotState state) (promptkit.RunRequest, error) { + builderCalls++ + if gotContext != ctx || gotContext.Value(contextKey{}) != "validation invocation" || request.Selection != selection || request.Request.PromptID != seed.PromptID || request.Request.Vars["topic"] != "seed topic" || gotInput != wantInput || gotState != wantState { + t.Errorf("builder args = (%v, %#v, %#v, %#v)", gotContext, request, gotInput, gotState) + } + built := request.Request + built.Inputs = map[string]promptkit.ArtifactRef{"topic": promptkit.Inline("required artifact from builder")} + built.SessionID = "builder-session" + return built, nil + }) + + acquired, err := suite.ValidateRequest(ctx, ValidationRequest{Selection: selection, Request: seed}, wantInput, wantState) + if err != nil || acquired.Value != (struct{}{}) || builderCalls != 1 || loaderCalls != 1 { + t.Fatalf("built request validation = (%#v, %v), builder calls = %d, loader calls = %d", acquired, err, builderCalls, loaderCalls) + } + if client.calls != 0 { + t.Fatalf("request validation made %d model calls", client.calls) + } +} + +func TestWithRequestValidationRetainsNilEngineAndLoadDiagnostics(t *testing.T) { + request := ValidationRequest{Request: promptkit.RunRequest{PromptID: "empty", ProfileID: "default"}} + nilEngineSuite := WithRequestValidation(Suite[struct{}, struct{}]{}, func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) { + return nil, nil + }, nil) + acquired, err := nilEngineSuite.ValidateRequest(context.Background(), request, struct{}{}, struct{}{}) + if err == nil || !strings.Contains(err.Error(), "nil engine") || len(acquired.Diagnostics) != 0 { + t.Fatalf("nil engine result = (%#v, %v)", acquired, err) + } + + _, loadErr := configkit.Load[map[string]any](configkit.Bytes("invalid validation configuration", []byte("broken: ["))) + if loadErr == nil { + t.Fatal("invalid configuration unexpectedly loaded") + } + loadFailure := WithRequestValidation(Suite[struct{}, struct{}]{}, func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) { + return nil, loadErr + }, nil) + acquired, err = loadFailure.ValidateRequest(context.Background(), request, struct{}{}, struct{}{}) + var retained *configkit.LoadError + if !errors.Is(err, loadErr) || !errors.As(err, &retained) || retained != loadErr || len(acquired.Diagnostics) == 0 { + t.Fatalf("load failure = (%#v, %v), want original ConfigKit error and diagnostics", acquired, err) + } +} + +func TestWithRequestValidationCancellationBoundaries(t *testing.T) { + engine := adapterEngine(t) + request := ValidationRequest{Request: promptkit.RunRequest{PromptID: "missing", ProfileID: "default"}} + + t.Run("before builder", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + builderCalls, loaderCalls := 0, 0 + suite := WithRequestValidation(Suite[int, int]{}, func(context.Context, ckconfig.Selection, int, int) (*promptkit.Engine, error) { + loaderCalls++ + return engine, nil + }, func(context.Context, ValidationRequest, int, int) (promptkit.RunRequest, error) { + builderCalls++ + return request.Request, nil + }) + acquired, err := suite.ValidateRequest(ctx, request, 0, 0) + if !errors.Is(err, context.Canceled) || builderCalls != 0 || loaderCalls != 0 || len(acquired.Diagnostics) != 0 { + t.Fatalf("pre-build cancellation = (%#v, %v), builder calls=%d, loader calls=%d", acquired, err, builderCalls, loaderCalls) + } + }) + + t.Run("after builder", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + builderCalls, loaderCalls := 0, 0 + suite := WithRequestValidation(Suite[int, int]{}, func(context.Context, ckconfig.Selection, int, int) (*promptkit.Engine, error) { + loaderCalls++ + return engine, nil + }, func(context.Context, ValidationRequest, int, int) (promptkit.RunRequest, error) { + builderCalls++ + cancel() + return request.Request, nil + }) + acquired, err := suite.ValidateRequest(ctx, request, 0, 0) + if !errors.Is(err, context.Canceled) || builderCalls != 1 || loaderCalls != 0 || len(acquired.Diagnostics) != 0 { + t.Fatalf("post-build cancellation = (%#v, %v), builder calls=%d, loader calls=%d", acquired, err, builderCalls, loaderCalls) + } + }) + + t.Run("during load retains load failure", func(t *testing.T) { + _, loadErr := configkit.Load[map[string]any](configkit.Bytes("cancelled validation configuration", []byte("broken: ["))) + if loadErr == nil { + t.Fatal("invalid configuration unexpectedly loaded") + } + ctx, cancel := context.WithCancel(context.Background()) + loaderCalls := 0 + suite := WithRequestValidation(Suite[int, int]{}, func(context.Context, ckconfig.Selection, int, int) (*promptkit.Engine, error) { + loaderCalls++ + cancel() + return nil, loadErr + }, nil) + acquired, err := suite.ValidateRequest(ctx, request, 0, 0) + var retained *configkit.LoadError + if !errors.Is(err, loadErr) || !errors.As(err, &retained) || retained != loadErr || len(acquired.Diagnostics) == 0 || loaderCalls != 1 { + t.Fatalf("cancelled load = (%#v, %v), loader calls=%d", acquired, err, loaderCalls) + } + }) + + t.Run("after successful load skips native validation", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + loaderCalls := 0 + suite := WithRequestValidation(Suite[int, int]{}, func(context.Context, ckconfig.Selection, int, int) (*promptkit.Engine, error) { + loaderCalls++ + cancel() + return engine, nil + }, nil) + acquired, err := suite.ValidateRequest(ctx, request, 0, 0) + if !errors.Is(err, context.Canceled) || loaderCalls != 1 || len(acquired.Diagnostics) != 0 { + t.Fatalf("post-load cancellation = (%#v, %v), loader calls=%d", acquired, err, loaderCalls) + } + }) + + t.Run("native artifact cancellation retains PromptKit category", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + client := &validationFailOnGenerate{} + engine := validationTestEngine(t, adapterPrompt, promptkit.WithArtifactReader(validationCancelArtifactReader{cancel: cancel}), promptkit.WithLLMClient(client)) + suite := WithRequestValidation(Suite[int, int]{}, func(context.Context, ckconfig.Selection, int, int) (*promptkit.Engine, error) { + return engine, nil + }, nil) + request := ValidationRequest{Request: promptkit.RunRequest{ + PromptID: "briefing", + ProfileID: "default", + Inputs: map[string]promptkit.ArtifactRef{"topic": promptkit.File("cancel-topic.txt")}, + }} + acquired, err := suite.ValidateRequest(ctx, request, 0, 0) + if !errors.Is(err, promptkit.ErrArtifactLoad) || len(acquired.Diagnostics) != 0 { + t.Fatalf("native artifact cancellation = (%#v, %v), want PromptKit artifact-load category", acquired, err) + } + if client.calls != 0 { + t.Fatalf("request validation made %d model calls", client.calls) + } + }) +}