From 35e958c210f0fb62181d9fc3680b1eb50350aac2 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sat, 17 Oct 2026 19:06:05 +0000 Subject: [PATCH] Demonstrate PromptKit catalogs and smoke tests --- docs/roadmap/implementation.md | 13 +++++- .../ckprompt/examples/composed/README.md | 41 +++++++++++++++--- .../ckprompt/examples/composed/main.go | 42 ++++++++++++++++++- .../ckprompt/examples/composed/main_test.go | 39 +++++++++++++++++ 4 files changed, 127 insertions(+), 8 deletions(-) diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 3b46443..18f1474 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1,6 +1,6 @@ # Dependency upgrades, discovery, and smoke testing implementation plan -Status: Stages 1–14 complete; Stage 15 is next. +Status: Stages 1–15 complete; Stage 16 is next. ## Authority and baseline @@ -738,6 +738,17 @@ native API's success criteria. **Complete when:** the consumer demonstrates the new interfaces with little application plumbing and no model-service prerequisite. +**Completed:** 2026-10-17. The composed consumer now explicitly wires smoke +testing with a nil request builder and exposes backend, namespace, collection, +and namespace-filtered prompt discovery using one synthetic collection. Its +shared invocation state carries an injected in-memory PromptKit client; tests +confirm metadata and requirements leave it unused and smoke inference invokes +it. The README documents ID-only inspection, optional version metadata, all +catalogs, namespace filtering, typed builder customization, and single and +aggregate smoke commands. The composed test suite passed, as did the documented +offline discovery, inspection, requirements, and smoke commands. Native HTTP +checks reported their configured unsupported/not-applicable observations. + ## Stage 16: Reconcile dependency graphs and notices **Primary files:** all three module manifests/sums and attribution documents. diff --git a/integrations/ckprompt/examples/composed/README.md b/integrations/ckprompt/examples/composed/README.md index 6ca3520..694b405 100644 --- a/integrations/ckprompt/examples/composed/README.md +++ b/integrations/ckprompt/examples/composed/README.md @@ -12,6 +12,10 @@ go run . config diff --scope app --config fixtures/before.yml --against-config f go run . config diff --scope promptkit --config fixtures/before.yml --against-config fixtures/after.yml --campaign spring go run . promptkit list prompts --config fixtures/before.yml --campaign spring go run . promptkit list profiles --config fixtures/after.yml --campaign autumn --format yaml +go run . promptkit list backends --campaign spring +go run . promptkit list namespaces --campaign spring +go run . promptkit list collections --campaign spring +go run . promptkit list prompts --namespace field-notes --campaign spring go run . promptkit show prompt briefing --campaign spring --format yaml go run . promptkit show profile spring --campaign spring go run . promptkit requirements show briefing --campaign spring @@ -19,6 +23,8 @@ go run . promptkit requirements show briefing --campaign spring --var mode=full go run . promptkit requirements validate briefing --campaign spring --input topic=fixtures/topic.txt go run . promptkit requirements validate briefing --campaign spring --input topic=fixtures/topic.txt --var mode=full --input notes=fixtures/notes.txt --format yaml go run . promptkit requirements validate briefing --campaign spring +go run . promptkit test text_inference --campaign spring --prompt briefing --profile spring +go run . promptkit test all --campaign spring --prompt briefing --profile spring go run . session restore session-42 --campaign spring ``` @@ -38,11 +44,36 @@ on both sides and reports the native engine change under the spring profile. The application input check requires `--limit=0` when `--campaign` is supplied. Catalogs use the same selection-aware loader: the prompt catalog includes `briefing` version `1`, and the profile catalog includes the selected `autumn` -profile. The example's shared run cache keys successful results by the complete -supplied configuration/overlay selection and the supplied campaign, including -presence. The campaign override is applied through ConfigKit before loading. -These are choices for this example's loader and do not define a CommandKit cache -policy. +profile. Prompt inspection uses an ID only; its optional version is descriptive +metadata in the report, not a selector. The synthetic `field-notes` collection +provides one backend, profile, and prompt for collection discovery and native +namespace filtering. The example's shared run cache keys successful results by +the complete supplied configuration/overlay selection and the supplied +campaign, including presence. The campaign override is applied through ConfigKit +before loading. These are choices for this example's loader and do not define a +CommandKit cache policy. + +Smoke testing is explicitly wired with `ckprompt.WithSmokeTesting` and its +optional request builder is nil, so CommandKit forwards the selected native +request. Applications that need to adjust the request can provide a typed +builder that starts with the seed and returns the chosen native request: + +```go +definitions = ckprompt.WithSmokeTesting(definitions, loadEngine, + func(ctx context.Context, seed ckprompt.SmokeRequest, input *inspectionInput, state *runCache) (promptkit.SmokeTestRequest, error) { + request := seed.Request + request.ProfileID = "spring" + return request, nil + }) +``` + +The example injects a small in-memory `promptkit.LLMClient` through its shared +run state. Catalog, inspection, and requirements commands do not call it; +inference smoke commands do. No model service or credentials are required. +Native HTTP checks may retain unsupported or not-applicable observations because +the example does not configure a network probe. An aggregate smoke command can +return an operational outcome for a native diagnostic while still delivering +the complete aggregate report. The first requirements command reports incomplete discovery: PromptKit records the conditional `notes` input and a missing `mode` issue. Supplying `mode=full` diff --git a/integrations/ckprompt/examples/composed/main.go b/integrations/ckprompt/examples/composed/main.go index 9072cd3..0f3b015 100644 --- a/integrations/ckprompt/examples/composed/main.go +++ b/integrations/ckprompt/examples/composed/main.go @@ -40,6 +40,18 @@ type applicationCacheKey struct { // chooses to pass it to its loaders. type runCache struct { applications map[applicationCacheKey]*configkit.Result[appConfiguration] + llmClient promptkit.LLMClient +} + +type exampleLLMClient struct { + calls *int +} + +func (client *exampleLLMClient) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + if client.calls != nil { + (*client.calls)++ + } + return &promptkit.GenerateResponse{Content: "Example response."}, nil } type restoreInput struct { @@ -110,6 +122,20 @@ func loadPromptEngine(ctx context.Context, selection ckconfig.Selection, input * return nil, err } engineConfig := configkit.Mapping("campaign "+name, map[string]any{ + "collections": map[string]any{ + "field-notes": map[string]any{ + "release_version": "1", + "backends": map[string]any{"demo": map[string]any{"endpoint": "https://example.invalid/v1"}}, + "profiles": map[string]any{"assistant": map[string]any{"backend": "field-notes/demo", "model": "example-model"}}, + "prompts": map[string]any{"briefing": map[string]any{ + "version": "1", + "default_profile": "field-notes/assistant", + "inputs": []any{map[string]any{"name": "topic", "required": true, "content_type": "text/plain", "description": "Topic to brief"}}, + "messages": []any{map[string]any{"role": "user", "content": "Write a concise briefing about {{.topic}}."}}, + "output": map[string]any{"format": "text", "validation_mode": "none"}, + }}, + }, + }, "profiles": map[string]any{chosen.ProfileID: map[string]any{ "endpoint": "https://example.invalid/v1", "model": chosen.Model, @@ -125,7 +151,11 @@ func loadPromptEngine(ctx context.Context, selection ckconfig.Selection, input * "output": map[string]any{"format": "text", "validation_mode": "none"}, }}, }) - return promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{Source: engineConfig}}}) + var client promptkit.LLMClient = &exampleLLMClient{} + if cache != nil && cache.llmClient != nil { + client = cache.llmClient + } + return promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{Source: engineConfig}}}, promptkit.WithLLMClient(client)) } func selectedCampaign(configuration appConfiguration, supplied commandkit.Supplied[string]) (string, campaign, error) { @@ -166,11 +196,18 @@ func buildValidationRequest(ctx context.Context, seed ckprompt.ValidationRequest } func newApp() (*commandkit.App, error) { + return newAppWithLLMClient(&exampleLLMClient{}) +} + +func newAppWithLLMClient(client promptkit.LLMClient) (*commandkit.App, error) { + if client == nil { + client = &exampleLLMClient{} + } limitDefault := 0 limitFlag := commandkit.ValueFlag(commandkit.FlagMeta{Name: "limit", Usage: "Override the application item limit."}, &limitDefault, strconv.Atoi, func(input **inspectionInput, value commandkit.Supplied[int]) { (*input).Limit = value }) inputFactory := func() *inspectionInput { return &inspectionInput{} } - stateFactory := func() *runCache { return &runCache{} } + stateFactory := func() *runCache { return &runCache{llmClient: client} } appScope := ckconfig.FromConfigKit("app", "Application settings.", loadApplicationConfig) engineScope := ckprompt.ConfigScope(loadPromptEngine) @@ -195,6 +232,7 @@ func newApp() (*commandkit.App, error) { definitions.NewState = stateFactory definitions.Selectors = []commandkit.Flag[*inspectionInput]{campaignSelector()} definitions = ckprompt.WithRequestValidation(definitions, loadPromptEngine, buildValidationRequest) + definitions = ckprompt.WithSmokeTesting(definitions, loadPromptEngine, nil) definitionCommand := ckprompt.Command(definitions) store := map[string]session{ diff --git a/integrations/ckprompt/examples/composed/main_test.go b/integrations/ckprompt/examples/composed/main_test.go index ef38f53..a5953cf 100644 --- a/integrations/ckprompt/examples/composed/main_test.go +++ b/integrations/ckprompt/examples/composed/main_test.go @@ -311,6 +311,45 @@ func TestPromptKitScopeDiffAndCatalogsUseSelectedCampaign(t *testing.T) { } } +func TestComposedCatalogsAndSmokeUseInjectedClientOnlyForInference(t *testing.T) { + calls := 0 + app, err := newAppWithLLMClient(&exampleLLMClient{calls: &calls}) + if err != nil { + t.Fatal(err) + } + + commands := [][]string{ + {"promptkit", "list", "backends", "--campaign", "spring"}, + {"promptkit", "list", "namespaces", "--campaign", "spring"}, + {"promptkit", "list", "collections", "--campaign", "spring"}, + {"promptkit", "list", "prompts", "--namespace", "field-notes", "--campaign", "spring"}, + {"promptkit", "show", "prompt", "briefing", "--campaign", "spring"}, + {"promptkit", "requirements", "show", "briefing", "--campaign", "spring"}, + } + for _, args := range commands { + var output bytes.Buffer + outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeSuccess { + t.Errorf("metadata command %q outcome = %#v; output %q", args, outcome, output.String()) + } + if output.Len() == 0 { + t.Errorf("metadata command %q produced no report", args) + } + } + if calls != 0 { + t.Fatalf("metadata and requirements commands invoked the LLM client %d times", calls) + } + + var output bytes.Buffer + outcome := app.Execute(context.Background(), []string{"promptkit", "test", "all", "--campaign", "spring", "--prompt", "briefing", "--profile", "spring"}, commandkit.Streams{Out: &output, Err: io.Discard}) + if output.Len() == 0 || !bytes.Contains(output.Bytes(), []byte("tests:")) { + t.Fatalf("aggregate smoke report missing: outcome=%#v output=%q", outcome, output.String()) + } + if calls == 0 { + t.Fatalf("smoke command did not use the injected LLM client: outcome=%#v output=%q", outcome, output.String()) + } +} + func composedReportValue(t *testing.T, data []byte, category string, fields ...string) *yaml.Node { t.Helper() var document yaml.Node