From 526a033da55a70478793ebe95d9268ed1f630a5f Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Mon, 12 Oct 2026 02:24:25 +0000 Subject: [PATCH] Cover PromptKit catalog command behavior --- docs/roadmap/implementation.md | 22 +- integrations/ckprompt/command_test.go | 292 +++++++++++++++++++++++++- 2 files changed, 312 insertions(+), 2 deletions(-) diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 20b4d86..1d049f5 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1,6 +1,6 @@ # Implement PromptKit comparison and catalogs -Status: Stages 1–7 complete; remaining stages are planned. +Status: Stages 1–8 complete; remaining stages are planned. ## Scope and authority @@ -480,6 +480,26 @@ Run focused catalog and existing named-show tests through assembled public commands. Native membership belongs to Stage 5. Add no provider network access, benchmark thresholds, concurrency guarantees, or general fixture framework. +**Completion evidence (2026-10-12)** + +- Expanded assembled-command coverage for independent prompts/profiles, mixed + show/list trees, catalog-only help, existing show-only and valid-absence + behavior, and nil-loader errors after all four callbacks are cleared. +- Verified construction, installation, help, and version do not invoke suite + factories or providers. Help with a supplied selector still converts it. + Invocation tests observe one input and state after checks, usage rejection + before acquisition, nil-factory zero values, original selector capture, + supplied empty/false/zero/repeated values, and fresh values on a second run. +- Decoded command output to verify prompt `id`/`version` string fields, profile + string sequences, no envelope, exact values and order, default and explicit + YAML, and `[]` for nil and allocated-empty inventories. Extra operands and + unavailable version/scope flags fail as usage without reaching providers. +- Covered provider error identity and diagnostics, cancellation after provider + completion preventing output, report writer errors, and short writes with + incomplete-report state and retained diagnostics. +- Focused assembled catalog/named-show tests and the complete ckprompt package + suite passed with `GOTOOLCHAIN=local go test -count=1`. + ## Stage 9: Demonstrate capabilities in the composed consumer **Outcome:** the maintained example exercises both features through bundled diff --git a/integrations/ckprompt/command_test.go b/integrations/ckprompt/command_test.go index 385be96..dc7cad5 100644 --- a/integrations/ckprompt/command_test.go +++ b/integrations/ckprompt/command_test.go @@ -98,6 +98,20 @@ func TestCatalogCommandTreesAreIndependentAndOmitShow(t *testing.T) { } }) } + + mixed := Command(Suite[struct{}, struct{}]{ + InspectPrompt: func(context.Context, PromptRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.PromptInspection], error) { + return commandkit.Acquired[*promptkit.PromptInspection]{}, nil + }, + ListPrompts: func(context.Context, ckconfig.Selection, struct{}, struct{}) (commandkit.Acquired[[]promptkit.PromptIdentity], error) { + return commandkit.Acquired[[]promptkit.PromptIdentity]{}, nil + }, + }) + output := new(bytes.Buffer) + outcome := newPromptTestApp(t, mixed).Execute(context.Background(), []string{"help", "promptkit"}, commandkit.Streams{Out: output, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(output.String(), "show") || !strings.Contains(output.String(), "list") { + t.Fatalf("mixed catalog/show help = (%#v, %q)", outcome, output.String()) + } } func TestCatalogListUsesPreparedInputAndPreflightsFormat(t *testing.T) { @@ -140,11 +154,287 @@ func TestCatalogListUsesPreparedInputAndPreflightsFormat(t *testing.T) { if outcome.Kind != commandkit.OutcomeSuccess || newInputCalls != 1 || newStateCalls != 1 || providerCalls != 1 { t.Fatalf("catalog invocation = %#v, factories/providers=%d/%d/%d output=%q", outcome, newInputCalls, newStateCalls, providerCalls, output.String()) } - if !received.campaign.Set || received.campaign.Value != "spring" || !receivedSelection.Config.Set || receivedSelection.Config.Value != "" || receivedState.token != 1 { + if !received.campaign.Set || received.campaign.Value != "spring" || !receivedSelection.Config.Set || receivedSelection.Config.Value != "" || !receivedSelection.Overlay.Set || receivedSelection.Overlay.Value != "base.yml" || receivedState.token != 1 { t.Fatalf("provider values = input %#v selection %#v state %#v", received, receivedSelection, receivedState) } } +func TestCatalogLifecycleCapturesSelectorsAndRefreshesInvocationValues(t *testing.T) { + type input struct { + campaign commandkit.Supplied[string] + shadow commandkit.Supplied[string] + enabled commandkit.Supplied[bool] + limit commandkit.Supplied[int] + labels commandkit.Supplied[[]string] + } + type state struct{ run int } + conversions, newInputs, newStates, checks, providers := 0, 0, 0, 0, 0 + reject := false + flag := commandkit.ValueFlag(commandkit.FlagMeta{Name: "campaign"}, (*string)(nil), func(value string) (string, error) { + conversions++ + return value, nil + }, func(target *input, value commandkit.Supplied[string]) { target.campaign = value }) + lookalike := commandkit.ValueFlag(commandkit.FlagMeta{Name: "campaign"}, (*string)(nil), + func(value string) (string, error) { return value, nil }, + func(target *input, value commandkit.Supplied[string]) { target.shadow = value }) + enabled := commandkit.BoolFlag(commandkit.FlagMeta{Name: "enabled"}, true, func(target *input, value commandkit.Supplied[bool]) { target.enabled = value }) + limit := commandkit.ValueFlag(commandkit.FlagMeta{Name: "limit"}, (*int)(nil), func(value string) (int, error) { + if value != "0" { + return 0, errors.New("test accepts only zero") + } + return 0, nil + }, func(target *input, value commandkit.Supplied[int]) { target.limit = value }) + labels := commandkit.RepeatFlag(commandkit.FlagMeta{Name: "label"}, []string(nil), func(value string) (string, error) { return value, nil }, + func(target *input, value commandkit.Supplied[[]string]) { target.labels = value }) + selectors := []commandkit.Flag[input]{flag, enabled, limit, labels} + suite := Suite[input, *state]{ + NewInput: func() input { newInputs++; return input{} }, + NewState: func() *state { + newStates++ + if checks != newStates { + t.Errorf("state initialized after %d checks; state count %d", checks, newStates) + } + return &state{run: newStates} + }, + Selectors: selectors, + CheckInput: func(selection ckconfig.Selection, got input) error { + checks++ + if reject { + return errors.New("catalog input rejected") + } + if checks == 1 { + if !selection.Config.Set || selection.Config.Value != "" || !selection.Overlay.Set || selection.Overlay.Value != "" || !got.campaign.Set || got.campaign.Value != "" || got.shadow.Set || !got.enabled.Set || got.enabled.Value || !got.limit.Set || got.limit.Value != 0 || !got.labels.Set || len(got.labels.Value) != 2 || got.labels.Value[0] != "late" || got.labels.Value[1] != "early" { + return errors.New("supplied catalog values were not preserved") + } + } else if selection.Config.Set || selection.Overlay.Set || got.campaign.Set || got.shadow.Set || got.enabled.Set || got.limit.Set || got.labels.Set { + return errors.New("catalog values leaked between invocations") + } + return nil + }, + ListPrompts: func(_ context.Context, selection ckconfig.Selection, got input, gotState *state) (commandkit.Acquired[[]promptkit.PromptIdentity], error) { + providers++ + if gotState == nil || gotState.run != providers { + t.Errorf("provider state = %#v at invocation %d", gotState, providers) + } + if providers == 1 && (!selection.Config.Set || selection.Config.Value != "" || !selection.Overlay.Set || selection.Overlay.Value != "" || !got.campaign.Set || !got.enabled.Set || !got.limit.Set || !got.labels.Set) { + t.Errorf("first provider values = selection %#v input %#v", selection, got) + } + return commandkit.Acquired[[]promptkit.PromptIdentity]{Value: []promptkit.PromptIdentity{}}, nil + }, + } + declaration := Command(suite) + if conversions+newInputs+newStates+checks+providers != 0 { + t.Fatal("command construction invoked suite callbacks") + } + selectors[0] = lookalike + app := newPromptTestApp(t, declaration) + if conversions+newInputs+newStates+checks+providers != 0 { + t.Fatal("installation invoked suite callbacks") + } + var output bytes.Buffer + for _, args := range [][]string{{"help", "promptkit", "list", "prompts"}, {"version"}} { + outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeSuccess { + t.Fatalf("non-invoking command %v = %#v", args, outcome) + } + } + output.Reset() + outcome := app.Execute(context.Background(), []string{"promptkit", "list", "prompts", "--campaign", "spring", "--help"}, commandkit.Streams{Out: &output, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeSuccess || conversions != 1 || newInputs+newStates+checks+providers != 0 { + t.Fatalf("help with supplied flag = %#v conversions=%d callbacks=%d/%d/%d/%d output=%q", outcome, conversions, newInputs, newStates, providers, checks, output.String()) + } + + output.Reset() + outcome = app.Execute(context.Background(), []string{"promptkit", "list", "prompts", "--campaign", "", "--enabled=false", "--limit", "0", "--label", "late", "--label=early", "--config", "", "--overlay", ""}, commandkit.Streams{Out: &output, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeSuccess || conversions != 2 || newInputs != 1 || newStates != 1 || checks != 1 || providers != 1 { + t.Fatalf("supplied catalog invocation = %#v counts convert/input/state/check/provider=%d/%d/%d/%d/%d output=%q", outcome, conversions, newInputs, newStates, checks, providers, output.String()) + } + output.Reset() + outcome = app.Execute(context.Background(), []string{"promptkit", "list", "prompts"}, commandkit.Streams{Out: &output, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeSuccess || newInputs != 2 || newStates != 2 || checks != 2 || providers != 2 { + t.Fatalf("fresh catalog invocation = %#v counts input/state/check/provider=%d/%d/%d/%d", outcome, newInputs, newStates, checks, providers) + } + reject = true + outcome = app.Execute(context.Background(), []string{"promptkit", "list", "prompts"}, commandkit.Streams{Out: &output, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeUsage || newInputs != 3 || newStates != 2 || checks != 3 || providers != 2 { + t.Fatalf("rejected input reached state/provider: %#v counts input/state/check/provider=%d/%d/%d/%d", outcome, newInputs, newStates, checks, providers) + } +} + +func TestCatalogNilFactoriesSupplyZeroValues(t *testing.T) { + app := newPromptTestApp(t, Command(Suite[*struct{}, *struct{}]{ + ListPrompts: func(_ context.Context, selection ckconfig.Selection, input, state *struct{}) (commandkit.Acquired[[]promptkit.PromptIdentity], error) { + if input != nil || state != nil || selection.Config.Set || selection.Overlay.Set { + t.Errorf("nil-factory values = selection %#v input %#v state %#v", selection, input, state) + } + return commandkit.Acquired[[]promptkit.PromptIdentity]{}, nil + }, + })) + outcome := app.Execute(context.Background(), []string{"promptkit", "list", "prompts"}, commandkit.Streams{Out: io.Discard, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeSuccess { + t.Fatalf("nil-factory catalog outcome = %#v", outcome) + } +} + +func TestCatalogCommandsRenderNativeValuesAndEmptySequences(t *testing.T) { + prompts := []promptkit.PromptIdentity{ + {ID: "later", Version: "1.10"}, + {ID: "001", Version: " 03 "}, + {ID: "a: true", Version: "yes"}, + } + profiles := []string{"later", "001", "a: true", "later"} + suite := Suite[struct{}, struct{}]{ + ListPrompts: func(context.Context, ckconfig.Selection, struct{}, struct{}) (commandkit.Acquired[[]promptkit.PromptIdentity], error) { + return commandkit.Acquired[[]promptkit.PromptIdentity]{Value: prompts}, nil + }, + ListProfiles: func(context.Context, ckconfig.Selection, struct{}, struct{}) (commandkit.Acquired[[]string], error) { + return commandkit.Acquired[[]string]{Value: profiles}, nil + }, + } + app := newPromptTestApp(t, Command(suite)) + var output bytes.Buffer + outcome := app.Execute(context.Background(), []string{"promptkit", "list", "prompts"}, commandkit.Streams{Out: &output, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeSuccess { + t.Fatalf("default prompt YAML outcome = %#v", outcome) + } + var promptEntries []map[string]any + if err := yaml.Unmarshal(output.Bytes(), &promptEntries); err != nil { + t.Fatalf("decode prompt command output %q: %v", output.String(), err) + } + if len(promptEntries) != len(prompts) { + t.Fatalf("prompt output is not the native sequence: %#v", promptEntries) + } + for index, entry := range promptEntries { + id, idOK := entry["id"].(string) + version, versionOK := entry["version"].(string) + if len(entry) != 2 || !idOK || !versionOK || id != prompts[index].ID || version != prompts[index].Version { + t.Errorf("prompt entry %d = %#v, want string id/version %q/%q", index, entry, prompts[index].ID, prompts[index].Version) + } + } + + output.Reset() + outcome = app.Execute(context.Background(), []string{"promptkit", "list", "profiles", "--format", "yaml"}, commandkit.Streams{Out: &output, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeSuccess { + t.Fatalf("explicit profile YAML outcome = %#v", outcome) + } + var profileEntries []any + if err := yaml.Unmarshal(output.Bytes(), &profileEntries); err != nil { + t.Fatalf("decode profile command output %q: %v", output.String(), err) + } + if len(profileEntries) != len(profiles) { + t.Fatalf("profile output is not the native sequence: %#v", profileEntries) + } + for index, entry := range profileEntries { + value, ok := entry.(string) + if !ok || value != profiles[index] { + t.Errorf("profile entry %d = %#v, want string %q", index, entry, profiles[index]) + } + } + + promptEmptyCalls, profileEmptyCalls := 0, 0 + emptyApp := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ + ListPrompts: func(context.Context, ckconfig.Selection, struct{}, struct{}) (commandkit.Acquired[[]promptkit.PromptIdentity], error) { + promptEmptyCalls++ + if promptEmptyCalls%2 == 0 { + return commandkit.Acquired[[]promptkit.PromptIdentity]{Value: []promptkit.PromptIdentity{}}, nil + } + return commandkit.Acquired[[]promptkit.PromptIdentity]{Value: nil}, nil + }, + ListProfiles: func(context.Context, ckconfig.Selection, struct{}, struct{}) (commandkit.Acquired[[]string], error) { + profileEmptyCalls++ + if profileEmptyCalls%2 == 0 { + return commandkit.Acquired[[]string]{Value: []string{}}, nil + } + return commandkit.Acquired[[]string]{Value: nil}, nil + }, + })) + for _, resource := range []string{"prompts", "profiles"} { + for _, inventory := range []string{"nil", "allocated"} { + output.Reset() + outcome := emptyApp.Execute(context.Background(), []string{"promptkit", "list", resource}, commandkit.Streams{Out: &output, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeSuccess || strings.TrimSpace(output.String()) != "[]" { + t.Errorf("%s %s catalog = %#v, %q", inventory, resource, outcome, output.String()) + } + } + } +} + +func TestCatalogRejectsOperandsAndUnavailableSelectors(t *testing.T) { + calls := 0 + app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ListPrompts: func(context.Context, ckconfig.Selection, struct{}, struct{}) (commandkit.Acquired[[]promptkit.PromptIdentity], error) { + calls++ + return commandkit.Acquired[[]promptkit.PromptIdentity]{}, nil + }})) + for _, args := range [][]string{ + {"promptkit", "list", "prompts", "unexpected"}, + {"promptkit", "list", "prompts", "--version", "2"}, + {"promptkit", "list", "prompts", "--scope", "app"}, + } { + outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: io.Discard, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeUsage || calls != 0 { + t.Errorf("unsupported catalog arguments %v = %#v after %d provider calls", args, outcome, calls) + } + } +} + +func TestCatalogFailuresRetainDiagnosticsAndDeliveryState(t *testing.T) { + diagnostic := commandkit.Diagnostic{Severity: commandkit.SeverityWarning, Write: func(writer io.Writer) error { + _, err := io.WriteString(writer, "catalog warning\n") + return err + }} + hasCatalogWarning := func(diagnostics []commandkit.Diagnostic) bool { + for _, got := range diagnostics { + if got.Severity != diagnostic.Severity || got.Write == nil { + continue + } + var output bytes.Buffer + if got.Write(&output) == nil && output.String() == "catalog warning\n" { + return true + } + } + return false + } + providerErr := errors.New("catalog provider failed") + app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ListPrompts: func(context.Context, ckconfig.Selection, struct{}, struct{}) (commandkit.Acquired[[]promptkit.PromptIdentity], error) { + return commandkit.Acquired[[]promptkit.PromptIdentity]{Diagnostics: []commandkit.Diagnostic{diagnostic}}, providerErr + }})) + outcome := app.Execute(context.Background(), []string{"promptkit", "list", "prompts"}, commandkit.Streams{Out: io.Discard, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, providerErr) || !hasCatalogWarning(outcome.Diagnostics) { + t.Fatalf("provider failure outcome = %#v", outcome) + } + + ctx, cancel := context.WithCancel(context.Background()) + canceled := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ListPrompts: func(context.Context, ckconfig.Selection, struct{}, struct{}) (commandkit.Acquired[[]promptkit.PromptIdentity], error) { + cancel() + return commandkit.Acquired[[]promptkit.PromptIdentity]{Value: []promptkit.PromptIdentity{{ID: "briefing", Version: "1"}}, Diagnostics: []commandkit.Diagnostic{diagnostic}}, nil + }})) + var output bytes.Buffer + outcome = canceled.Execute(ctx, []string{"promptkit", "list", "prompts"}, commandkit.Streams{Out: &output, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, context.Canceled) || !hasCatalogWarning(outcome.Diagnostics) || output.Len() != 0 { + t.Fatalf("post-provider cancellation outcome = %#v output=%q", outcome, output.String()) + } + + writeErr := errors.New("catalog report writer failed") + for _, test := range []struct { + name string + writer io.Writer + want error + }{ + {name: "writer error", writer: partialReportWriter{err: writeErr}, want: writeErr}, + {name: "short write", writer: partialReportWriter{}, want: io.ErrShortWrite}, + } { + t.Run(test.name, func(t *testing.T) { + writerApp := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ListPrompts: func(context.Context, ckconfig.Selection, struct{}, struct{}) (commandkit.Acquired[[]promptkit.PromptIdentity], error) { + return commandkit.Acquired[[]promptkit.PromptIdentity]{Value: []promptkit.PromptIdentity{{ID: "briefing", Version: "1"}}, Diagnostics: []commandkit.Diagnostic{diagnostic}}, nil + }})) + outcome := writerApp.Execute(context.Background(), []string{"promptkit", "list", "prompts"}, commandkit.Streams{Out: test.writer, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.DeliveryError, test.want) || !errors.Is(outcome.Cause, test.want) || !outcome.ReportIncomplete || !hasCatalogWarning(outcome.Diagnostics) { + t.Fatalf("catalog delivery outcome = %#v", outcome) + } + }) + } +} + func TestCommandUsesInstalledNamespaceCollisionChecks(t *testing.T) { command := Command(Suite[struct{}, struct{}]{InspectPrompt: func(context.Context, PromptRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.PromptInspection], error) { return commandkit.Acquired[*promptkit.PromptInspection]{}, nil