Demonstrate PromptKit comparison and catalogs

This commit is contained in:
2026-10-12 02:28:20 +00:00
parent 526a033da5
commit 4bc0689c44
4 changed files with 87 additions and 19 deletions

View File

@@ -1,6 +1,6 @@
# Implement PromptKit comparison and catalogs
Status: Stages 1–8 complete; remaining stages are planned.
Status: Stages 1–9 complete; remaining stages are planned.
## Scope and authority
@@ -531,6 +531,24 @@ Run example tests and the complete ckprompt suite. Existing ordinary scope
operations, named reports, app comparison, and the domain command must work with
the upgraded dependency.
**Completion evidence (2026-10-12)**
- Replaced the obsolete negative PromptKit diff test with a public composed-app
invocation that compares the before/after fixtures using the spring selector
and decodes the native change under the spring profile. Updated the after
fixture's spring model to make that comparison meaningful. The application
diff test still checks the changed default and now also checks the model value
under an explicit spring override.
- Added composed public invocations for both catalogs with config and campaign
selection. Tests decode the YAML and find the configured `briefing` version
and selected `autumn` profile without pinning PromptKit's full catalog.
- Updated the example README with copyable PromptKit diff/catalog commands and
described how the selection-aware loader and sample cache behave. Existing
ordinary scope operations, named reports, app comparison/cache behavior, and
session command coverage remain in place.
- `GOTOOLCHAIN=local go test -count=1 .` passed in the composed example, and
`GOTOOLCHAIN=local go test -count=1 ./...` passed in `integrations/ckprompt`.
## Stage 10: Reconcile canonical documentation and capability status
**Outcome:** current documentation describes delivered behavior and separates

View File

@@ -1,13 +1,17 @@
# Composed consumer
This example composes application configuration, PromptKit configuration,
definition inspection, and one application-owned command:
PromptKit comparison and catalogs, definition inspection, and one
application-owned command:
```sh
go run . config scopes
go run . config validate --scope all --campaign spring --limit=0
go run . config diff --scope app --config fixtures/before.yml --against-config fixtures/after.yml
go run . config diff --scope app --config fixtures/before.yml --against-config fixtures/after.yml --campaign spring --limit=0
go run . config diff --scope promptkit --config fixtures/before.yml --against-config fixtures/after.yml --campaign spring
go run . promptkit list prompts --config fixtures/before.yml --campaign spring
go run . promptkit list profiles --config fixtures/after.yml --campaign autumn --format yaml
go run . promptkit show prompt briefing --campaign spring --format yaml
go run . promptkit show profile spring --campaign spring
go run . session restore session-42 --campaign spring
@@ -23,11 +27,14 @@ loader functions. See the
[consumer guide](https://gitea.maximumdirect.net/eric/CommandKit/src/branch/main/docs/consumers/guide.md)
and the suites' GoDoc for initialization timing and ownership.
The application-scope comparison loads the local `before.yml` and `after.yml`
fixtures and reports `changes.default_campaign: autumn`. The second invocation
supplies `spring` to both sides, so the effective defaults compare equal; the
application input check also requires `--limit=0` when `--campaign` is supplied.
The example's shared run cache keys successful results by the complete supplied
configuration/overlay selection and the supplied campaign, including presence.
The campaign override is applied through ConfigKit before loading. These are
choices for this example's loader and do not define a CommandKit cache policy.
The application-scope comparison reports the changed default campaign and the
updated `campaigns.spring.model`. The PromptKit-scope comparison selects spring
on both sides and reports the native engine change under the spring profile.
The application input check requires `--limit=0` when `--campaign` is supplied.
Catalogs use the same selection-aware loader: the prompt catalog includes
`briefing` version `1`, and the profile catalog includes the selected `autumn`
profile. The example's shared run cache keys successful results by the complete
supplied configuration/overlay selection and the supplied campaign, including
presence. The campaign override is applied through ConfigKit before loading.
These are choices for this example's loader and do not define a CommandKit cache
policy.

View File

@@ -4,7 +4,7 @@ campaigns:
prompt_id: briefing
prompt_version: "1"
profile_id: spring
model: spring-model
model: spring-model-after
autumn:
prompt_id: briefing
prompt_version: "1"

View File

@@ -34,10 +34,9 @@ func TestComposedApplicationDiffUsesFixturesAndCampaignOverride(t *testing.T) {
if outcome.Kind != commandkit.OutcomeSuccess {
t.Fatalf("overridden fixture diff outcome = %#v", outcome)
}
for _, category := range []string{"additions", "changes", "deletions"} {
if node := composedReportValue(t, output.Bytes(), category); len(node.Content) != 0 {
t.Fatalf("campaign override left %s: %q", category, output.String())
}
value = composedReportValue(t, output.Bytes(), "changes", "campaigns", "spring", "model")
if value == nil || value.Value != "spring-model-after" {
t.Fatalf("campaign override spring model = %#v; report %q", value, output.String())
}
}
@@ -118,15 +117,59 @@ func TestComposedPromptKitAndDomainCommandsRemainAvailable(t *testing.T) {
}
}
func TestPromptKitScopeDoesNotSupportConfigurationDiff(t *testing.T) {
func TestPromptKitScopeDiffAndCatalogsUseSelectedCampaign(t *testing.T) {
app, err := newApp()
if err != nil {
t.Fatal(err)
}
var output bytes.Buffer
outcome := app.Execute(context.Background(), []string{"config", "diff", "--scope", "promptkit"}, commandkit.Streams{Out: &output, Err: io.Discard})
if outcome.Kind != commandkit.OutcomeUsage || output.Len() != 0 {
t.Fatalf("PromptKit diff outcome/output = %#v / %q", outcome, output.String())
outcome := app.Execute(context.Background(), []string{"config", "diff", "--scope", "promptkit", "--config", "fixtures/before.yml", "--against-config", "fixtures/after.yml", "--campaign", "spring"}, commandkit.Streams{Out: &output, Err: io.Discard})
if outcome.Kind != commandkit.OutcomeSuccess {
t.Fatalf("PromptKit diff outcome = %#v", outcome)
}
value := composedReportValue(t, output.Bytes(), "changes", "profiles", "spring", "model")
if value == nil || value.Value != "spring-model-after" {
t.Fatalf("PromptKit diff spring model = %#v; report %q", value, output.String())
}
output.Reset()
outcome = app.Execute(context.Background(), []string{"promptkit", "list", "prompts", "--config", "fixtures/before.yml", "--campaign", "spring"}, commandkit.Streams{Out: &output, Err: io.Discard})
if outcome.Kind != commandkit.OutcomeSuccess {
t.Fatalf("prompt catalog outcome = %#v", outcome)
}
var prompts []map[string]any
if err := yaml.Unmarshal(output.Bytes(), &prompts); err != nil {
t.Fatalf("decode prompt catalog %q: %v", output.String(), err)
}
foundPrompt := false
for _, prompt := range prompts {
if prompt["id"] == "briefing" && prompt["version"] == "1" {
foundPrompt = true
break
}
}
if !foundPrompt {
t.Fatalf("selected prompt identity missing from catalog: %#v", prompts)
}
output.Reset()
outcome = app.Execute(context.Background(), []string{"promptkit", "list", "profiles", "--config", "fixtures/after.yml", "--campaign", "autumn", "--format", "yaml"}, commandkit.Streams{Out: &output, Err: io.Discard})
if outcome.Kind != commandkit.OutcomeSuccess {
t.Fatalf("profile catalog outcome = %#v", outcome)
}
var profiles []string
if err := yaml.Unmarshal(output.Bytes(), &profiles); err != nil {
t.Fatalf("decode profile catalog %q: %v", output.String(), err)
}
foundProfile := false
for _, profile := range profiles {
if profile == "autumn" {
foundProfile = true
break
}
}
if !foundProfile {
t.Fatalf("selected profile missing from catalog: %#v", profiles)
}
}