From ddd14b70a866ca1f3411a25c31851d4f91d521fc Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Mon, 12 Oct 2026 02:01:10 +0000 Subject: [PATCH] Add PromptKit configuration comparison --- docs/roadmap/implementation.md | 24 +++- integrations/ckprompt/adapters.go | 40 ++++++- integrations/ckprompt/adapters_test.go | 145 ++++++++++++++++++++++++- 3 files changed, 204 insertions(+), 5 deletions(-) diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 74233f6..445cad6 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1,6 +1,6 @@ # Implement PromptKit comparison and catalogs -Status: Stages 1–2 complete; remaining stages are planned. +Status: Stages 1–3 complete; remaining stages are planned. ## Scope and authority @@ -210,6 +210,28 @@ side diagnostics. Exercise reachable native failures without adding a public comparator injection hook solely for tests. Add no reconstructed configuration model or duplicate ckprompt comparison renderer. +**Completion evidence (2026-10-12)** + +- `ConfigScope` now declares `ckconfig.Diff` and supplies a pair callback that + loads left then right with each side's selection/input and the shared state, + collects diagnostics in order, checks cancellation at operation boundaries, + and returns PromptKit's native `ConfigurationInspection.Diff` records. Ordinary + acquisition and the retained nil-loader issue remain available. +- Added fast native-engine adapter coverage for the PromptKit result, equal + selections reaching both loader calls, side inputs/shared state, left failure + stopping right, right ConfigKit load diagnostics and error identity, and + cancellation stopping before right load. `GOTOOLCHAIN=local go test -count=1 .` + passed in `integrations/ckprompt`. +- PromptKit v0.14.0 exposes no public way to add successful inspection warnings, + and valid engine-derived inspection views do not expose a reachable native Diff + failure. Therefore successful left-side warning retention on right failure, + both-side warning retention on native failure, and a native Diff failure are + not exercisable without adding a test-only injection path. The implementation + accumulates reached diagnostics and preserves error causes; these limits remain + recorded alongside Stage 2's warning-retention limitation. +- Installed-command verification and the composed consumer's obsolete negative + comparison expectation remain for Stages 4 and 9 respectively. + ## Stage 4: Verify comparison through installed config commands **Outcome:** the adapter works through public assembly, existing typed-input diff --git a/integrations/ckprompt/adapters.go b/integrations/ckprompt/adapters.go index 0acf4af..51e2344 100644 --- a/integrations/ckprompt/adapters.go +++ b/integrations/ckprompt/adapters.go @@ -69,7 +69,7 @@ func ConfigScope[I, S any](load func(context.Context, ckconfig.Selection, I, S) scope := ckconfig.Scope[I, S]{ Name: "promptkit", Summary: "PromptKit engine configuration.", - Operations: []ckconfig.Operation{ckconfig.Validate, ckconfig.Show, ckconfig.Sources, ckconfig.Explain}, + Operations: []ckconfig.Operation{ckconfig.Validate, ckconfig.Show, ckconfig.Sources, ckconfig.Explain, ckconfig.Diff}, } if load == nil { return ckconfig.WithScopeError(scope, errors.New("PromptKit configuration loader is nil")) @@ -89,9 +89,47 @@ func ConfigScope[I, S any](load func(context.Context, ckconfig.Selection, I, S) Diagnostics: diagnostics, }, nil } + scope.Compare = func(ctx context.Context, request ckconfig.ComparisonRequest[I], state S) (commandkit.Acquired[[]configkit.Difference], error) { + if err := promptKitContextError(ctx); err != nil { + return commandkit.Acquired[[]configkit.Difference]{}, fmt.Errorf("PromptKit scope %q comparison before left load: %w", scope.Name, err) + } + + _, leftInspection, leftDiagnostics, err := loadPromptKitEngine(ctx, request.Left.Selection, request.Left.Input, state, "left", load) + diagnostics := append([]commandkit.Diagnostic(nil), leftDiagnostics...) + if err != nil { + return commandkit.Acquired[[]configkit.Difference]{Diagnostics: diagnostics}, fmt.Errorf("PromptKit scope %q left comparison load: %w", scope.Name, err) + } + + if err := promptKitContextError(ctx); err != nil { + return commandkit.Acquired[[]configkit.Difference]{Diagnostics: diagnostics}, fmt.Errorf("PromptKit scope %q comparison after left load: %w", scope.Name, err) + } + + _, rightInspection, rightDiagnostics, err := loadPromptKitEngine(ctx, request.Right.Selection, request.Right.Input, state, "right", load) + diagnostics = append(diagnostics, rightDiagnostics...) + if err != nil { + return commandkit.Acquired[[]configkit.Difference]{Diagnostics: diagnostics}, fmt.Errorf("PromptKit scope %q right comparison load: %w", scope.Name, err) + } + + if err := promptKitContextError(ctx); err != nil { + return commandkit.Acquired[[]configkit.Difference]{Diagnostics: diagnostics}, fmt.Errorf("PromptKit scope %q comparison before native diff: %w", scope.Name, err) + } + + differences, err := leftInspection.Diff(rightInspection) + if err != nil { + return commandkit.Acquired[[]configkit.Difference]{Diagnostics: diagnostics}, fmt.Errorf("PromptKit scope %q native configuration diff: %w", scope.Name, err) + } + return commandkit.Acquired[[]configkit.Difference]{Value: differences, Diagnostics: diagnostics}, nil + } return scope } +func promptKitContextError(ctx context.Context) error { + if ctx == nil { + return nil + } + return ctx.Err() +} + // FromEngine creates named prompt and profile inspection callbacks backed by a // PromptKit engine loader. It does not call the loader during construction. Each // selected inspection passes its context, supplied selection, initialized input, diff --git a/integrations/ckprompt/adapters_test.go b/integrations/ckprompt/adapters_test.go index 3fe1e82..f4a4396 100644 --- a/integrations/ckprompt/adapters_test.go +++ b/integrations/ckprompt/adapters_test.go @@ -5,6 +5,7 @@ import ( "context" "errors" "io" + "reflect" "strings" "testing" "testing/fstest" @@ -65,9 +66,13 @@ profiles: ` func adapterEngine(t *testing.T) *promptkit.Engine { + return adapterEngineFrom(t, "adapter fixture", adapterPrompt) +} + +func adapterEngineFrom(t *testing.T, label, configuration string) *promptkit.Engine { t.Helper() - inputs, err := promptkit.FilesystemInputs("adapter fixture", fstest.MapFS{ - "config/prompt.yml": &fstest.MapFile{Data: []byte(adapterPrompt)}, + inputs, err := promptkit.FilesystemInputs(label, fstest.MapFS{ + "config/prompt.yml": &fstest.MapFile{Data: []byte(configuration)}, }, "config") if err != nil { t.Fatal(err) @@ -98,7 +103,7 @@ func TestConfigScopeUsesNativeInspectionFacetsAndInvocationValues(t *testing.T) } return engine, nil }) - if scope.Name != "promptkit" || len(scope.Operations) != 4 || scope.Acquire == nil { + if scope.Name != "promptkit" || len(scope.Operations) != 5 || scope.Acquire == nil { t.Fatalf("configuration scope = %#v", scope) } acquired, err := scope.Acquire(wantContext, ckconfig.Request{ @@ -136,6 +141,140 @@ func TestConfigScopeRetainsNilLoaderIssueAfterCapabilitiesAreCleared(t *testing. } } +func TestConfigScopeCompareUsesNativeDiffAndLoadsBothEqualSelections(t *testing.T) { + leftEngine := adapterEngine(t) + rightConfig := strings.Replace(adapterPrompt, `content: "Hello"`, `content: "Goodbye"`, 1) + if rightConfig == adapterPrompt { + t.Fatal("comparison fixture did not change the prompt content") + } + rightEngine := adapterEngineFrom(t, "right comparison fixture", rightConfig) + type input struct{ label string } + type call struct { + selection ckconfig.Selection + input input + state *int + } + var calls []call + scope := ConfigScope(func(_ context.Context, selection ckconfig.Selection, gotInput input, state *int) (*promptkit.Engine, error) { + calls = append(calls, call{selection: selection, input: gotInput, state: state}) + if gotInput.label == "before" { + return leftEngine, nil + } + return rightEngine, nil + }) + if scope.Compare == nil || len(scope.Operations) != 5 || scope.Operations[4] != ckconfig.Diff { + t.Fatalf("configuration scope does not declare comparison: %#v", scope) + } + + selection := ckconfig.Selection{Config: commandkit.Supplied[string]{Value: "same-selection.yml", Set: true}} + state := new(int) + request := ckconfig.ComparisonRequest[input]{ + Left: ckconfig.ComparisonSide[input]{Selection: selection, Input: input{label: "before"}}, + Right: ckconfig.ComparisonSide[input]{Selection: selection, Input: input{label: "after"}}, + } + leftInspection, err := leftEngine.InspectConfiguration() + if err != nil { + t.Fatal(err) + } + rightInspection, err := rightEngine.InspectConfiguration() + if err != nil { + t.Fatal(err) + } + want, err := leftInspection.Diff(rightInspection) + if err != nil { + t.Fatal(err) + } + if len(want) == 0 { + t.Fatal("native PromptKit fixture comparison produced no differences") + } + + got, err := scope.Compare(context.Background(), request, state) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got.Value, want) { + t.Fatalf("adapter differences = %#v, want native PromptKit result %#v", got.Value, want) + } + if len(calls) != 2 || calls[0].selection != selection || calls[1].selection != selection || calls[0].input.label != "before" || calls[1].input.label != "after" || calls[0].state != state || calls[1].state != state { + t.Fatalf("loader calls = %#v", calls) + } +} + +func TestConfigScopeCompareStopsAfterLeftLoadFailure(t *testing.T) { + loadErr := errors.New("left engine load failed") + rightCalled := false + scope := ConfigScope(func(_ context.Context, selection ckconfig.Selection, _ struct{}, _ struct{}) (*promptkit.Engine, error) { + if selection.Config.Value == "left" { + return nil, loadErr + } + rightCalled = true + return adapterEngine(t), nil + }) + request := ckconfig.ComparisonRequest[struct{}]{ + Left: ckconfig.ComparisonSide[struct{}]{Selection: ckconfig.Selection{Config: commandkit.Supplied[string]{Value: "left", Set: true}}}, + Right: ckconfig.ComparisonSide[struct{}]{Selection: ckconfig.Selection{Config: commandkit.Supplied[string]{Value: "right", Set: true}}}, + } + + acquired, err := scope.Compare(context.Background(), request, struct{}{}) + if !errors.Is(err, loadErr) || rightCalled || acquired.Value != nil { + t.Fatalf("left failure = (%#v, %v), right called=%t", acquired, err, rightCalled) + } +} + +func TestConfigScopeCompareRetainsRightLoadDiagnosticsAndErrorIdentity(t *testing.T) { + _, loadErr := configkit.Load[map[string]any](configkit.Bytes("invalid right prompt config", []byte("broken: ["))) + if loadErr == nil { + t.Fatal("invalid right configuration unexpectedly loaded") + } + leftEngine := adapterEngine(t) + var calls []string + scope := ConfigScope(func(_ context.Context, selection ckconfig.Selection, _ struct{}, _ struct{}) (*promptkit.Engine, error) { + calls = append(calls, selection.Config.Value) + if selection.Config.Value == "left" { + return leftEngine, nil + } + return nil, loadErr + }) + request := ckconfig.ComparisonRequest[struct{}]{ + Left: ckconfig.ComparisonSide[struct{}]{Selection: ckconfig.Selection{Config: commandkit.Supplied[string]{Value: "left", Set: true}}}, + Right: ckconfig.ComparisonSide[struct{}]{Selection: ckconfig.Selection{Config: commandkit.Supplied[string]{Value: "right", Set: true}}}, + } + + acquired, err := scope.Compare(context.Background(), request, struct{}{}) + var retained *configkit.LoadError + if !errors.Is(err, loadErr) || !errors.As(err, &retained) || retained != loadErr || len(calls) != 2 || len(acquired.Diagnostics) == 0 { + t.Fatalf("right failure = (%#v, %v), calls=%#v", acquired, err, calls) + } + var output bytes.Buffer + if writeErr := acquired.Diagnostics[0].Write(&output); writeErr != nil || !strings.Contains(output.String(), `Configuration scope "promptkit" (right):`) { + t.Fatalf("right diagnostic = %q, %v", output.String(), writeErr) + } +} + +func TestConfigScopeCompareCancellationAfterLeftLoadStopsRight(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + leftEngine := adapterEngine(t) + rightCalled := false + scope := ConfigScope(func(_ context.Context, selection ckconfig.Selection, _ struct{}, _ struct{}) (*promptkit.Engine, error) { + if selection.Config.Value == "left" { + cancel() + return leftEngine, nil + } + rightCalled = true + return adapterEngine(t), nil + }) + request := ckconfig.ComparisonRequest[struct{}]{ + Left: ckconfig.ComparisonSide[struct{}]{Selection: ckconfig.Selection{Config: commandkit.Supplied[string]{Value: "left", Set: true}}}, + Right: ckconfig.ComparisonSide[struct{}]{Selection: ckconfig.Selection{Config: commandkit.Supplied[string]{Value: "right", Set: true}}}, + } + + acquired, err := scope.Compare(ctx, request, struct{}{}) + if !errors.Is(err, context.Canceled) || rightCalled || acquired.Value != nil { + t.Fatalf("cancellation = (%#v, %v), right called=%t", acquired, err, rightCalled) + } +} + func TestFromEnginePassesRequestsAndRetainsNilLoaderIssue(t *testing.T) { type input struct{ value string } type state struct{ value string }