From 20d2924338af6253b7b614d900ec1d1d4382fbff Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Mon, 16 Nov 2026 18:06:45 +0000 Subject: [PATCH] Verify smoke and composed configuration outcomes --- docs/roadmap/implementation.md | 8 +- .../ckprompt/comparison_composition_test.go | 2 +- ...configuration_composition_outcomes_test.go | 100 ++++++++++++++++++ .../ckprompt/examples/composed/main_test.go | 2 +- 4 files changed, 109 insertions(+), 3 deletions(-) create mode 100644 integrations/ckprompt/configuration_composition_outcomes_test.go diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 6bfc69f..7d83d0a 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -252,7 +252,13 @@ packages using the aligned workspace. ## Stage 4 — Verify smoke aggregation and configuration composition -Status — pending. +Status — complete. Verified individual and aggregate smoke outcomes, mixed +failures and cancellation with retained reports and causes, and all five +operations through the composed PromptKit configuration scope, including the +right comparison side and pre-canceled invocations. Updated the existing +comparison and composed-example expectations for input diagnostics. With Go +1.27.0, `GOTOOLCHAIN=local GOPROXY=off go test ./integrations/ckprompt/...` +passed in the aligned workspace using a temporary Go build cache. In `integrations/ckprompt/smoke_command.go`, complete the context argument updates and apply the shared classification to individual failures and the final joined diff --git a/integrations/ckprompt/comparison_composition_test.go b/integrations/ckprompt/comparison_composition_test.go index 69d955c..a423b9e 100644 --- a/integrations/ckprompt/comparison_composition_test.go +++ b/integrations/ckprompt/comparison_composition_test.go @@ -154,7 +154,7 @@ func TestPublicConfigDiffUsesPromptKitPairAndPreparedInputs(t *testing.T) { var diagnostics bytes.Buffer failure := app.Execute(context.Background(), []string{"config", "diff", "--scope", "promptkit", "--config", "bad-left.yml", "--against-config", "right.yml", "--campaign", "spring", "--against-campaign", "spring"}, commandkit.Streams{Out: io.Discard, Err: &diagnostics}) - if failure.Kind != commandkit.OutcomeOperational || !strings.Contains(diagnostics.String(), `Configuration scope "promptkit" (left):`) { + if failure.Kind != commandkit.OutcomeUsage || !strings.Contains(diagnostics.String(), `Configuration scope "promptkit" (left):`) { t.Fatalf("installed load failure = %#v, diagnostics %q", failure, diagnostics.String()) } var loadFailure *configkit.LoadError diff --git a/integrations/ckprompt/configuration_composition_outcomes_test.go b/integrations/ckprompt/configuration_composition_outcomes_test.go new file mode 100644 index 0000000..3375d09 --- /dev/null +++ b/integrations/ckprompt/configuration_composition_outcomes_test.go @@ -0,0 +1,100 @@ +package ckprompt + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "strings" + "testing" + + "go.maximumdirect.net/ckconfig" + "go.maximumdirect.net/commandkit" + "go.maximumdirect.net/configkit" + "go.maximumdirect.net/promptkit" +) + +func TestComposedConfigurationInputOutcomes(t *testing.T) { + _, loadErr := configkit.Load[map[string]any](configkit.Bytes("prompt-invalid", []byte("broken: ["))) + engine := adapterEngine(t) + calls := 0 + scope := ConfigScope(func(_ context.Context, selection ckconfig.Selection, _, _ struct{}) (*promptkit.Engine, error) { + calls++ + if selection.Config.Value == "good" { + return engine, nil + } + return nil, fmt.Errorf("wrapped: %w", loadErr) + }) + app := newPromptTestApp(t, ckconfig.Command(ckconfig.Suite[struct{}, struct{}]{Scopes: []ckconfig.Scope[struct{}, struct{}]{scope}})) + cases := []struct { + args []string + report string + side string + }{ + {[]string{"config", "show"}, "", ""}, + {[]string{"config", "sources", "--format=json"}, `"sources"`, ""}, + {[]string{"config", "validate", "--details", "--format=json"}, `"code": "yaml.invalid"`, ""}, + {[]string{"config", "explain", "--list"}, "", ""}, + {[]string{"config", "diff"}, "", "left"}, + {[]string{"config", "diff", "--config=good", "--against-config=bad"}, "", "right"}, + } + for _, tc := range cases { + t.Run(strings.Join(tc.args, "_"), func(t *testing.T) { + var out, diag bytes.Buffer + args := append(append([]string{}, tc.args...), "--scope=promptkit") + result := app.Execute(context.Background(), args, commandkit.Streams{Out: &out, Err: &diag}) + var native *configkit.LoadError + if result.Status != 2 || !errors.Is(result.Cause, loadErr) || !errors.As(result.Cause, &native) || !strings.Contains(diag.String(), "prompt-invalid") || (tc.report != "" && !strings.Contains(out.String(), tc.report)) || (tc.side != "" && !strings.Contains(diag.String(), "("+tc.side+")")) { + t.Fatalf("result=%#v output=%q diagnostics=%q", result, out.String(), diag.String()) + } + }) + } + for _, tc := range cases { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + before := calls + var out bytes.Buffer + args := append(append([]string{}, tc.args...), "--scope=promptkit") + result := app.Execute(ctx, args, commandkit.Streams{Out: &out, Err: io.Discard}) + if result.Status != 1 || !errors.Is(result.Cause, context.Canceled) || calls != before || out.Len() != 0 { + t.Fatalf("canceled %v: %#v output=%q", tc.args, result, out.String()) + } + } +} + +func TestSmokeConfigurationFailureOutcomes(t *testing.T) { + _, first := configkit.Load[map[string]any](configkit.Bytes("first", []byte("broken: ["))) + _, second := configkit.Load[map[string]any](configkit.Bytes("second", []byte("other: ["))) + operational := errors.New("provider failure") + for _, tc := range []struct { + name, command string + failures []error + cancel bool + want int + }{ + {"single", "reachability", []error{first}, false, 2}, + {"all input", "all", []error{first, second, first, second}, false, 2}, + {"mixed", "all", []error{first, operational, second, first}, false, 1}, + {"cancellation", "all", []error{first}, true, 1}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + calls := 0 + app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{SmokeTest: func(_ context.Context, req SmokeRequest, _, _ struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) { + err := tc.failures[calls] + calls++ + if tc.cancel { + cancel() + } + return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: req.Request.Kind}, Explanation: commandkit.Explanation{Message: fmt.Sprintf("context-%d", calls)}}, err + }})) + var out, diag bytes.Buffer + result := app.Execute(ctx, []string{"promptkit", "test", tc.command, "--format=json"}, commandkit.Streams{Out: &out, Err: &diag}) + if result.Status != tc.want || calls != len(tc.failures) || !errors.Is(result.Cause, first) || (tc.name == "all input" && !errors.Is(result.Cause, second)) || (tc.name == "mixed" && (!errors.Is(result.Cause, second) || !errors.Is(result.Cause, operational))) || (tc.cancel && !errors.Is(result.Cause, context.Canceled)) || out.Len() == 0 || !strings.Contains(diag.String(), "context-1") { + t.Fatalf("outcome=%#v calls=%d report=%q diagnostics=%q", result, calls, out.String(), diag.String()) + } + }) + } +} diff --git a/integrations/ckprompt/examples/composed/main_test.go b/integrations/ckprompt/examples/composed/main_test.go index e998c51..0275041 100644 --- a/integrations/ckprompt/examples/composed/main_test.go +++ b/integrations/ckprompt/examples/composed/main_test.go @@ -170,7 +170,7 @@ func TestComposedIndependentOverridesAndFailureEvidence(t *testing.T) { } outcome = app.Execute(context.Background(), []string{"config", "validate", "--scope", "app", "--campaign", "spring", "--limit=0", "--set", `/default_campaign="autumn"`}, commandkit.Streams{Out: io.Discard, Err: io.Discard}) - if outcome.Kind != commandkit.OutcomeOperational || outcome.Cause == nil { + if outcome.Kind != commandkit.OutcomeUsage || outcome.Cause == nil { t.Fatalf("overlapping native overrides: %#v", outcome) } loads := 0