diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index cc0fe9e..9148326 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1,6 +1,6 @@ # Invocation reporting implementation plan -Status: Stages 1–16 complete. Stage 17 is next. +Status: Stages 1–17 complete. Stage 18 is next. ## Objective and authority @@ -642,7 +642,7 @@ explanation, then exits with status 1 as expected. ## Stage 17: Migrate maintained integration consumers -Status: pending. +Status: complete. Primary directories: [ckconfig example](../../integrations/ckconfig/examples/simple) and @@ -664,6 +664,16 @@ Validation: both example packages compile in their owning modules and run without external services. Confirm custom commands and integration commands share the same reporting flags and terminal behavior. +Completion: migrated both example-owned handlers to invocation reporters and +explicit explanations with optional causes. The ConfigKit consumer demonstrates +a provider schema warning in verbose mode, ordinary report output, and quiet +suppression. The composed consumer retains aliased help and offline smoke usage; +its tests cover a native report with an independent provider warning, terminal +context, and an operation error, plus quiet custom-command success and domain +failure. Both integration modules pass `GOTOOLCHAIN=local go test ./...`, +including both example packages. Representative verbose ConfigKit, quiet +ConfigKit, aliased quiet help, and offline PromptKit smoke commands also pass. + ## Stage 18: Reconcile the full test suite and architectural boundaries Status: pending. diff --git a/integrations/ckconfig/examples/simple/README.md b/integrations/ckconfig/examples/simple/README.md index 0b67d70..46a9419 100644 --- a/integrations/ckconfig/examples/simple/README.md +++ b/integrations/ckconfig/examples/simple/README.md @@ -4,6 +4,8 @@ Run the maintained example with: ```sh go run . config show +go run . config show --config fixtures/before.yml --verbose +go run . config show --config fixtures/before.yml --quiet go run . config diff --config fixtures/before.yml --against-config fixtures/after.yml go run . greet go run . greet --count=0 --tag=first --tag second @@ -15,3 +17,6 @@ omitted `--count` and an explicitly supplied zero, plus ordered repeated tags. With no `--config`, application settings use the in-memory default. A supplied configuration and overlay are passed to ConfigKit unchanged. The comparison command loads the two local fixtures and reports `changes.greeting: Goodbye`. +The before fixture uses schema version 1, which ConfigKit reports as a provider +warning in verbose mode. Ordinary mode hides the warning while preserving the +effective YAML report; quiet mode suppresses that report. diff --git a/integrations/ckconfig/examples/simple/fixtures/after.yml b/integrations/ckconfig/examples/simple/fixtures/after.yml index 9bb850c..a5fc104 100644 --- a/integrations/ckconfig/examples/simple/fixtures/after.yml +++ b/integrations/ckconfig/examples/simple/fixtures/after.yml @@ -1 +1,3 @@ +schema: + version: 2 greeting: Goodbye diff --git a/integrations/ckconfig/examples/simple/fixtures/before.yml b/integrations/ckconfig/examples/simple/fixtures/before.yml index 0f3a310..8192e75 100644 --- a/integrations/ckconfig/examples/simple/fixtures/before.yml +++ b/integrations/ckconfig/examples/simple/fixtures/before.yml @@ -1 +1,3 @@ +schema: + version: 1 greeting: Hello diff --git a/integrations/ckconfig/examples/simple/main.go b/integrations/ckconfig/examples/simple/main.go index a13a0b4..fc88ce7 100644 --- a/integrations/ckconfig/examples/simple/main.go +++ b/integrations/ckconfig/examples/simple/main.go @@ -3,6 +3,7 @@ package main import ( "context" "fmt" + "io" "os" "strconv" @@ -12,6 +13,9 @@ import ( ) type appConfiguration struct { + Schema struct { + Version int `yaml:"version"` + } `yaml:"schema"` Greeting string `yaml:"greeting"` } @@ -22,12 +26,25 @@ type greetInput struct { func loadAppConfiguration(_ context.Context, selection ckconfig.Selection) (*configkit.Result[appConfiguration], error) { source := configkit.Mapping("simple application settings", map[string]any{ + "schema": map[string]any{"version": 2}, "greeting": "Hello", }) if selection.Config.Set { source = configkit.File(selection.Config.Value) } - options := configkit.Options[appConfiguration]{} + options := configkit.Options[appConfiguration]{ + Definition: configkit.Definition[appConfiguration]{ + Version: &configkit.VersionRule{ + Path: configkit.Keys("schema", "version"), + Check: func(version configkit.Version, validation *configkit.Validation) error { + if version.Text == "1" { + validation.Warning(configkit.Keys("schema", "version"), "schema version 1 is supported; upgrade to version 2 before the next example release") + } + return nil + }, + }, + }, + } if selection.Overlay.Set { options.Overlay = selection.Overlay.Value options.Filesystem = &configkit.FilesystemContext{Root: ".", BaseDir: "."} @@ -50,9 +67,10 @@ func newApp() (*commandkit.App, error) { }), }, Run: func(invocation commandkit.Invocation, input greetInput) commandkit.Outcome { + reporter := invocation.Reporter() configuration, err := loadAppConfiguration(invocation.Context, ckconfig.Selection{}) if err != nil { - return commandkit.Failure(err) + return commandkit.Failure("Could not load application configuration.").WithCause(err) } count := fmt.Sprintf("default %d (not supplied)", input.Count.Value) if input.Count.Set { @@ -62,9 +80,10 @@ func newApp() (*commandkit.App, error) { if input.Tags.Set { tags = fmt.Sprintf("explicit %v", input.Tags.Value) } - if _, err := fmt.Fprintf(invocation.Streams.Out, "%s; count=%s; tags=%s\n", configuration.Config.Greeting, count, tags); err != nil { - return commandkit.Failure(err) - } + _ = reporter.RenderOutput(func(writer io.Writer) error { + _, err := fmt.Fprintf(writer, "%s; count=%s; tags=%s\n", configuration.Config.Greeting, count, tags) + return err + }) return commandkit.Success() }, }) diff --git a/integrations/ckconfig/examples/simple/main_test.go b/integrations/ckconfig/examples/simple/main_test.go index 1ef36cd..200a96a 100644 --- a/integrations/ckconfig/examples/simple/main_test.go +++ b/integrations/ckconfig/examples/simple/main_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "io" + "strings" "testing" commandkit "gitea.maximumdirect.net/eric/CommandKit" @@ -42,7 +43,7 @@ func TestSimpleAppComparesFixturesAndKeepsDefaultCommands(t *testing.T) { output.Reset() outcome = app.Execute(context.Background(), []string{"config", "show"}, commandkit.Streams{Out: &output, Err: io.Discard}) - if outcome.Kind != commandkit.OutcomeSuccess || output.String() != "greeting: Hello\n" { + if outcome.Kind != commandkit.OutcomeSuccess || output.String() != "greeting: Hello\nschema:\n version: 2\n" { t.Fatalf("default show outcome/output = %#v / %q", outcome, output.String()) } @@ -60,6 +61,73 @@ func TestSimpleLoaderDoesNotReplaceExplicitEmptyConfiguration(t *testing.T) { } } +func TestSimpleConfigurationReportingModes(t *testing.T) { + for _, test := range []struct { + name string + args []string + wantOutput string + wantDiagnostic string + quiet bool + }{ + { + name: "ordinary output hides provider warning", + args: []string{"config", "show", "--config", "fixtures/before.yml"}, + wantOutput: "greeting: Hello\nschema:\n version: 1\n", + }, + { + name: "quiet suppresses configuration report", + args: []string{"config", "show", "--config", "fixtures/before.yml", "--quiet"}, + quiet: true, + }, + { + name: "verbose shows ConfigKit warning", + args: []string{"config", "show", "--config", "fixtures/before.yml", "--verbose"}, + wantOutput: "greeting: Hello\nschema:\n version: 1\n", + wantDiagnostic: "schema version 1 is supported", + }, + } { + t.Run(test.name, func(t *testing.T) { + outcome, output, diagnostics := executeSimple(t, test.args...) + if outcome.Kind != commandkit.OutcomeSuccess || outcome.Cause != nil { + t.Fatalf("outcome = %#v", outcome) + } + if output != test.wantOutput { + t.Errorf("output = %q, want %q", output, test.wantOutput) + } + if test.wantDiagnostic != "" && !strings.Contains(diagnostics, test.wantDiagnostic) { + t.Errorf("diagnostics = %q, want provider warning %q", diagnostics, test.wantDiagnostic) + } + if test.wantDiagnostic == "" && diagnostics != "" { + t.Errorf("diagnostics = %q, want no warning outside verbose mode", diagnostics) + } + if strings.Contains(diagnostics, "ERROR:") { + t.Errorf("diagnostics %q unexpectedly contain a terminal failure", diagnostics) + } + if test.quiet && (outcome.Presentation.Output.State != commandkit.DeliverySuppressed || outcome.Presentation.Diagnostics.State != commandkit.DeliverySuppressed) { + t.Errorf("quiet presentation = %#v, want report and provider warning suppressed", outcome.Presentation) + } + }) + } +} + +func TestSimpleCustomCommandUsesSharedReportingFlags(t *testing.T) { + outcome, output, diagnostics := executeSimple(t, "greet", "--quiet") + if outcome.Kind != commandkit.OutcomeSuccess || output != "" || diagnostics != "" { + t.Fatalf("quiet greet outcome/output/diagnostics = %#v / %q / %q", outcome, output, diagnostics) + } +} + +func executeSimple(t *testing.T, args ...string) (commandkit.Outcome, string, string) { + t.Helper() + app, err := newApp() + if err != nil { + t.Fatal(err) + } + var output, diagnostics bytes.Buffer + outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: &diagnostics}) + return outcome, output.String(), diagnostics.String() +} + func field(mapping *yaml.Node, name string) *yaml.Node { if mapping == nil || mapping.Kind != yaml.MappingNode { return nil diff --git a/integrations/ckprompt/examples/composed/main.go b/integrations/ckprompt/examples/composed/main.go index f94390f..9c2c63d 100644 --- a/integrations/ckprompt/examples/composed/main.go +++ b/integrations/ckprompt/examples/composed/main.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "io" "os" "strconv" @@ -65,6 +66,8 @@ type session struct { Summary string } +var errSessionNotFound = errors.New("saved session was not found") + func campaignSelector() commandkit.Flag[*inspectionInput] { return commandkit.ValueFlag(commandkit.FlagMeta{Name: "campaign", Usage: "Select campaign inputs."}, (*string)(nil), parseString, func(input **inspectionInput, value commandkit.Supplied[string]) { (*input).Campaign = value }) @@ -200,10 +203,14 @@ func newApp() (*commandkit.App, error) { } func newAppWithLLMClient(client promptkit.LLMClient) (*commandkit.App, error) { - return newAppWithEngineLoader(client, loadPromptEngine) + return newAppWithEngineLoader(client, loadPromptEngine, nil) } -func newAppWithEngineLoader(client promptkit.LLMClient, engineLoader func(context.Context, ckconfig.Selection, *inspectionInput, *runCache) (*promptkit.Engine, error)) (*commandkit.App, error) { +func newAppWithEngineLoader( + client promptkit.LLMClient, + engineLoader func(context.Context, ckconfig.Selection, *inspectionInput, *runCache) (*promptkit.Engine, error), + smokeTest func(context.Context, ckprompt.SmokeRequest, *inspectionInput, *runCache) (commandkit.Acquired[*promptkit.SmokeTestReport], error), +) (*commandkit.App, error) { if client == nil { client = &exampleLLMClient{} } @@ -237,6 +244,9 @@ func newAppWithEngineLoader(client promptkit.LLMClient, engineLoader func(contex definitions.Selectors = []commandkit.Flag[*inspectionInput]{campaignSelector()} definitions = ckprompt.WithRequestValidation(definitions, engineLoader, buildValidationRequest) definitions = ckprompt.WithSmokeTesting(definitions, engineLoader, nil) + if smokeTest != nil { + definitions.SmokeTest = smokeTest + } definitionCommand := ckprompt.Command(definitions) store := map[string]session{ @@ -250,27 +260,30 @@ func newAppWithEngineLoader(client promptkit.LLMClient, engineLoader func(contex Arguments: []commandkit.Argument[restoreInput]{commandkit.ValueArgument(commandkit.ArgumentMeta{Name: "ID", Usage: "Session identifier."}, true, parseString, func(input *restoreInput, value commandkit.Supplied[string]) { input.Session = value.Value })}, Run: func(invocation commandkit.Invocation, input restoreInput) commandkit.Outcome { + reporter := invocation.Reporter() if err := invocation.Context.Err(); err != nil { - return commandkit.Failure(err) + return commandkit.Failure("Session restore was canceled.").WithCause(err) } application, err := loadApplicationConfig(invocation.Context, ckconfig.Selection{}, nil, nil) if err != nil { - return commandkit.Failure(err) + return commandkit.Failure("Could not load application configuration.").WithCause(err) } name, _, err := selectedCampaign(application.Config, input.Campaign) if err != nil { - return commandkit.Failure(err) + return commandkit.Failure(err.Error()).WithCause(err) } record, ok := store[input.Session] if !ok { - return commandkit.Outcome{Kind: commandkit.OutcomeDomain, Status: 4, Domain: "session_not_found", Cause: errors.New("saved session was not found")} + return commandkit.Outcome{Kind: commandkit.OutcomeDomain, Status: 4, Domain: "session_not_found", Cause: errSessionNotFound, Explanation: commandkit.Explanation{Message: "Saved session was not found."}} } if record.Campaign != name { - return commandkit.Outcome{Kind: commandkit.OutcomeDomain, Status: 3, Domain: "campaign_mismatch", Cause: fmt.Errorf("session belongs to campaign %q", record.Campaign)} - } - if _, err := fmt.Fprintf(invocation.Streams.Out, "restored %s: %s\n", record.ID, record.Summary); err != nil { - return commandkit.Failure(err) + cause := fmt.Errorf("session belongs to campaign %q", record.Campaign) + return commandkit.Outcome{Kind: commandkit.OutcomeDomain, Status: 3, Domain: "campaign_mismatch", Cause: cause, Explanation: commandkit.Explanation{Message: "Session campaign does not match the selected campaign."}} } + _ = reporter.RenderOutput(func(writer io.Writer) error { + _, err := fmt.Fprintf(writer, "restored %s: %s\n", record.ID, record.Summary) + return err + }) return commandkit.Success() }, }) diff --git a/integrations/ckprompt/examples/composed/main_test.go b/integrations/ckprompt/examples/composed/main_test.go index cca2c50..08c1bc9 100644 --- a/integrations/ckprompt/examples/composed/main_test.go +++ b/integrations/ckprompt/examples/composed/main_test.go @@ -3,6 +3,7 @@ package main import ( "bytes" "context" + "errors" "io" "reflect" "strings" @@ -351,6 +352,66 @@ func TestComposedCatalogsAndSmokeUseInjectedClientOnlyForInference(t *testing.T) } } +func TestComposedSmokeReportSurvivesInferenceFailure(t *testing.T) { + wantCause := errors.New("local example inference failed") + smokeTest := func(_ context.Context, request ckprompt.SmokeRequest, _ *inspectionInput, _ *runCache) (commandkit.Acquired[*promptkit.SmokeTestReport], error) { + return commandkit.Acquired[*promptkit.SmokeTestReport]{ + Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind, Completed: true}, + Diagnostics: []commandkit.Diagnostic{commandkit.MessageDiagnostic(commandkit.SeverityWarning, "independent smoke warning")}, + Explanation: commandkit.Explanation{Details: []commandkit.Detail{func(writer io.Writer) error { + _, err := io.WriteString(writer, "Native text inference context.\n") + return err + }}}, + }, wantCause + } + app, err := newAppWithEngineLoader(&exampleLLMClient{}, loadPromptEngine, smokeTest) + if err != nil { + t.Fatal(err) + } + var output, diagnostics bytes.Buffer + outcome := app.Execute(context.Background(), []string{"promptkit", "test", "text_inference", "--campaign", "spring", "--prompt", "briefing", "--profile", "spring", "--verbose"}, commandkit.Streams{Out: &output, Err: &diagnostics}) + if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || !errors.Is(outcome.Cause, wantCause) { + t.Fatalf("smoke failure outcome = %#v; report %q; diagnostics %q", outcome, output.String(), diagnostics.String()) + } + if !strings.Contains(output.String(), "kind: text_inference") || !strings.Contains(output.String(), "completed: true") { + t.Fatalf("native smoke report was not delivered: %q", output.String()) + } + warningAt := strings.Index(diagnostics.String(), "independent smoke warning") + contextAt := strings.Index(diagnostics.String(), "Native text inference context") + if warningAt < 0 || contextAt < 0 || warningAt >= contextAt || strings.Contains(diagnostics.String(), wantCause.Error()) { + t.Fatalf("terminal explanation/diagnostics = %q / %q", outcome.Explanation.Message, diagnostics.String()) + } + if outcome.Explanation.Message != "" { + t.Errorf("unexpected synthesized terminal explanation %q", outcome.Explanation.Message) + } +} + +func TestComposedCustomRestoreUsesSharedReportingAndTerminalFailure(t *testing.T) { + app, err := newApp() + if err != nil { + t.Fatal(err) + } + + var output, diagnostics bytes.Buffer + outcome := app.Execute(context.Background(), []string{"session", "restore", "session-42", "--campaign", "spring", "--quiet"}, commandkit.Streams{Out: &output, Err: &diagnostics}) + if outcome.Kind != commandkit.OutcomeSuccess || output.Len() != 0 || diagnostics.Len() != 0 { + t.Fatalf("quiet custom command outcome/output/diagnostics = %#v / %q / %q", outcome, output.String(), diagnostics.String()) + } + + output.Reset() + diagnostics.Reset() + outcome = app.Execute(context.Background(), []string{"session", "restore", "missing", "--quiet"}, commandkit.Streams{Out: &output, Err: &diagnostics}) + if outcome.Kind != commandkit.OutcomeDomain || outcome.Status != 4 || outcome.Domain != "session_not_found" || !errors.Is(outcome.Cause, errSessionNotFound) { + t.Fatalf("missing session outcome = %#v", outcome) + } + if output.Len() != 0 || outcome.Explanation.Message == "" || !strings.Contains(diagnostics.String(), outcome.Explanation.Message) { + t.Fatalf("terminal failure output/explanation/diagnostics = %q / %q / %q", output.String(), outcome.Explanation.Message, diagnostics.String()) + } + if strings.Contains(diagnostics.String(), errSessionNotFound.Error()) { + t.Errorf("terminal presentation exposed the underlying cause: %q", diagnostics.String()) + } +} + func TestComposedAliasesReachCanonicalCommands(t *testing.T) { clientCalls := 0 engineLoads := 0 @@ -358,7 +419,7 @@ func TestComposedAliasesReachCanonicalCommands(t *testing.T) { engineLoads++ return loadPromptEngine(ctx, selection, input, cache) } - app, err := newAppWithEngineLoader(&exampleLLMClient{calls: &clientCalls}, engineLoader) + app, err := newAppWithEngineLoader(&exampleLLMClient{calls: &clientCalls}, engineLoader, nil) if err != nil { t.Fatal(err) } @@ -368,6 +429,11 @@ func TestComposedAliasesReachCanonicalCommands(t *testing.T) { if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(output.String(), "--hlep → --help") || strings.Contains(output.String(), "field-notes") || engineLoads != 0 || clientCalls != 0 { t.Fatalf("group help alias outcome/output/engine loads/client calls = (%#v, %q, %d, %d)", outcome, output.String(), engineLoads, clientCalls) } + output.Reset() + outcome = app.Execute(context.Background(), []string{"pk", "--hlep", "--quiet"}, commandkit.Streams{Out: &output, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeSuccess || output.Len() != 0 || engineLoads != 0 || clientCalls != 0 { + t.Fatalf("quiet aliased help outcome/output/engine loads/client calls = (%#v, %q, %d, %d)", outcome, output.String(), engineLoads, clientCalls) + } for _, args := range [][]string{ {"list", "collections", "--campaign", "spring"},