base-layer/commandkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Migrate maintained examples to invocation reporting

Browse Source
This commit is contained in:
Eric Rakestraw
2026-10-26 03:44:19 +00:00
parent b6f22462d3
commit 6264a87183
8 changed files with 204 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
docs/roadmap/implementation.md
View File

@@ -1,6 +1,6 @@
# Invocation reporting implementation plan
Status: Stages 1–16 complete. Stage 17 is next.
Status: Stages 1–17 complete. Stage 18 is next.
## Objective and authority
@@ -642,7 +642,7 @@ explanation, then exits with status 1 as expected.
## Stage 17: Migrate maintained integration consumers
Status: pending.
Status: complete.
Primary directories:
[ckconfig example](../../integrations/ckconfig/examples/simple) and
@@ -664,6 +664,16 @@ Validation: both example packages compile in their owning modules and run
without external services. Confirm custom commands and integration commands
share the same reporting flags and terminal behavior.
Completion: migrated both example-owned handlers to invocation reporters and
explicit explanations with optional causes. The ConfigKit consumer demonstrates
a provider schema warning in verbose mode, ordinary report output, and quiet
suppression. The composed consumer retains aliased help and offline smoke usage;
its tests cover a native report with an independent provider warning, terminal
context, and an operation error, plus quiet custom-command success and domain
failure. Both integration modules pass `GOTOOLCHAIN=local go test ./...`,
including both example packages. Representative verbose ConfigKit, quiet
ConfigKit, aliased quiet help, and offline PromptKit smoke commands also pass.
## Stage 18: Reconcile the full test suite and architectural boundaries
Status: pending.

5
integrations/ckconfig/examples/simple/README.md
View File

@@ -4,6 +4,8 @@ Run the maintained example with:
```sh
go run . config show
go run . config show --config fixtures/before.yml --verbose
go run . config show --config fixtures/before.yml --quiet
go run . config diff --config fixtures/before.yml --against-config fixtures/after.yml
go run . greet
go run . greet --count=0 --tag=first --tag second
@@ -15,3 +17,6 @@ omitted `--count` and an explicitly supplied zero, plus ordered repeated tags.
With no `--config`, application settings use the in-memory default. A supplied
configuration and overlay are passed to ConfigKit unchanged. The comparison
command loads the two local fixtures and reports `changes.greeting: Goodbye`.
The before fixture uses schema version 1, which ConfigKit reports as a provider
warning in verbose mode. Ordinary mode hides the warning while preserving the
effective YAML report; quiet mode suppresses that report.

2
integrations/ckconfig/examples/simple/fixtures/after.yml
View File

@@ -1 +1,3 @@
schema:
version: 2
greeting: Goodbye

2
integrations/ckconfig/examples/simple/fixtures/before.yml
View File

@@ -1 +1,3 @@
schema:
version: 1
greeting: Hello

29
integrations/ckconfig/examples/simple/main.go
View File

@@ -3,6 +3,7 @@ package main
import (
"context"
"fmt"
"io"
"os"
"strconv"
@@ -12,6 +13,9 @@ import (
)
type appConfiguration struct {
Schema struct {
Version int `yaml:"version"`
} `yaml:"schema"`
Greeting string `yaml:"greeting"`
}
@@ -22,12 +26,25 @@ type greetInput struct {
func loadAppConfiguration(_ context.Context, selection ckconfig.Selection) (*configkit.Result[appConfiguration], error) {
source := configkit.Mapping("simple application settings", map[string]any{
"schema": map[string]any{"version": 2},
"greeting": "Hello",
})
if selection.Config.Set {
source = configkit.File(selection.Config.Value)
}
options := configkit.Options[appConfiguration]{}
options := configkit.Options[appConfiguration]{
Definition: configkit.Definition[appConfiguration]{
Version: &configkit.VersionRule{
Path: configkit.Keys("schema", "version"),
Check: func(version configkit.Version, validation *configkit.Validation) error {
if version.Text == "1" {
validation.Warning(configkit.Keys("schema", "version"), "schema version 1 is supported; upgrade to version 2 before the next example release")
}
return nil
},
},
},
}
if selection.Overlay.Set {
options.Overlay = selection.Overlay.Value
options.Filesystem = &configkit.FilesystemContext{Root: ".", BaseDir: "."}
@@ -50,9 +67,10 @@ func newApp() (*commandkit.App, error) {
}),
},
Run: func(invocation commandkit.Invocation, input greetInput) commandkit.Outcome {
reporter := invocation.Reporter()
configuration, err := loadAppConfiguration(invocation.Context, ckconfig.Selection{})
if err != nil {
return commandkit.Failure(err)
return commandkit.Failure("Could not load application configuration.").WithCause(err)
}
count := fmt.Sprintf("default %d (not supplied)", input.Count.Value)
if input.Count.Set {
@@ -62,9 +80,10 @@ func newApp() (*commandkit.App, error) {
if input.Tags.Set {
tags = fmt.Sprintf("explicit %v", input.Tags.Value)
}
if _, err := fmt.Fprintf(invocation.Streams.Out, "%s; count=%s; tags=%s\n", configuration.Config.Greeting, count, tags); err != nil {
return commandkit.Failure(err)
}
_ = reporter.RenderOutput(func(writer io.Writer) error {
_, err := fmt.Fprintf(writer, "%s; count=%s; tags=%s\n", configuration.Config.Greeting, count, tags)
return err
})
return commandkit.Success()
},
})

70
integrations/ckconfig/examples/simple/main_test.go
View File

@@ -4,6 +4,7 @@ import (
"bytes"
"context"
"io"
"strings"
"testing"
commandkit "gitea.maximumdirect.net/eric/CommandKit"
@@ -42,7 +43,7 @@ func TestSimpleAppComparesFixturesAndKeepsDefaultCommands(t *testing.T) {
output.Reset()
outcome = app.Execute(context.Background(), []string{"config", "show"}, commandkit.Streams{Out: &output, Err: io.Discard})
if outcome.Kind != commandkit.OutcomeSuccess || output.String() != "greeting: Hello\n" {
if outcome.Kind != commandkit.OutcomeSuccess || output.String() != "greeting: Hello\nschema:\n version: 2\n" {
t.Fatalf("default show outcome/output = %#v / %q", outcome, output.String())
}
@@ -60,6 +61,73 @@ func TestSimpleLoaderDoesNotReplaceExplicitEmptyConfiguration(t *testing.T) {
}
}
func TestSimpleConfigurationReportingModes(t *testing.T) {
for _, test := range []struct {
name string
args []string
wantOutput string
wantDiagnostic string
quiet bool
}{
{
name: "ordinary output hides provider warning",
args: []string{"config", "show", "--config", "fixtures/before.yml"},
wantOutput: "greeting: Hello\nschema:\n version: 1\n",
},
{
name: "quiet suppresses configuration report",
args: []string{"config", "show", "--config", "fixtures/before.yml", "--quiet"},
quiet: true,
},
{
name: "verbose shows ConfigKit warning",
args: []string{"config", "show", "--config", "fixtures/before.yml", "--verbose"},
wantOutput: "greeting: Hello\nschema:\n version: 1\n",
wantDiagnostic: "schema version 1 is supported",
},
} {
t.Run(test.name, func(t *testing.T) {
outcome, output, diagnostics := executeSimple(t, test.args...)
if outcome.Kind != commandkit.OutcomeSuccess || outcome.Cause != nil {
t.Fatalf("outcome = %#v", outcome)
}
if output != test.wantOutput {
t.Errorf("output = %q, want %q", output, test.wantOutput)
}
if test.wantDiagnostic != "" && !strings.Contains(diagnostics, test.wantDiagnostic) {
t.Errorf("diagnostics = %q, want provider warning %q", diagnostics, test.wantDiagnostic)
}
if test.wantDiagnostic == "" && diagnostics != "" {
t.Errorf("diagnostics = %q, want no warning outside verbose mode", diagnostics)
}
if strings.Contains(diagnostics, "ERROR:") {
t.Errorf("diagnostics %q unexpectedly contain a terminal failure", diagnostics)
}
if test.quiet && (outcome.Presentation.Output.State != commandkit.DeliverySuppressed || outcome.Presentation.Diagnostics.State != commandkit.DeliverySuppressed) {
t.Errorf("quiet presentation = %#v, want report and provider warning suppressed", outcome.Presentation)
}
})
}
}
func TestSimpleCustomCommandUsesSharedReportingFlags(t *testing.T) {
outcome, output, diagnostics := executeSimple(t, "greet", "--quiet")
if outcome.Kind != commandkit.OutcomeSuccess || output != "" || diagnostics != "" {
t.Fatalf("quiet greet outcome/output/diagnostics = %#v / %q / %q", outcome, output, diagnostics)
}
}
func executeSimple(t *testing.T, args ...string) (commandkit.Outcome, string, string) {
t.Helper()
app, err := newApp()
if err != nil {
t.Fatal(err)
}
var output, diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: &diagnostics})
return outcome, output.String(), diagnostics.String()
}
func field(mapping *yaml.Node, name string) *yaml.Node {
if mapping == nil || mapping.Kind != yaml.MappingNode {
return nil

33
integrations/ckprompt/examples/composed/main.go
View File

@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"io"
"os"
"strconv"
@@ -65,6 +66,8 @@ type session struct {
Summary string
}
var errSessionNotFound = errors.New("saved session was not found")
func campaignSelector() commandkit.Flag[*inspectionInput] {
return commandkit.ValueFlag(commandkit.FlagMeta{Name: "campaign", Usage: "Select campaign inputs."}, (*string)(nil), parseString,
func(input **inspectionInput, value commandkit.Supplied[string]) { (*input).Campaign = value })
@@ -200,10 +203,14 @@ func newApp() (*commandkit.App, error) {
}
func newAppWithLLMClient(client promptkit.LLMClient) (*commandkit.App, error) {
return newAppWithEngineLoader(client, loadPromptEngine)
return newAppWithEngineLoader(client, loadPromptEngine, nil)
}
func newAppWithEngineLoader(client promptkit.LLMClient, engineLoader func(context.Context, ckconfig.Selection, *inspectionInput, *runCache) (*promptkit.Engine, error)) (*commandkit.App, error) {
func newAppWithEngineLoader(
client promptkit.LLMClient,
engineLoader func(context.Context, ckconfig.Selection, *inspectionInput, *runCache) (*promptkit.Engine, error),
smokeTest func(context.Context, ckprompt.SmokeRequest, *inspectionInput, *runCache) (commandkit.Acquired[*promptkit.SmokeTestReport], error),
) (*commandkit.App, error) {
if client == nil {
client = &exampleLLMClient{}
}
@@ -237,6 +244,9 @@ func newAppWithEngineLoader(client promptkit.LLMClient, engineLoader func(contex
definitions.Selectors = []commandkit.Flag[*inspectionInput]{campaignSelector()}
definitions = ckprompt.WithRequestValidation(definitions, engineLoader, buildValidationRequest)
definitions = ckprompt.WithSmokeTesting(definitions, engineLoader, nil)
if smokeTest != nil {
definitions.SmokeTest = smokeTest
}
definitionCommand := ckprompt.Command(definitions)
store := map[string]session{
@@ -250,27 +260,30 @@ func newAppWithEngineLoader(client promptkit.LLMClient, engineLoader func(contex
Arguments: []commandkit.Argument[restoreInput]{commandkit.ValueArgument(commandkit.ArgumentMeta{Name: "ID", Usage: "Session identifier."}, true, parseString,
func(input *restoreInput, value commandkit.Supplied[string]) { input.Session = value.Value })},
Run: func(invocation commandkit.Invocation, input restoreInput) commandkit.Outcome {
reporter := invocation.Reporter()
if err := invocation.Context.Err(); err != nil {
return commandkit.Failure(err)
return commandkit.Failure("Session restore was canceled.").WithCause(err)
}
application, err := loadApplicationConfig(invocation.Context, ckconfig.Selection{}, nil, nil)
if err != nil {
return commandkit.Failure(err)
return commandkit.Failure("Could not load application configuration.").WithCause(err)
}
name, _, err := selectedCampaign(application.Config, input.Campaign)
if err != nil {
return commandkit.Failure(err)
return commandkit.Failure(err.Error()).WithCause(err)
}
record, ok := store[input.Session]
if !ok {
return commandkit.Outcome{Kind: commandkit.OutcomeDomain, Status: 4, Domain: "session_not_found", Cause: errors.New("saved session was not found")}
return commandkit.Outcome{Kind: commandkit.OutcomeDomain, Status: 4, Domain: "session_not_found", Cause: errSessionNotFound, Explanation: commandkit.Explanation{Message: "Saved session was not found."}}
}
if record.Campaign != name {
return commandkit.Outcome{Kind: commandkit.OutcomeDomain, Status: 3, Domain: "campaign_mismatch", Cause: fmt.Errorf("session belongs to campaign %q", record.Campaign)}
}
if _, err := fmt.Fprintf(invocation.Streams.Out, "restored %s: %s\n", record.ID, record.Summary); err != nil {
return commandkit.Failure(err)
cause := fmt.Errorf("session belongs to campaign %q", record.Campaign)
return commandkit.Outcome{Kind: commandkit.OutcomeDomain, Status: 3, Domain: "campaign_mismatch", Cause: cause, Explanation: commandkit.Explanation{Message: "Session campaign does not match the selected campaign."}}
}
_ = reporter.RenderOutput(func(writer io.Writer) error {
_, err := fmt.Fprintf(writer, "restored %s: %s\n", record.ID, record.Summary)
return err
})
return commandkit.Success()
},
})

68
integrations/ckprompt/examples/composed/main_test.go
View File

@@ -3,6 +3,7 @@ package main
import (
"bytes"
"context"
"errors"
"io"
"reflect"
"strings"
@@ -351,6 +352,66 @@ func TestComposedCatalogsAndSmokeUseInjectedClientOnlyForInference(t *testing.T)
}
}
func TestComposedSmokeReportSurvivesInferenceFailure(t *testing.T) {
wantCause := errors.New("local example inference failed")
smokeTest := func(_ context.Context, request ckprompt.SmokeRequest, _ *inspectionInput, _ *runCache) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
return commandkit.Acquired[*promptkit.SmokeTestReport]{
Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind, Completed: true},
Diagnostics: []commandkit.Diagnostic{commandkit.MessageDiagnostic(commandkit.SeverityWarning, "independent smoke warning")},
Explanation: commandkit.Explanation{Details: []commandkit.Detail{func(writer io.Writer) error {
_, err := io.WriteString(writer, "Native text inference context.\n")
return err
}}},
}, wantCause
}
app, err := newAppWithEngineLoader(&exampleLLMClient{}, loadPromptEngine, smokeTest)
if err != nil {
t.Fatal(err)
}
var output, diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), []string{"promptkit", "test", "text_inference", "--campaign", "spring", "--prompt", "briefing", "--profile", "spring", "--verbose"}, commandkit.Streams{Out: &output, Err: &diagnostics})
if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || !errors.Is(outcome.Cause, wantCause) {
t.Fatalf("smoke failure outcome = %#v; report %q; diagnostics %q", outcome, output.String(), diagnostics.String())
}
if !strings.Contains(output.String(), "kind: text_inference") || !strings.Contains(output.String(), "completed: true") {
t.Fatalf("native smoke report was not delivered: %q", output.String())
}
warningAt := strings.Index(diagnostics.String(), "independent smoke warning")
contextAt := strings.Index(diagnostics.String(), "Native text inference context")
if warningAt < 0 || contextAt < 0 || warningAt >= contextAt || strings.Contains(diagnostics.String(), wantCause.Error()) {
t.Fatalf("terminal explanation/diagnostics = %q / %q", outcome.Explanation.Message, diagnostics.String())
}
if outcome.Explanation.Message != "" {
t.Errorf("unexpected synthesized terminal explanation %q", outcome.Explanation.Message)
}
}
func TestComposedCustomRestoreUsesSharedReportingAndTerminalFailure(t *testing.T) {
app, err := newApp()
if err != nil {
t.Fatal(err)
}
var output, diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), []string{"session", "restore", "session-42", "--campaign", "spring", "--quiet"}, commandkit.Streams{Out: &output, Err: &diagnostics})
if outcome.Kind != commandkit.OutcomeSuccess || output.Len() != 0 || diagnostics.Len() != 0 {
t.Fatalf("quiet custom command outcome/output/diagnostics = %#v / %q / %q", outcome, output.String(), diagnostics.String())
}
output.Reset()
diagnostics.Reset()
outcome = app.Execute(context.Background(), []string{"session", "restore", "missing", "--quiet"}, commandkit.Streams{Out: &output, Err: &diagnostics})
if outcome.Kind != commandkit.OutcomeDomain || outcome.Status != 4 || outcome.Domain != "session_not_found" || !errors.Is(outcome.Cause, errSessionNotFound) {
t.Fatalf("missing session outcome = %#v", outcome)
}
if output.Len() != 0 || outcome.Explanation.Message == "" || !strings.Contains(diagnostics.String(), outcome.Explanation.Message) {
t.Fatalf("terminal failure output/explanation/diagnostics = %q / %q / %q", output.String(), outcome.Explanation.Message, diagnostics.String())
}
if strings.Contains(diagnostics.String(), errSessionNotFound.Error()) {
t.Errorf("terminal presentation exposed the underlying cause: %q", diagnostics.String())
}
}
func TestComposedAliasesReachCanonicalCommands(t *testing.T) {
clientCalls := 0
engineLoads := 0
@@ -358,7 +419,7 @@ func TestComposedAliasesReachCanonicalCommands(t *testing.T) {
engineLoads++
return loadPromptEngine(ctx, selection, input, cache)
}
app, err := newAppWithEngineLoader(&exampleLLMClient{calls: &clientCalls}, engineLoader)
app, err := newAppWithEngineLoader(&exampleLLMClient{calls: &clientCalls}, engineLoader, nil)
if err != nil {
t.Fatal(err)
}
@@ -368,6 +429,11 @@ func TestComposedAliasesReachCanonicalCommands(t *testing.T) {
if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(output.String(), "--hlep → --help") || strings.Contains(output.String(), "field-notes") || engineLoads != 0 || clientCalls != 0 {
t.Fatalf("group help alias outcome/output/engine loads/client calls = (%#v, %q, %d, %d)", outcome, output.String(), engineLoads, clientCalls)
}
output.Reset()
outcome = app.Execute(context.Background(), []string{"pk", "--hlep", "--quiet"}, commandkit.Streams{Out: &output, Err: io.Discard})
if outcome.Kind != commandkit.OutcomeSuccess || output.Len() != 0 || engineLoads != 0 || clientCalls != 0 {
t.Fatalf("quiet aliased help outcome/output/engine loads/client calls = (%#v, %q, %d, %d)", outcome, output.String(), engineLoads, clientCalls)
}
for _, args := range [][]string{
{"list", "collections", "--campaign", "spring"},