diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 3729d15..b009de0 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1,6 +1,6 @@ # Performance implementation plan -Status: ready for implementation; Stages 1–11 have not started. +Status: implementation in progress; Stage 1 is complete, Stages 2–11 have not started. Date: 2026-10-09. @@ -150,7 +150,7 @@ benchmark-analysis dependency or permanent numeric pass threshold is selected. ## Stage 1: establish dispatch and registration measurements -**Status:** not started. **Prerequisite:** orientation and this plan. +**Status:** complete. **Prerequisite:** orientation and this plan. **Files:** new `performance_dispatch_test.go`, `internal/cobra/performance_test.go`, and evidence here. Production code stays @@ -182,7 +182,49 @@ production instrumentation, or full dependency test matrix is needed. **Completion:** reproducible baseline results distinguish registration, cold execution, repeated execution, parsing, and help/usage costs. -**Evidence:** pending. +**Evidence:** Benchmark fixtures were measured from source revision `1274ae7` +with the new benchmark files in the working tree (committed with this evidence). +Environment: Go 1.26.8, linux/amd64, Linux 6.17.0-41-generic, Intel Core +i5-7260U CPU @ 2.20 GHz. The required compile-only command passed: +`GOWORK=off GOTOOLCHAIN=local GOPROXY=off GONOPROXY=none GOSUMDB=off go test -mod=readonly -run '^$' . ./internal/cobra`. +The five-sample measurement command also passed: +`GOWORK=off GOTOOLCHAIN=local GOPROXY=off GONOPROXY=none GOSUMDB=off go test -mod=readonly -run '^$' -bench '^BenchmarkPerformance' -benchmem -benchtime=200ms -count=5 . ./internal/cobra`. +Raw output is in `/tmp/commandkit-performance-baseline.txt`; focused repeat output +for noisy cases is in `/tmp/commandkit-performance-baseline-repeat.txt`. + +Fixtures: root dispatch used 8 and 128 flat commands, a 24-level path with +siblings at each parent, and a leaf with 24 flags. Dispatch covered success, +local and explicit help, parser and handler usage, cold construction plus first +execution, and repeated execution. Registration used 8 and 128 prebuilt +declarations with fresh `New` in individual and batch paths. Adapter parsing +used 128 shallow siblings and the same depth-24 path. All repeated app and +adapter fixtures were built outside the timed loops; benchmark outcomes were +checked for their intended classes. + +Representative five-sample ranges (ns/op; B/op; allocs/op): small repeated +success 31,066–41,041; 33,209–33,210; 287. Small local help 103,022–116,183; +78,176–78,177; 810. Parser usage 62,596–74,475; 66,363–66,364; 591. Wide +success 218,073–253,010; 269,063–269,070; 1,898. Deep success 119,958–139,625; +139,787–139,791; 1,074. Wide help 491,172–622,188; 549,923–549,940; 4,033. +Explicit help measured 101,998–111,895 ns/op, 78,152–78,153 B/op, and 815 +allocs/op; handler usage measured 61,878–72,264 ns/op, 66,211–66,212 B/op, and +583 allocs/op. Cold construction plus first execution measured 38,152–50,235 +ns/op, 38,275–38,276 B/op, and 339 allocs/op for the small app, and +251,945–315,276 ns/op, 291,822–291,827 B/op, and 2,184 allocs/op for the wide +app. The 24-flag leaf focused repeat was 43,662–56,492 ns/op, 40,027–40,028 +B/op, and 401 allocs/op. +Deep help was noisy in the full run (429,873–1,634,363 ns/op); a focused repeat +was 300,155–326,094 ns/op, 294,882–294,896 B/op, and 2,408 allocs/op. The initial +flag-heavy timing also varied widely before the focused range reported above. +Registration at 8 declarations measured individual 7,141–7,645 ns/op, +5,497 B/op, 64 allocs/op versus batch 4,159–5,190 ns/op, 3,456 B/op, +48 allocs/op. At 128 declarations, individual measured 414,618–492,261 ns/op, +405,011–405,018 B/op, 784 allocs/op versus batch 33,985–36,207 ns/op, +23,013–23,014 B/op, 288 allocs/op. Adapter parse measured shallow/wide +150,918–171,598 ns/op, 242,936 B/op, 1,705 allocs/op, and deep +86,282–109,697 ns/op, 119,920 B/op, 827 allocs/op. These are baselines, not +speedup claims; the large initial deep-help and flag-heavy timing variation +shows why later comparisons should report run variation. ## Stage 2: establish report and scope measurements diff --git a/internal/cobra/performance_test.go b/internal/cobra/performance_test.go new file mode 100644 index 0000000..84ead94 --- /dev/null +++ b/internal/cobra/performance_test.go @@ -0,0 +1,62 @@ +package cobra + +import ( + "context" + "fmt" + "io" + "testing" +) + +func BenchmarkPerformanceParse(b *testing.B) { + wide := performanceFlatTree(128) + wideArgs := []string{"wide127"} + deep, deepArgs := performanceNestedTree(24) + options := ParseOptions{Context: context.Background(), Out: io.Discard, Err: io.Discard} + + for _, benchmark := range []struct { + name string + root *Command + args []string + }{ + {name: "shallow_wide", root: wide, args: wideArgs}, + {name: "deep", root: deep, args: deepArgs}, + } { + b.Run(benchmark.name, func(b *testing.B) { + result := Parse(benchmark.root, benchmark.args, options) + if result.Failure != nil { + b.Fatalf("fixture parse failed: %s", result.Failure.Message) + } + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + result := Parse(benchmark.root, benchmark.args, options) + if result.Failure != nil { + b.Fatalf("parse failed: %s", result.Failure.Message) + } + } + }) + } +} + +func performanceFlatTree(count int) *Command { + root := &Command{Name: "app", Children: make([]*Command, count)} + for index := range root.Children { + root.Children[index] = &Command{Name: fmt.Sprintf("wide%03d", index), Runnable: true} + } + return root +} + +func performanceNestedTree(depth int) (*Command, []string) { + path := make([]string, depth) + for index := range path { + path[index] = fmt.Sprintf("level%02d", index) + } + current := &Command{Name: path[depth-1], Runnable: true} + for index := depth - 2; index >= 0; index-- { + current = &Command{Name: path[index], Children: []*Command{ + current, + {Name: fmt.Sprintf("sibling%02d", index), Runnable: true}, + }} + } + return &Command{Name: "app", Children: []*Command{current}}, path +} diff --git a/performance_dispatch_test.go b/performance_dispatch_test.go new file mode 100644 index 0000000..a8b8db3 --- /dev/null +++ b/performance_dispatch_test.go @@ -0,0 +1,184 @@ +package commandkit_test + +import ( + "context" + "errors" + "fmt" + "io" + "testing" + + commandkit "gitea.maximumdirect.net/eric/CommandKit" +) + +var performanceUsageCause = errors.New("benchmark usage result") + +func BenchmarkPerformanceDispatch(b *testing.B) { + small := performanceFlatCommands(b, 8, "cmd") + wide := performanceFlatCommands(b, 128, "wide") + deep, deepPath := performanceNestedCommand(b) + smallApp := performanceApp(b, small) + wideApp := performanceApp(b, wide) + deepApp := performanceApp(b, []commandkit.Command{deep}) + streams := commandkit.Streams{Out: io.Discard, Err: io.Discard} + + benchmarks := []struct { + name string + app *commandkit.App + args []string + kind commandkit.OutcomeKind + }{ + {name: "success", app: smallApp, args: []string{"cmd00"}, kind: commandkit.OutcomeSuccess}, + {name: "local_help", app: smallApp, args: []string{"cmd01", "--help"}, kind: commandkit.OutcomeSuccess}, + {name: "explicit_help", app: smallApp, args: []string{"help", "cmd02"}, kind: commandkit.OutcomeSuccess}, + {name: "parser_usage", app: smallApp, args: []string{"cmd03", "--unknown"}, kind: commandkit.OutcomeUsage}, + {name: "handler_usage", app: smallApp, args: []string{"cmd04"}, kind: commandkit.OutcomeUsage}, + {name: "wide_success", app: wideApp, args: []string{"wide127"}, kind: commandkit.OutcomeSuccess}, + {name: "wide_help", app: wideApp, args: []string{"wide127", "--help"}, kind: commandkit.OutcomeSuccess}, + {name: "deep_success", app: deepApp, args: deepPath, kind: commandkit.OutcomeSuccess}, + {name: "deep_help", app: deepApp, args: append(append([]string(nil), deepPath...), "--help"), kind: commandkit.OutcomeSuccess}, + } + for _, benchmark := range benchmarks { + b.Run(benchmark.name, func(b *testing.B) { + verifyPerformanceOutcome(b, benchmark.app.Execute(context.Background(), benchmark.args, streams), benchmark.kind) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + verifyPerformanceOutcome(b, benchmark.app.Execute(context.Background(), benchmark.args, streams), benchmark.kind) + } + }) + } + + flagArgs := []string{"cmd07"} + for index := 0; index < 24; index++ { + flagArgs = append(flagArgs, fmt.Sprintf("--flag%02d=value", index)) + } + b.Run("flag_heavy_success", func(b *testing.B) { + verifyPerformanceOutcome(b, smallApp.Execute(context.Background(), flagArgs, streams), commandkit.OutcomeSuccess) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + verifyPerformanceOutcome(b, smallApp.Execute(context.Background(), flagArgs, streams), commandkit.OutcomeSuccess) + } + }) + + for _, fixture := range []struct { + name string + commands []commandkit.Command + }{ + {name: "small", commands: small}, + {name: "wide", commands: wide}, + } { + b.Run("construction_first_execution/"+fixture.name, func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + app := performanceApp(b, fixture.commands) + verifyPerformanceOutcome(b, app.Execute(context.Background(), []string{performanceCommandName(fixture.name)}, streams), commandkit.OutcomeSuccess) + } + }) + } +} + +func BenchmarkPerformanceRegistration(b *testing.B) { + for _, count := range []int{8, 128} { + commands := performanceFlatCommands(b, count, "register") + b.Run(fmt.Sprintf("individual/%d", count), func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + app, err := commandkit.New(commandkit.Identity{Name: "bench"}) + if err != nil { + b.Fatal(err) + } + for _, command := range commands { + if err := app.Register(command); err != nil { + b.Fatal(err) + } + } + } + }) + b.Run(fmt.Sprintf("batch/%d", count), func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + app, err := commandkit.New(commandkit.Identity{Name: "bench"}) + if err != nil { + b.Fatal(err) + } + if err := app.Register(commands...); err != nil { + b.Fatal(err) + } + } + }) + } +} + +func performanceFlatCommands(tb testing.TB, count int, prefix string) []commandkit.Command { + tb.Helper() + commands := make([]commandkit.Command, 0, count) + for index := 0; index < count; index++ { + name := fmt.Sprintf("%s%02d", prefix, index) + flags := []commandkit.Flag[struct{}](nil) + if prefix == "cmd" && index == 7 { + for flagIndex := 0; flagIndex < 24; flagIndex++ { + flags = append(flags, commandkit.ValueFlag(commandkit.FlagMeta{Name: fmt.Sprintf("flag%02d", flagIndex)}, (*string)(nil), func(value string) (string, error) { return value, nil }, func(*struct{}, commandkit.Supplied[string]) {})) + } + } + run := func(commandkit.Invocation, struct{}) commandkit.Outcome { return commandkit.Success() } + if prefix == "cmd" && index == 4 { + run = func(commandkit.Invocation, struct{}) commandkit.Outcome { + return commandkit.Usage(performanceUsageCause) + } + } + command, err := commandkit.NewCommand(commandkit.CommandSpec[struct{}]{Name: name, Summary: "Benchmark command.", Flags: flags, Run: run}) + if err != nil { + tb.Fatal(err) + } + commands = append(commands, command) + } + return commands +} + +func performanceNestedCommand(tb testing.TB) (commandkit.Command, []string) { + tb.Helper() + path := make([]string, 24) + for index := range path { + path[index] = fmt.Sprintf("level%02d", index) + } + leaf, err := commandkit.NewCommand(commandkit.CommandSpec[struct{}]{Name: path[len(path)-1], Summary: "Nested benchmark command.", Run: func(commandkit.Invocation, struct{}) commandkit.Outcome { return commandkit.Success() }}) + if err != nil { + tb.Fatal(err) + } + current := leaf + for index := len(path) - 2; index >= 0; index-- { + sibling, err := commandkit.NewCommand(commandkit.CommandSpec[struct{}]{Name: fmt.Sprintf("sibling%02d", index), Summary: "Sibling command.", Run: func(commandkit.Invocation, struct{}) commandkit.Outcome { return commandkit.Success() }}) + if err != nil { + tb.Fatal(err) + } + current, err = commandkit.Group(path[index], "Nested benchmark group.", current, sibling) + if err != nil { + tb.Fatal(err) + } + } + return current, path +} + +func performanceApp(tb testing.TB, commands []commandkit.Command) *commandkit.App { + tb.Helper() + app, err := commandkit.New(commandkit.Identity{Name: "bench"}, commands...) + if err != nil { + tb.Fatal(err) + } + return app +} + +func performanceCommandName(fixture string) string { + if fixture == "wide" { + return "wide00" + } + return "cmd00" +} + +func verifyPerformanceOutcome(tb testing.TB, outcome commandkit.Outcome, expected commandkit.OutcomeKind) { + tb.Helper() + if outcome.Kind != expected { + tb.Fatalf("outcome kind = %v, want %v (cause %v)", outcome.Kind, expected, outcome.Cause) + } +}