commandkit/performance_dispatch_test.go

package commandkit_test

import (
	"context"
	"errors"
	"fmt"
	"io"
	"testing"

	commandkit "gitea.maximumdirect.net/eric/CommandKit"
)

var performanceUsageCause = errors.New("benchmark usage result")

func BenchmarkPerformanceDispatch(b *testing.B) {
	small := performanceFlatCommands(b, 8, "cmd")
	wide := performanceFlatCommands(b, 128, "wide")
	deep, deepPath := performanceNestedCommand(b)
	smallApp := performanceApp(b, small)
	wideApp := performanceApp(b, wide)
	deepApp := performanceApp(b, []commandkit.Command{deep})
	streams := commandkit.Streams{Out: io.Discard, Err: io.Discard}

	benchmarks := []struct {
		name string
		app  *commandkit.App
		args []string
		kind commandkit.OutcomeKind
	}{
		{name: "success", app: smallApp, args: []string{"cmd00"}, kind: commandkit.OutcomeSuccess},
		{name: "local_help", app: smallApp, args: []string{"cmd01", "--help"}, kind: commandkit.OutcomeSuccess},
		{name: "explicit_help", app: smallApp, args: []string{"help", "cmd02"}, kind: commandkit.OutcomeSuccess},
		{name: "parser_usage", app: smallApp, args: []string{"cmd03", "--unknown"}, kind: commandkit.OutcomeUsage},
		{name: "handler_usage", app: smallApp, args: []string{"cmd04"}, kind: commandkit.OutcomeUsage},
		{name: "wide_success", app: wideApp, args: []string{"wide127"}, kind: commandkit.OutcomeSuccess},
		{name: "wide_help", app: wideApp, args: []string{"wide127", "--help"}, kind: commandkit.OutcomeSuccess},
		{name: "deep_success", app: deepApp, args: deepPath, kind: commandkit.OutcomeSuccess},
		{name: "deep_help", app: deepApp, args: append(append([]string(nil), deepPath...), "--help"), kind: commandkit.OutcomeSuccess},
	}
	for _, benchmark := range benchmarks {
		b.Run(benchmark.name, func(b *testing.B) {
			verifyPerformanceOutcome(b, benchmark.app.Execute(context.Background(), benchmark.args, streams), benchmark.kind)
			b.ReportAllocs()
			b.ResetTimer()
			for i := 0; i < b.N; i++ {
				verifyPerformanceOutcome(b, benchmark.app.Execute(context.Background(), benchmark.args, streams), benchmark.kind)
			}
		})
	}

	flagArgs := []string{"cmd07"}
	for index := 0; index < 24; index++ {
		flagArgs = append(flagArgs, fmt.Sprintf("--flag%02d=value", index))
	}
	b.Run("flag_heavy_success", func(b *testing.B) {
		verifyPerformanceOutcome(b, smallApp.Execute(context.Background(), flagArgs, streams), commandkit.OutcomeSuccess)
		b.ReportAllocs()
		b.ResetTimer()
		for i := 0; i < b.N; i++ {
			verifyPerformanceOutcome(b, smallApp.Execute(context.Background(), flagArgs, streams), commandkit.OutcomeSuccess)
		}
	})

	for _, fixture := range []struct {
		name     string
		commands []commandkit.Command
	}{
		{name: "small", commands: small},
		{name: "wide", commands: wide},
	} {
		b.Run("construction_first_execution/"+fixture.name, func(b *testing.B) {
			b.ReportAllocs()
			for i := 0; i < b.N; i++ {
				app := performanceApp(b, fixture.commands)
				verifyPerformanceOutcome(b, app.Execute(context.Background(), []string{performanceCommandName(fixture.name)}, streams), commandkit.OutcomeSuccess)
			}
		})
	}
}

func BenchmarkPerformanceRegistration(b *testing.B) {
	for _, count := range []int{8, 128} {
		commands := performanceFlatCommands(b, count, "register")
		b.Run(fmt.Sprintf("individual/%d", count), func(b *testing.B) {
			b.ReportAllocs()
			for i := 0; i < b.N; i++ {
				app, err := commandkit.New(commandkit.Identity{Name: "bench"})
				if err != nil {
					b.Fatal(err)
				}
				for _, command := range commands {
					if err := app.Register(command); err != nil {
						b.Fatal(err)
					}
				}
			}
		})
		b.Run(fmt.Sprintf("batch/%d", count), func(b *testing.B) {
			b.ReportAllocs()
			for i := 0; i < b.N; i++ {
				app, err := commandkit.New(commandkit.Identity{Name: "bench"})
				if err != nil {
					b.Fatal(err)
				}
				if err := app.Register(commands...); err != nil {
					b.Fatal(err)
				}
			}
		})
	}
}

func performanceFlatCommands(tb testing.TB, count int, prefix string) []commandkit.Command {
	tb.Helper()
	commands := make([]commandkit.Command, 0, count)
	for index := 0; index < count; index++ {
		name := fmt.Sprintf("%s%02d", prefix, index)
		flags := []commandkit.Flag[struct{}](nil)
		if prefix == "cmd" && index == 7 {
			for flagIndex := 0; flagIndex < 24; flagIndex++ {
				flags = append(flags, commandkit.ValueFlag(commandkit.FlagMeta{Name: fmt.Sprintf("flag%02d", flagIndex)}, (*string)(nil), func(value string) (string, error) { return value, nil }, func(*struct{}, commandkit.Supplied[string]) {}))
			}
		}
		run := func(commandkit.Invocation, struct{}) commandkit.Outcome { return commandkit.Success() }
		if prefix == "cmd" && index == 4 {
			run = func(commandkit.Invocation, struct{}) commandkit.Outcome {
				return commandkit.Usage(performanceUsageCause)
			}
		}
		command := commandkit.NewCommand(commandkit.CommandSpec[struct{}]{Name: name, Summary: "Benchmark command.", Flags: flags, Run: run})
		commands = append(commands, command)
	}
	return commands
}

func performanceNestedCommand(tb testing.TB) (commandkit.Command, []string) {
	tb.Helper()
	path := make([]string, 24)
	for index := range path {
		path[index] = fmt.Sprintf("level%02d", index)
	}
	leaf := commandkit.NewCommand(commandkit.CommandSpec[struct{}]{Name: path[len(path)-1], Summary: "Nested benchmark command.", Run: func(commandkit.Invocation, struct{}) commandkit.Outcome { return commandkit.Success() }})
	current := leaf
	for index := len(path) - 2; index >= 0; index-- {
		sibling := commandkit.NewCommand(commandkit.CommandSpec[struct{}]{Name: fmt.Sprintf("sibling%02d", index), Summary: "Sibling command.", Run: func(commandkit.Invocation, struct{}) commandkit.Outcome { return commandkit.Success() }})
		current = commandkit.Group(path[index], "Nested benchmark group.", current, sibling)
	}
	return current, path
}

func performanceApp(tb testing.TB, commands []commandkit.Command) *commandkit.App {
	tb.Helper()
	app, err := commandkit.New(commandkit.Identity{Name: "bench"}, commands...)
	if err != nil {
		tb.Fatal(err)
	}
	return app
}

func performanceCommandName(fixture string) string {
	if fixture == "wide" {
		return "wide00"
	}
	return "cmd00"
}

func verifyPerformanceOutcome(tb testing.TB, outcome commandkit.Outcome, expected commandkit.OutcomeKind) {
	tb.Helper()
	if outcome.Kind != expected {
		tb.Fatalf("outcome kind = %v, want %v (cause %v)", outcome.Kind, expected, outcome.Cause)
	}
}