From a431540e190b82dc43dc5eff82ec6682278a6fa2 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Fri, 9 Oct 2026 01:23:35 +0000 Subject: [PATCH] Add report and scope benchmarks --- docs/roadmap/implementation.md | 56 +++++- performance_reports_test.go | 352 +++++++++++++++++++++++++++++++++ 2 files changed, 405 insertions(+), 3 deletions(-) create mode 100644 performance_reports_test.go diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index b009de0..f9fcd6b 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1,6 +1,6 @@ # Performance implementation plan -Status: implementation in progress; Stage 1 is complete, Stages 2–11 have not started. +Status: implementation in progress; Stages 1–2 are complete, Stages 3–11 have not started. Date: 2026-10-09. @@ -228,7 +228,7 @@ shows why later comparisons should report run variation. ## Stage 2: establish report and scope measurements -**Status:** not started. **Prerequisite:** Stage 1. +**Status:** complete. **Prerequisite:** Stage 1. **Files:** new `performance_reports_test.go` and evidence here. Production code stays unchanged in this stage. @@ -258,7 +258,57 @@ owner of actual library loading semantics. **Completion:** PF-01, PF-04, and PF-06 have comparable baseline measurements. -**Evidence:** pending. +**Evidence:** Fixtures were measured from source revision `aed4c59` with +`performance_reports_test.go` in the working tree (committed with this evidence). +Environment matched the first baseline: Go 1.26.8, linux/amd64, Linux +6.17.0-41-generic, Intel Core i5-7260U CPU @ 2.20 GHz. Root package compilation +passed with `GOWORK=off GOTOOLCHAIN=local GOPROXY=off GONOPROXY=none GOSUMDB=off go test -mod=readonly -run '^$' .`. The five-sample benchmark command passed: +`GOWORK=off GOTOOLCHAIN=local GOPROXY=off GONOPROXY=none GOSUMDB=off go test -mod=readonly -run '^$' -bench '^BenchmarkPerformance(PromptReports|YAMLFraming|ConfigScopes|ConfigRegistration)$' -benchmem -benchtime=200ms -count=5 .`. +Raw output is in `/tmp/commandkit-performance-reports-baseline.txt`. + +Fixtures: PromptKit callbacks returned prebuilt inspection records directly; +prompt reports used 4 and 128 input records, while profile `ExtraParams` used +2 groups with 3 nested values each or 24 groups with 8 nested values each and a +nested sequence. Each was measured as text and YAML through `App.Execute`. +Native YAML fixtures contain unique short mapping keys and are 4,096 bytes with +an ordinary comment preamble and 1,048,591 bytes with a `%YAML` directive +preamble. Framing was measured directly; aggregate `config show --scope all` +measured two documents of each selected fixture size/style, and single-scope +display used the 4 KiB ordinary document. Scope operations used 2 or 32 scopes +with the requested show/validate capabilities and cheap in-memory acquisition +callbacks. Suite registration used 4 or 64 distinct selectors and a fresh app. +The scope counts also covered discovery, named validation, and aggregate +validation. Setup verified success and report shape outside timed loops; native +fixtures, records, suites, and apps were prepared outside repeated loops. + +Representative five-sample ranges (ns/op; B/op; allocs/op): prompt, 4 inputs, +text 41,287–52,115; 26,160–26,161; 294, and YAML 88,991–104,671; +74,730–74,733; 557. Prompt, 128 inputs, text 169,155–198,281; +145,524–145,531; 1,171, and YAML 1,418,692–1,490,181; 1,393,508–1,393,568; +6,039–6,040. Profile, small extra map, text 74,109–89,337; 70,914–70,918; +462, and YAML 161,470–368,813; 162,688–162,717; 755. Profile, large extra map, +text 1,277,396–2,583,950; 836,141–836,296; 3,027–3,028, and YAML +1,756,571–2,377,499; 1,586,728–1,586,849; 5,878–5,879. Larger profile timings +varied, while allocation counts and bytes were stable. + +Direct framing measured 4 KiB ordinary at 5,883–6,842 ns/op, 11,056 B/op, +3 allocs/op, and 1 MiB directive at 1,208,678–1,502,417 ns/op, +2,539,520–2,539,523 B/op, 2 allocs/op. Aggregate show measured 4 KiB documents +at 44,395–48,161 ns/op, 55,537–55,538 B/op, 211 allocs/op, and 1 MiB documents +at 4,818,907–6,009,877 ns/op, 8,695,182–8,695,197 B/op, 215–216 allocs/op. +Single-scope 4 KiB display measured 20,229–21,791 ns/op, 17,935–17,936 B/op, +183 allocs/op. With 2 scopes, discovery measured 23,840–30,007 ns/op, +21,161–21,162 B/op, 223 allocs/op; named validation 24,841–26,340 ns/op, +21,817 B/op, 228 allocs/op; aggregate validation 25,231–31,126 ns/op, +22,361–22,362 B/op, 237 allocs/op. With 32 scopes, discovery measured +31,439–37,167 ns/op, 28,141–28,142 B/op, 257 allocs/op; named validation +29,123–74,374 ns/op, 22,775–22,778 B/op, 228 allocs/op; aggregate validation +45,548–63,471 ns/op, 29,781–29,782 B/op, 269 allocs/op. Registration with 4 +selectors measured 12,573–14,789 ns/op, 8,911 B/op, 104 allocs/op; with 64 it +measured 91,675–105,548 ns/op, 82,817–82,818 B/op, 485 allocs/op. Large-report +cost includes full output framing/delivery; no segmented-delivery comparison +was selected. These measurements establish baselines and show timing variation; +they are not fixed thresholds or speedup claims. ## Stage 3: render only the selected PromptKit format diff --git a/performance_reports_test.go b/performance_reports_test.go new file mode 100644 index 0000000..7d5bae5 --- /dev/null +++ b/performance_reports_test.go @@ -0,0 +1,352 @@ +package commandkit + +import ( + "bytes" + "context" + "fmt" + "io" + "strings" + "testing" + + promptkit "gitea.maximumdirect.net/eric/promptkit" +) + +var performanceFramedBytes []byte + +func BenchmarkPerformancePromptReports(b *testing.B) { + for _, inputCount := range []int{4, 128} { + inspection := performancePromptInspection(inputCount) + app := performancePromptApp(b, inspection, performanceProfileInspection(nil)) + for _, format := range []string{"text", "yaml"} { + name := fmt.Sprintf("prompt/inputs_%d/%s", inputCount, format) + args := []string{"promptkit", "show", "prompt", "briefing", "--format", format} + b.Run(name, func(b *testing.B) { + verifyPromptReport(b, app, args, format, "prompt_id") + streams := Streams{Out: io.Discard, Err: io.Discard} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if outcome := app.Execute(context.Background(), args, streams); outcome.Kind != OutcomeSuccess { + b.Fatalf("prompt report outcome = %#v", outcome) + } + } + }) + } + } + + for _, fixture := range []struct { + name string + extra map[string]any + }{ + {name: "small", extra: performanceExtraParams(2, 3)}, + {name: "large", extra: performanceExtraParams(24, 8)}, + } { + profile := performanceProfileInspection(fixture.extra) + app := performancePromptApp(b, performancePromptInspection(4), profile) + for _, format := range []string{"text", "yaml"} { + name := fmt.Sprintf("profile/extra_%s/%s", fixture.name, format) + args := []string{"promptkit", "show", "profile", "baseline", "--format", format} + b.Run(name, func(b *testing.B) { + verifyPromptReport(b, app, args, format, "profile_id") + streams := Streams{Out: io.Discard, Err: io.Discard} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if outcome := app.Execute(context.Background(), args, streams); outcome.Kind != OutcomeSuccess { + b.Fatalf("profile report outcome = %#v", outcome) + } + } + }) + } + } +} + +func BenchmarkPerformanceYAMLFraming(b *testing.B) { + fixtures := []struct { + name string + scope string + native []byte + }{ + {name: "4k_ordinary", scope: "app", native: performanceNativeDocument(4<<10, false)}, + {name: "1m_directive", scope: "promptkit", native: performanceNativeDocument(1<<20, true)}, + } + for _, fixture := range fixtures { + b.Run("framing/"+fixture.name, func(b *testing.B) { + framed := frameEffectiveYAML(fixture.scope, fixture.native) + if len(framed) <= len(fixture.native) || hasYAMLDirective(fixture.native) != strings.Contains(fixture.name, "directive") { + b.Fatalf("framing fixture result has unexpected size or directive classification: native=%d framed=%d", len(fixture.native), len(framed)) + } + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + performanceFramedBytes = frameEffectiveYAML(fixture.scope, fixture.native) + } + }) + b.Run("aggregate/"+fixture.name, func(b *testing.B) { + app := performanceShowApp(b, fixture.native, fixture.native) + args := []string{"config", "show", "--scope", "all"} + verifyConfigReport(b, app, args, len(fixture.native)*2) + streams := Streams{Out: io.Discard, Err: io.Discard} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if outcome := app.Execute(context.Background(), args, streams); outcome.Kind != OutcomeSuccess { + b.Fatalf("aggregate config show outcome = %#v", outcome) + } + } + }) + } + + single := performanceNativeDocument(4<<10, false) + app := performanceShowApp(b, single) + args := []string{"config", "show"} + b.Run("single_scope/4k_ordinary", func(b *testing.B) { + verifyConfigReport(b, app, args, len(single)) + streams := Streams{Out: io.Discard, Err: io.Discard} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if outcome := app.Execute(context.Background(), args, streams); outcome.Kind != OutcomeSuccess { + b.Fatalf("single-scope config show outcome = %#v", outcome) + } + } + }) +} + +func BenchmarkPerformanceConfigScopes(b *testing.B) { + for _, scopeCount := range []int{2, 32} { + suite := performanceConfigSuite(scopeCount, []ConfigOperation{ConfigValidate, ConfigShow}) + app := performanceConfigApp(b, suite) + selected := fmt.Sprintf("scope%02d", scopeCount-1) + for _, operation := range []struct { + name string + args []string + }{ + {name: "discovery", args: []string{"config", "scopes"}}, + {name: "named_selection", args: []string{"config", "validate", "--scope", selected}}, + {name: "aggregate_validation", args: []string{"config", "validate", "--scope", "all"}}, + } { + b.Run(fmt.Sprintf("%s/scopes_%d", operation.name, scopeCount), func(b *testing.B) { + verifyConfigReport(b, app, operation.args, 1) + streams := Streams{Out: io.Discard, Err: io.Discard} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if outcome := app.Execute(context.Background(), operation.args, streams); outcome.Kind != OutcomeSuccess { + b.Fatalf("configuration operation outcome = %#v", outcome) + } + } + }) + } + } +} + +func BenchmarkPerformanceConfigRegistration(b *testing.B) { + for _, selectorCount := range []int{4, 64} { + suite := performanceSelectorSuite(selectorCount) + performanceConfigApp(b, suite) + b.Run(fmt.Sprintf("selectors_%d", selectorCount), func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + app, err := New(Identity{Name: "bench"}) + if err != nil { + b.Fatal(err) + } + if err := RegisterConfig(app, suite); err != nil { + b.Fatal(err) + } + } + }) + } +} + +func performancePromptInspection(inputCount int) *promptkit.PromptInspection { + inputs := make([]promptkit.PromptInputDefinition, inputCount) + for index := range inputs { + inputs[index] = promptkit.PromptInputDefinition{ + Name: fmt.Sprintf("input%03d", index), + Required: index%2 == 0, + ContentType: "text/plain", + Description: "A representative prompt input.", + } + } + return &promptkit.PromptInspection{ + PromptID: "briefing", + PromptVersion: "1", + PromptHash: "fixed-hash", + DefaultProfileID: "baseline", + Inputs: inputs, + OutputContract: promptkit.OutputContract{ + Format: promptkit.FormatText, + ValidationMode: promptkit.ValidationBasic, + RepairAttempts: 1, + }, + } +} + +func performanceProfileInspection(extra map[string]any) *promptkit.ProfileInspection { + return &promptkit.ProfileInspection{ + ProfileID: "baseline", + EffectiveModelParams: promptkit.ExecutionTarget{ + BackendID: "benchmark-backend", + Endpoint: "https://example.invalid/v1", + Model: "benchmark-model", + Temperature: 0.5, + MaxTokens: 512, + ExtraParams: extra, + }, + } +} + +func performanceExtraParams(keys, width int) map[string]any { + values := make(map[string]any, keys) + for keyIndex := 0; keyIndex < keys; keyIndex++ { + nested := make(map[string]any, width) + for valueIndex := 0; valueIndex < width; valueIndex++ { + nested[fmt.Sprintf("value%02d", valueIndex)] = fmt.Sprintf("sample-%02d-%02d", keyIndex, valueIndex) + } + values[fmt.Sprintf("group%02d", keyIndex)] = map[string]any{ + "enabled": keyIndex%2 == 0, + "nested": nested, + "sequence": []any{ + "first", + keyIndex, + map[string]any{"leaf": true}, + }, + } + } + return values +} + +func performancePromptApp(tb testing.TB, prompt *promptkit.PromptInspection, profile *promptkit.ProfileInspection) *App { + tb.Helper() + app, err := New(Identity{Name: "bench"}) + if err != nil { + tb.Fatal(err) + } + suite := PromptKitSuite[struct{}]{ + InspectPrompt: func(context.Context, Selection, struct{}, string, string) (Acquired[*promptkit.PromptInspection], error) { + return Acquired[*promptkit.PromptInspection]{Value: prompt}, nil + }, + InspectProfile: func(context.Context, Selection, struct{}, string) (Acquired[*promptkit.ProfileInspection], error) { + return Acquired[*promptkit.ProfileInspection]{Value: profile}, nil + }, + } + if err := RegisterPromptKit(app, suite); err != nil { + tb.Fatal(err) + } + return app +} + +func performanceNativeDocument(targetBytes int, directive bool) []byte { + preamble := "# generated native configuration\n" + if directive { + preamble = "%YAML 1.2\n---\n" + } + var body strings.Builder + body.Grow(targetBytes) + for index := 0; len(preamble)+body.Len() < targetBytes; index++ { + fmt.Fprintf(&body, "key%06d: value\n", index) + } + return []byte(preamble + body.String()) +} + +type performanceEffectiveWriter []byte + +func (document performanceEffectiveWriter) WriteEffective(writer io.Writer) error { + _, err := writer.Write(document) + return err +} + +func performanceShowApp(tb testing.TB, documents ...[]byte) *App { + tb.Helper() + scopes := make([]ConfigScope[struct{}], len(documents)) + for index, document := range documents { + name := "app" + if index > 0 { + name = fmt.Sprintf("scope%02d", index) + } + writer := performanceEffectiveWriter(document) + scopes[index] = ConfigScope[struct{}]{ + Name: name, + Operations: []ConfigOperation{ConfigShow}, + Acquire: func(context.Context, ConfigRequest, struct{}) (Acquired[ConfigView], error) { + return Acquired[ConfigView]{Value: ConfigView{Effective: writer}}, nil + }, + } + } + suite := ConfigSuite[struct{}]{Scopes: scopes} + return performanceConfigApp(tb, suite) +} + +func performanceConfigSuite(scopeCount int, operations []ConfigOperation) ConfigSuite[struct{}] { + scopes := make([]ConfigScope[struct{}], scopeCount) + for index := range scopes { + name := "app" + if index > 0 { + name = fmt.Sprintf("scope%02d", index) + } + scopes[index] = ConfigScope[struct{}]{ + Name: name, + Operations: append([]ConfigOperation(nil), operations...), + Acquire: func(context.Context, ConfigRequest, struct{}) (Acquired[ConfigView], error) { + return Acquired[ConfigView]{}, nil + }, + } + } + return ConfigSuite[struct{}]{Scopes: scopes} +} + +func performanceSelectorSuite(selectorCount int) ConfigSuite[struct{}] { + suite := performanceConfigSuite(1, []ConfigOperation{ConfigValidate}) + suite.Selectors = make([]Selector[struct{}], selectorCount) + for index := range suite.Selectors { + flag := ValueFlag(FlagMeta{Name: fmt.Sprintf("selector%02d", index)}, (*string)(nil), parseString, func(*struct{}, Supplied[string]) {}) + suite.Selectors[index] = Selector[struct{}]{Binding: flag, Scopes: []string{"app"}} + } + return suite +} + +func performanceConfigApp(tb testing.TB, suite ConfigSuite[struct{}]) *App { + tb.Helper() + app, err := New(Identity{Name: "bench"}) + if err != nil { + tb.Fatal(err) + } + if err := RegisterConfig(app, suite); err != nil { + tb.Fatal(err) + } + return app +} + +func verifyPromptReport(tb testing.TB, app *App, args []string, format, marker string) { + tb.Helper() + var output bytes.Buffer + outcome := app.Execute(context.Background(), args, Streams{Out: &output, Err: io.Discard}) + wantMarker := marker + if format == "text" { + if marker == "prompt_id" { + wantMarker = "Prompt ID:" + } else { + wantMarker = "Profile ID:" + } + } + if outcome.Kind != OutcomeSuccess || !strings.Contains(output.String(), wantMarker) { + tb.Fatalf("%s report outcome = %#v, output length %d", format, outcome, output.Len()) + } + if format == "yaml" && !strings.Contains(output.String(), ":") { + tb.Fatalf("YAML report lacks mapping entries: %q", output.String()) + } + if format == "text" && !strings.Contains(output.String(), " ID:") { + tb.Fatalf("text report has unexpected shape: %q", output.String()[:min(output.Len(), 80)]) + } +} + +func verifyConfigReport(tb testing.TB, app *App, args []string, minimumBytes int) { + tb.Helper() + var output bytes.Buffer + outcome := app.Execute(context.Background(), args, Streams{Out: &output, Err: io.Discard}) + if outcome.Kind != OutcomeSuccess || output.Len() < minimumBytes { + tb.Fatalf("configuration report outcome = %#v, output bytes %d, want at least %d", outcome, output.Len(), minimumBytes) + } +}