Add report and scope benchmarks

2026-10-09 01:23:35 +00:00
parent aed4c5921c
commit a431540e19
2 changed files with 405 additions and 3 deletions
--- a/docs/roadmap/implementation.md
+++ b/docs/roadmap/implementation.md
@@ -1,6 +1,6 @@
 # Performance implementation plan

-Status: implementation in progress; Stage 1 is complete, Stages 2–11 have not started.
+Status: implementation in progress; Stages 1–2 are complete, Stages 3–11 have not started.

 Date: 2026-10-09.

@@ -228,7 +228,7 @@ shows why later comparisons should report run variation.

 ## Stage 2: establish report and scope measurements

-**Status:** not started. **Prerequisite:** Stage 1.
+**Status:** complete. **Prerequisite:** Stage 1.

 **Files:** new `performance_reports_test.go` and evidence here. Production code
 stays unchanged in this stage.
@@ -258,7 +258,57 @@ owner of actual library loading semantics.

 **Completion:** PF-01, PF-04, and PF-06 have comparable baseline measurements.

-**Evidence:** pending.
+**Evidence:** Fixtures were measured from source revision `aed4c59` with
+`performance_reports_test.go` in the working tree (committed with this evidence).
+Environment matched the first baseline: Go 1.26.8, linux/amd64, Linux
+6.17.0-41-generic, Intel Core i5-7260U CPU @ 2.20 GHz. Root package compilation
+passed with `GOWORK=off GOTOOLCHAIN=local GOPROXY=off GONOPROXY=none GOSUMDB=off go test -mod=readonly -run '^$' .`. The five-sample benchmark command passed:
+`GOWORK=off GOTOOLCHAIN=local GOPROXY=off GONOPROXY=none GOSUMDB=off go test -mod=readonly -run '^$' -bench '^BenchmarkPerformance(PromptReports|YAMLFraming|ConfigScopes|ConfigRegistration)$' -benchmem -benchtime=200ms -count=5 .`.
+Raw output is in `/tmp/commandkit-performance-reports-baseline.txt`.
+
+Fixtures: PromptKit callbacks returned prebuilt inspection records directly;
+prompt reports used 4 and 128 input records, while profile `ExtraParams` used
+2 groups with 3 nested values each or 24 groups with 8 nested values each and a
+nested sequence. Each was measured as text and YAML through `App.Execute`.
+Native YAML fixtures contain unique short mapping keys and are 4,096 bytes with
+an ordinary comment preamble and 1,048,591 bytes with a `%YAML` directive
+preamble. Framing was measured directly; aggregate `config show --scope all`
+measured two documents of each selected fixture size/style, and single-scope
+display used the 4 KiB ordinary document. Scope operations used 2 or 32 scopes
+with the requested show/validate capabilities and cheap in-memory acquisition
+callbacks. Suite registration used 4 or 64 distinct selectors and a fresh app.
+The scope counts also covered discovery, named validation, and aggregate
+validation. Setup verified success and report shape outside timed loops; native
+fixtures, records, suites, and apps were prepared outside repeated loops.
+
+Representative five-sample ranges (ns/op; B/op; allocs/op): prompt, 4 inputs,
+text 41,287–52,115; 26,160–26,161; 294, and YAML 88,991–104,671;
+74,730–74,733; 557. Prompt, 128 inputs, text 169,155–198,281;
+145,524–145,531; 1,171, and YAML 1,418,692–1,490,181; 1,393,508–1,393,568;
+6,039–6,040. Profile, small extra map, text 74,109–89,337; 70,914–70,918;
+462, and YAML 161,470–368,813; 162,688–162,717; 755. Profile, large extra map,
+text 1,277,396–2,583,950; 836,141–836,296; 3,027–3,028, and YAML
+1,756,571–2,377,499; 1,586,728–1,586,849; 5,878–5,879. Larger profile timings
+varied, while allocation counts and bytes were stable.
+
+Direct framing measured 4 KiB ordinary at 5,883–6,842 ns/op, 11,056 B/op,
+3 allocs/op, and 1 MiB directive at 1,208,678–1,502,417 ns/op,
+2,539,520–2,539,523 B/op, 2 allocs/op. Aggregate show measured 4 KiB documents
+at 44,395–48,161 ns/op, 55,537–55,538 B/op, 211 allocs/op, and 1 MiB documents
+at 4,818,907–6,009,877 ns/op, 8,695,182–8,695,197 B/op, 215–216 allocs/op.
+Single-scope 4 KiB display measured 20,229–21,791 ns/op, 17,935–17,936 B/op,
+183 allocs/op. With 2 scopes, discovery measured 23,840–30,007 ns/op,
+21,161–21,162 B/op, 223 allocs/op; named validation 24,841–26,340 ns/op,
+21,817 B/op, 228 allocs/op; aggregate validation 25,231–31,126 ns/op,
+22,361–22,362 B/op, 237 allocs/op. With 32 scopes, discovery measured
+31,439–37,167 ns/op, 28,141–28,142 B/op, 257 allocs/op; named validation
+29,123–74,374 ns/op, 22,775–22,778 B/op, 228 allocs/op; aggregate validation
+45,548–63,471 ns/op, 29,781–29,782 B/op, 269 allocs/op. Registration with 4
+selectors measured 12,573–14,789 ns/op, 8,911 B/op, 104 allocs/op; with 64 it
+measured 91,675–105,548 ns/op, 82,817–82,818 B/op, 485 allocs/op. Large-report
+cost includes full output framing/delivery; no segmented-delivery comparison
+was selected. These measurements establish baselines and show timing variation;
+they are not fixed thresholds or speedup claims.

 ## Stage 3: render only the selected PromptKit format

--- a/performance_reports_test.go
+++ b/performance_reports_test.go
@@ -0,0 +1,352 @@
+package commandkit
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"strings"
+	"testing"
+
+	promptkit "gitea.maximumdirect.net/eric/promptkit"
+)
+
+var performanceFramedBytes []byte
+
+func BenchmarkPerformancePromptReports(b *testing.B) {
+	for _, inputCount := range []int{4, 128} {
+		inspection := performancePromptInspection(inputCount)
+		app := performancePromptApp(b, inspection, performanceProfileInspection(nil))
+		for _, format := range []string{"text", "yaml"} {
+			name := fmt.Sprintf("prompt/inputs_%d/%s", inputCount, format)
+			args := []string{"promptkit", "show", "prompt", "briefing", "--format", format}
+			b.Run(name, func(b *testing.B) {
+				verifyPromptReport(b, app, args, format, "prompt_id")
+				streams := Streams{Out: io.Discard, Err: io.Discard}
+				b.ReportAllocs()
+				b.ResetTimer()
+				for i := 0; i < b.N; i++ {
+					if outcome := app.Execute(context.Background(), args, streams); outcome.Kind != OutcomeSuccess {
+						b.Fatalf("prompt report outcome = %#v", outcome)
+					}
+				}
+			})
+		}
+	}
+
+	for _, fixture := range []struct {
+		name  string
+		extra map[string]any
+	}{
+		{name: "small", extra: performanceExtraParams(2, 3)},
+		{name: "large", extra: performanceExtraParams(24, 8)},
+	} {
+		profile := performanceProfileInspection(fixture.extra)
+		app := performancePromptApp(b, performancePromptInspection(4), profile)
+		for _, format := range []string{"text", "yaml"} {
+			name := fmt.Sprintf("profile/extra_%s/%s", fixture.name, format)
+			args := []string{"promptkit", "show", "profile", "baseline", "--format", format}
+			b.Run(name, func(b *testing.B) {
+				verifyPromptReport(b, app, args, format, "profile_id")
+				streams := Streams{Out: io.Discard, Err: io.Discard}
+				b.ReportAllocs()
+				b.ResetTimer()
+				for i := 0; i < b.N; i++ {
+					if outcome := app.Execute(context.Background(), args, streams); outcome.Kind != OutcomeSuccess {
+						b.Fatalf("profile report outcome = %#v", outcome)
+					}
+				}
+			})
+		}
+	}
+}
+
+func BenchmarkPerformanceYAMLFraming(b *testing.B) {
+	fixtures := []struct {
+		name   string
+		scope  string
+		native []byte
+	}{
+		{name: "4k_ordinary", scope: "app", native: performanceNativeDocument(4<<10, false)},
+		{name: "1m_directive", scope: "promptkit", native: performanceNativeDocument(1<<20, true)},
+	}
+	for _, fixture := range fixtures {
+		b.Run("framing/"+fixture.name, func(b *testing.B) {
+			framed := frameEffectiveYAML(fixture.scope, fixture.native)
+			if len(framed) <= len(fixture.native) || hasYAMLDirective(fixture.native) != strings.Contains(fixture.name, "directive") {
+				b.Fatalf("framing fixture result has unexpected size or directive classification: native=%d framed=%d", len(fixture.native), len(framed))
+			}
+			b.ReportAllocs()
+			b.ResetTimer()
+			for i := 0; i < b.N; i++ {
+				performanceFramedBytes = frameEffectiveYAML(fixture.scope, fixture.native)
+			}
+		})
+		b.Run("aggregate/"+fixture.name, func(b *testing.B) {
+			app := performanceShowApp(b, fixture.native, fixture.native)
+			args := []string{"config", "show", "--scope", "all"}
+			verifyConfigReport(b, app, args, len(fixture.native)*2)
+			streams := Streams{Out: io.Discard, Err: io.Discard}
+			b.ReportAllocs()
+			b.ResetTimer()
+			for i := 0; i < b.N; i++ {
+				if outcome := app.Execute(context.Background(), args, streams); outcome.Kind != OutcomeSuccess {
+					b.Fatalf("aggregate config show outcome = %#v", outcome)
+				}
+			}
+		})
+	}
+
+	single := performanceNativeDocument(4<<10, false)
+	app := performanceShowApp(b, single)
+	args := []string{"config", "show"}
+	b.Run("single_scope/4k_ordinary", func(b *testing.B) {
+		verifyConfigReport(b, app, args, len(single))
+		streams := Streams{Out: io.Discard, Err: io.Discard}
+		b.ReportAllocs()
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			if outcome := app.Execute(context.Background(), args, streams); outcome.Kind != OutcomeSuccess {
+				b.Fatalf("single-scope config show outcome = %#v", outcome)
+			}
+		}
+	})
+}
+
+func BenchmarkPerformanceConfigScopes(b *testing.B) {
+	for _, scopeCount := range []int{2, 32} {
+		suite := performanceConfigSuite(scopeCount, []ConfigOperation{ConfigValidate, ConfigShow})
+		app := performanceConfigApp(b, suite)
+		selected := fmt.Sprintf("scope%02d", scopeCount-1)
+		for _, operation := range []struct {
+			name string
+			args []string
+		}{
+			{name: "discovery", args: []string{"config", "scopes"}},
+			{name: "named_selection", args: []string{"config", "validate", "--scope", selected}},
+			{name: "aggregate_validation", args: []string{"config", "validate", "--scope", "all"}},
+		} {
+			b.Run(fmt.Sprintf("%s/scopes_%d", operation.name, scopeCount), func(b *testing.B) {
+				verifyConfigReport(b, app, operation.args, 1)
+				streams := Streams{Out: io.Discard, Err: io.Discard}
+				b.ReportAllocs()
+				b.ResetTimer()
+				for i := 0; i < b.N; i++ {
+					if outcome := app.Execute(context.Background(), operation.args, streams); outcome.Kind != OutcomeSuccess {
+						b.Fatalf("configuration operation outcome = %#v", outcome)
+					}
+				}
+			})
+		}
+	}
+}
+
+func BenchmarkPerformanceConfigRegistration(b *testing.B) {
+	for _, selectorCount := range []int{4, 64} {
+		suite := performanceSelectorSuite(selectorCount)
+		performanceConfigApp(b, suite)
+		b.Run(fmt.Sprintf("selectors_%d", selectorCount), func(b *testing.B) {
+			b.ReportAllocs()
+			for i := 0; i < b.N; i++ {
+				app, err := New(Identity{Name: "bench"})
+				if err != nil {
+					b.Fatal(err)
+				}
+				if err := RegisterConfig(app, suite); err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}
+
+func performancePromptInspection(inputCount int) *promptkit.PromptInspection {
+	inputs := make([]promptkit.PromptInputDefinition, inputCount)
+	for index := range inputs {
+		inputs[index] = promptkit.PromptInputDefinition{
+			Name:        fmt.Sprintf("input%03d", index),
+			Required:    index%2 == 0,
+			ContentType: "text/plain",
+			Description: "A representative prompt input.",
+		}
+	}
+	return &promptkit.PromptInspection{
+		PromptID:         "briefing",
+		PromptVersion:    "1",
+		PromptHash:       "fixed-hash",
+		DefaultProfileID: "baseline",
+		Inputs:           inputs,
+		OutputContract: promptkit.OutputContract{
+			Format:         promptkit.FormatText,
+			ValidationMode: promptkit.ValidationBasic,
+			RepairAttempts: 1,
+		},
+	}
+}
+
+func performanceProfileInspection(extra map[string]any) *promptkit.ProfileInspection {
+	return &promptkit.ProfileInspection{
+		ProfileID: "baseline",
+		EffectiveModelParams: promptkit.ExecutionTarget{
+			BackendID:   "benchmark-backend",
+			Endpoint:    "https://example.invalid/v1",
+			Model:       "benchmark-model",
+			Temperature: 0.5,
+			MaxTokens:   512,
+			ExtraParams: extra,
+		},
+	}
+}
+
+func performanceExtraParams(keys, width int) map[string]any {
+	values := make(map[string]any, keys)
+	for keyIndex := 0; keyIndex < keys; keyIndex++ {
+		nested := make(map[string]any, width)
+		for valueIndex := 0; valueIndex < width; valueIndex++ {
+			nested[fmt.Sprintf("value%02d", valueIndex)] = fmt.Sprintf("sample-%02d-%02d", keyIndex, valueIndex)
+		}
+		values[fmt.Sprintf("group%02d", keyIndex)] = map[string]any{
+			"enabled": keyIndex%2 == 0,
+			"nested":  nested,
+			"sequence": []any{
+				"first",
+				keyIndex,
+				map[string]any{"leaf": true},
+			},
+		}
+	}
+	return values
+}
+
+func performancePromptApp(tb testing.TB, prompt *promptkit.PromptInspection, profile *promptkit.ProfileInspection) *App {
+	tb.Helper()
+	app, err := New(Identity{Name: "bench"})
+	if err != nil {
+		tb.Fatal(err)
+	}
+	suite := PromptKitSuite[struct{}]{
+		InspectPrompt: func(context.Context, Selection, struct{}, string, string) (Acquired[*promptkit.PromptInspection], error) {
+			return Acquired[*promptkit.PromptInspection]{Value: prompt}, nil
+		},
+		InspectProfile: func(context.Context, Selection, struct{}, string) (Acquired[*promptkit.ProfileInspection], error) {
+			return Acquired[*promptkit.ProfileInspection]{Value: profile}, nil
+		},
+	}
+	if err := RegisterPromptKit(app, suite); err != nil {
+		tb.Fatal(err)
+	}
+	return app
+}
+
+func performanceNativeDocument(targetBytes int, directive bool) []byte {
+	preamble := "# generated native configuration\n"
+	if directive {
+		preamble = "%YAML 1.2\n---\n"
+	}
+	var body strings.Builder
+	body.Grow(targetBytes)
+	for index := 0; len(preamble)+body.Len() < targetBytes; index++ {
+		fmt.Fprintf(&body, "key%06d: value\n", index)
+	}
+	return []byte(preamble + body.String())
+}
+
+type performanceEffectiveWriter []byte
+
+func (document performanceEffectiveWriter) WriteEffective(writer io.Writer) error {
+	_, err := writer.Write(document)
+	return err
+}
+
+func performanceShowApp(tb testing.TB, documents ...[]byte) *App {
+	tb.Helper()
+	scopes := make([]ConfigScope[struct{}], len(documents))
+	for index, document := range documents {
+		name := "app"
+		if index > 0 {
+			name = fmt.Sprintf("scope%02d", index)
+		}
+		writer := performanceEffectiveWriter(document)
+		scopes[index] = ConfigScope[struct{}]{
+			Name:       name,
+			Operations: []ConfigOperation{ConfigShow},
+			Acquire: func(context.Context, ConfigRequest, struct{}) (Acquired[ConfigView], error) {
+				return Acquired[ConfigView]{Value: ConfigView{Effective: writer}}, nil
+			},
+		}
+	}
+	suite := ConfigSuite[struct{}]{Scopes: scopes}
+	return performanceConfigApp(tb, suite)
+}
+
+func performanceConfigSuite(scopeCount int, operations []ConfigOperation) ConfigSuite[struct{}] {
+	scopes := make([]ConfigScope[struct{}], scopeCount)
+	for index := range scopes {
+		name := "app"
+		if index > 0 {
+			name = fmt.Sprintf("scope%02d", index)
+		}
+		scopes[index] = ConfigScope[struct{}]{
+			Name:       name,
+			Operations: append([]ConfigOperation(nil), operations...),
+			Acquire: func(context.Context, ConfigRequest, struct{}) (Acquired[ConfigView], error) {
+				return Acquired[ConfigView]{}, nil
+			},
+		}
+	}
+	return ConfigSuite[struct{}]{Scopes: scopes}
+}
+
+func performanceSelectorSuite(selectorCount int) ConfigSuite[struct{}] {
+	suite := performanceConfigSuite(1, []ConfigOperation{ConfigValidate})
+	suite.Selectors = make([]Selector[struct{}], selectorCount)
+	for index := range suite.Selectors {
+		flag := ValueFlag(FlagMeta{Name: fmt.Sprintf("selector%02d", index)}, (*string)(nil), parseString, func(*struct{}, Supplied[string]) {})
+		suite.Selectors[index] = Selector[struct{}]{Binding: flag, Scopes: []string{"app"}}
+	}
+	return suite
+}
+
+func performanceConfigApp(tb testing.TB, suite ConfigSuite[struct{}]) *App {
+	tb.Helper()
+	app, err := New(Identity{Name: "bench"})
+	if err != nil {
+		tb.Fatal(err)
+	}
+	if err := RegisterConfig(app, suite); err != nil {
+		tb.Fatal(err)
+	}
+	return app
+}
+
+func verifyPromptReport(tb testing.TB, app *App, args []string, format, marker string) {
+	tb.Helper()
+	var output bytes.Buffer
+	outcome := app.Execute(context.Background(), args, Streams{Out: &output, Err: io.Discard})
+	wantMarker := marker
+	if format == "text" {
+		if marker == "prompt_id" {
+			wantMarker = "Prompt ID:"
+		} else {
+			wantMarker = "Profile ID:"
+		}
+	}
+	if outcome.Kind != OutcomeSuccess || !strings.Contains(output.String(), wantMarker) {
+		tb.Fatalf("%s report outcome = %#v, output length %d", format, outcome, output.Len())
+	}
+	if format == "yaml" && !strings.Contains(output.String(), ":") {
+		tb.Fatalf("YAML report lacks mapping entries: %q", output.String())
+	}
+	if format == "text" && !strings.Contains(output.String(), " ID:") {
+		tb.Fatalf("text report has unexpected shape: %q", output.String()[:min(output.Len(), 80)])
+	}
+}
+
+func verifyConfigReport(tb testing.TB, app *App, args []string, minimumBytes int) {
+	tb.Helper()
+	var output bytes.Buffer
+	outcome := app.Execute(context.Background(), args, Streams{Out: &output, Err: io.Discard})
+	if outcome.Kind != OutcomeSuccess || output.Len() < minimumBytes {
+		tb.Fatalf("configuration report outcome = %#v, output bytes %d, want at least %d", outcome, output.Len(), minimumBytes)
+	}
+}