package commandkit import ( "bytes" "context" "fmt" "io" "strings" "testing" promptkit "gitea.maximumdirect.net/eric/promptkit" ) var performanceFramedBytes []byte func BenchmarkPerformancePromptReports(b *testing.B) { for _, inputCount := range []int{4, 128} { inspection := performancePromptInspection(inputCount) app := performancePromptApp(b, inspection, performanceProfileInspection(nil)) for _, format := range []string{"text", "yaml"} { name := fmt.Sprintf("prompt/inputs_%d/%s", inputCount, format) args := []string{"promptkit", "show", "prompt", "briefing", "--format", format} b.Run(name, func(b *testing.B) { verifyPromptReport(b, app, args, format, "prompt_id") streams := Streams{Out: io.Discard, Err: io.Discard} b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { if outcome := app.Execute(context.Background(), args, streams); outcome.Kind != OutcomeSuccess { b.Fatalf("prompt report outcome = %#v", outcome) } } }) } } for _, fixture := range []struct { name string extra map[string]any }{ {name: "small", extra: performanceExtraParams(2, 3)}, {name: "large", extra: performanceExtraParams(24, 8)}, } { profile := performanceProfileInspection(fixture.extra) app := performancePromptApp(b, performancePromptInspection(4), profile) for _, format := range []string{"text", "yaml"} { name := fmt.Sprintf("profile/extra_%s/%s", fixture.name, format) args := []string{"promptkit", "show", "profile", "baseline", "--format", format} b.Run(name, func(b *testing.B) { verifyPromptReport(b, app, args, format, "profile_id") streams := Streams{Out: io.Discard, Err: io.Discard} b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { if outcome := app.Execute(context.Background(), args, streams); outcome.Kind != OutcomeSuccess { b.Fatalf("profile report outcome = %#v", outcome) } } }) } } } func BenchmarkPerformanceYAMLFraming(b *testing.B) { fixtures := []struct { name string scope string native []byte }{ {name: "4k_ordinary", scope: "app", native: performanceNativeDocument(4<<10, false)}, {name: "1m_directive", scope: "promptkit", native: performanceNativeDocument(1<<20, true)}, } for _, fixture := range fixtures { b.Run("framing/"+fixture.name, func(b *testing.B) { framed, hasDirective := frameEffectiveYAML(fixture.scope, fixture.native) if len(framed) <= len(fixture.native) || hasDirective != strings.Contains(fixture.name, "directive") { b.Fatalf("framing fixture result has unexpected size or directive classification: native=%d framed=%d", len(fixture.native), len(framed)) } b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { performanceFramedBytes, _ = frameEffectiveYAML(fixture.scope, fixture.native) } }) b.Run("aggregate/"+fixture.name, func(b *testing.B) { app := performanceShowApp(b, fixture.native, fixture.native) args := []string{"config", "show", "--scope", "all"} verifyConfigReport(b, app, args, len(fixture.native)*2) streams := Streams{Out: io.Discard, Err: io.Discard} b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { if outcome := app.Execute(context.Background(), args, streams); outcome.Kind != OutcomeSuccess { b.Fatalf("aggregate config show outcome = %#v", outcome) } } }) } single := performanceNativeDocument(4<<10, false) app := performanceShowApp(b, single) args := []string{"config", "show"} b.Run("single_scope/4k_ordinary", func(b *testing.B) { verifyConfigReport(b, app, args, len(single)) streams := Streams{Out: io.Discard, Err: io.Discard} b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { if outcome := app.Execute(context.Background(), args, streams); outcome.Kind != OutcomeSuccess { b.Fatalf("single-scope config show outcome = %#v", outcome) } } }) } func BenchmarkPerformanceConfigScopes(b *testing.B) { for _, scopeCount := range []int{2, 32} { suite := performanceConfigSuite(scopeCount, []ConfigOperation{ConfigValidate, ConfigShow}) app := performanceConfigApp(b, suite) selected := fmt.Sprintf("scope%02d", scopeCount-1) for _, operation := range []struct { name string args []string }{ {name: "discovery", args: []string{"config", "scopes"}}, {name: "named_selection", args: []string{"config", "validate", "--scope", selected}}, {name: "aggregate_validation", args: []string{"config", "validate", "--scope", "all"}}, } { b.Run(fmt.Sprintf("%s/scopes_%d", operation.name, scopeCount), func(b *testing.B) { verifyConfigReport(b, app, operation.args, 1) streams := Streams{Out: io.Discard, Err: io.Discard} b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { if outcome := app.Execute(context.Background(), operation.args, streams); outcome.Kind != OutcomeSuccess { b.Fatalf("configuration operation outcome = %#v", outcome) } } }) } } } func BenchmarkPerformanceConfigRegistration(b *testing.B) { for _, selectorCount := range []int{4, 64} { suite := performanceSelectorSuite(selectorCount) performanceConfigApp(b, suite) b.Run(fmt.Sprintf("selectors_%d", selectorCount), func(b *testing.B) { b.ReportAllocs() for i := 0; i < b.N; i++ { app, err := New(Identity{Name: "bench"}) if err != nil { b.Fatal(err) } if err := RegisterConfig(app, suite); err != nil { b.Fatal(err) } } }) } } func performancePromptInspection(inputCount int) *promptkit.PromptInspection { inputs := make([]promptkit.PromptInputDefinition, inputCount) for index := range inputs { inputs[index] = promptkit.PromptInputDefinition{ Name: fmt.Sprintf("input%03d", index), Required: index%2 == 0, ContentType: "text/plain", Description: "A representative prompt input.", } } return &promptkit.PromptInspection{ PromptID: "briefing", PromptVersion: "1", PromptHash: "fixed-hash", DefaultProfileID: "baseline", Inputs: inputs, OutputContract: promptkit.OutputContract{ Format: promptkit.FormatText, ValidationMode: promptkit.ValidationBasic, RepairAttempts: 1, }, } } func performanceProfileInspection(extra map[string]any) *promptkit.ProfileInspection { return &promptkit.ProfileInspection{ ProfileID: "baseline", EffectiveModelParams: promptkit.ExecutionTarget{ BackendID: "benchmark-backend", Endpoint: "https://example.invalid/v1", Model: "benchmark-model", Temperature: 0.5, MaxTokens: 512, ExtraParams: extra, }, } } func performanceExtraParams(keys, width int) map[string]any { values := make(map[string]any, keys) for keyIndex := 0; keyIndex < keys; keyIndex++ { nested := make(map[string]any, width) for valueIndex := 0; valueIndex < width; valueIndex++ { nested[fmt.Sprintf("value%02d", valueIndex)] = fmt.Sprintf("sample-%02d-%02d", keyIndex, valueIndex) } values[fmt.Sprintf("group%02d", keyIndex)] = map[string]any{ "enabled": keyIndex%2 == 0, "nested": nested, "sequence": []any{ "first", keyIndex, map[string]any{"leaf": true}, }, } } return values } func performancePromptApp(tb testing.TB, prompt *promptkit.PromptInspection, profile *promptkit.ProfileInspection) *App { tb.Helper() app, err := New(Identity{Name: "bench"}) if err != nil { tb.Fatal(err) } suite := PromptKitSuite[struct{}]{ InspectPrompt: func(context.Context, Selection, struct{}, string, string) (Acquired[*promptkit.PromptInspection], error) { return Acquired[*promptkit.PromptInspection]{Value: prompt}, nil }, InspectProfile: func(context.Context, Selection, struct{}, string) (Acquired[*promptkit.ProfileInspection], error) { return Acquired[*promptkit.ProfileInspection]{Value: profile}, nil }, } if err := RegisterPromptKit(app, suite); err != nil { tb.Fatal(err) } return app } func performanceNativeDocument(targetBytes int, directive bool) []byte { preamble := "# generated native configuration\n" if directive { preamble = "%YAML 1.2\n---\n" } var body strings.Builder body.Grow(targetBytes) for index := 0; len(preamble)+body.Len() < targetBytes; index++ { fmt.Fprintf(&body, "key%06d: value\n", index) } return []byte(preamble + body.String()) } type performanceEffectiveWriter []byte func (document performanceEffectiveWriter) WriteEffective(writer io.Writer) error { _, err := writer.Write(document) return err } func performanceShowApp(tb testing.TB, documents ...[]byte) *App { tb.Helper() scopes := make([]ConfigScope[struct{}], len(documents)) for index, document := range documents { name := "app" if index > 0 { name = fmt.Sprintf("scope%02d", index) } writer := performanceEffectiveWriter(document) scopes[index] = ConfigScope[struct{}]{ Name: name, Operations: []ConfigOperation{ConfigShow}, Acquire: func(context.Context, ConfigRequest, struct{}) (Acquired[ConfigView], error) { return Acquired[ConfigView]{Value: ConfigView{Effective: writer}}, nil }, } } suite := ConfigSuite[struct{}]{Scopes: scopes} return performanceConfigApp(tb, suite) } func performanceConfigSuite(scopeCount int, operations []ConfigOperation) ConfigSuite[struct{}] { scopes := make([]ConfigScope[struct{}], scopeCount) for index := range scopes { name := "app" if index > 0 { name = fmt.Sprintf("scope%02d", index) } scopes[index] = ConfigScope[struct{}]{ Name: name, Operations: append([]ConfigOperation(nil), operations...), Acquire: func(context.Context, ConfigRequest, struct{}) (Acquired[ConfigView], error) { return Acquired[ConfigView]{}, nil }, } } return ConfigSuite[struct{}]{Scopes: scopes} } func performanceSelectorSuite(selectorCount int) ConfigSuite[struct{}] { suite := performanceConfigSuite(1, []ConfigOperation{ConfigValidate}) suite.Selectors = make([]Selector[struct{}], selectorCount) for index := range suite.Selectors { flag := ValueFlag(FlagMeta{Name: fmt.Sprintf("selector%02d", index)}, (*string)(nil), parseString, func(*struct{}, Supplied[string]) {}) suite.Selectors[index] = Selector[struct{}]{Binding: flag, Scopes: []string{"app"}} } return suite } func performanceConfigApp(tb testing.TB, suite ConfigSuite[struct{}]) *App { tb.Helper() app, err := New(Identity{Name: "bench"}) if err != nil { tb.Fatal(err) } if err := RegisterConfig(app, suite); err != nil { tb.Fatal(err) } return app } func verifyPromptReport(tb testing.TB, app *App, args []string, format, marker string) { tb.Helper() var output bytes.Buffer outcome := app.Execute(context.Background(), args, Streams{Out: &output, Err: io.Discard}) wantMarker := marker if format == "text" { if marker == "prompt_id" { wantMarker = "Prompt ID:" } else { wantMarker = "Profile ID:" } } if outcome.Kind != OutcomeSuccess || !strings.Contains(output.String(), wantMarker) { tb.Fatalf("%s report outcome = %#v, output length %d", format, outcome, output.Len()) } if format == "yaml" && !strings.Contains(output.String(), ":") { tb.Fatalf("YAML report lacks mapping entries: %q", output.String()) } if format == "text" && !strings.Contains(output.String(), " ID:") { tb.Fatalf("text report has unexpected shape: %q", output.String()[:min(output.Len(), 80)]) } } func verifyConfigReport(tb testing.TB, app *App, args []string, minimumBytes int) { tb.Helper() var output bytes.Buffer outcome := app.Execute(context.Background(), args, Streams{Out: &output, Err: io.Discard}) if outcome.Kind != OutcomeSuccess || output.Len() < minimumBytes { tb.Fatalf("configuration report outcome = %#v, output bytes %d, want at least %d", outcome, output.Len(), minimumBytes) } }