Add dispatch and registration benchmarks

2026-10-09 01:19:03 +00:00
parent 1274ae71c0
commit aed4c5921c
3 changed files with 291 additions and 3 deletions
--- a/docs/roadmap/implementation.md
+++ b/docs/roadmap/implementation.md
@@ -1,6 +1,6 @@
 # Performance implementation plan

-Status: ready for implementation; Stages 1–11 have not started.
+Status: implementation in progress; Stage 1 is complete, Stages 2–11 have not started.

 Date: 2026-10-09.

@@ -150,7 +150,7 @@ benchmark-analysis dependency or permanent numeric pass threshold is selected.

 ## Stage 1: establish dispatch and registration measurements

-**Status:** not started. **Prerequisite:** orientation and this plan.
+**Status:** complete. **Prerequisite:** orientation and this plan.

 **Files:** new `performance_dispatch_test.go`,
 `internal/cobra/performance_test.go`, and evidence here. Production code stays
@@ -182,7 +182,49 @@ production instrumentation, or full dependency test matrix is needed.
 **Completion:** reproducible baseline results distinguish registration, cold
 execution, repeated execution, parsing, and help/usage costs.

-**Evidence:** pending.
+**Evidence:** Benchmark fixtures were measured from source revision `1274ae7`
+with the new benchmark files in the working tree (committed with this evidence).
+Environment: Go 1.26.8, linux/amd64, Linux 6.17.0-41-generic, Intel Core
+i5-7260U CPU @ 2.20 GHz. The required compile-only command passed:
+`GOWORK=off GOTOOLCHAIN=local GOPROXY=off GONOPROXY=none GOSUMDB=off go test -mod=readonly -run '^$' . ./internal/cobra`.
+The five-sample measurement command also passed:
+`GOWORK=off GOTOOLCHAIN=local GOPROXY=off GONOPROXY=none GOSUMDB=off go test -mod=readonly -run '^$' -bench '^BenchmarkPerformance' -benchmem -benchtime=200ms -count=5 . ./internal/cobra`.
+Raw output is in `/tmp/commandkit-performance-baseline.txt`; focused repeat output
+for noisy cases is in `/tmp/commandkit-performance-baseline-repeat.txt`.
+
+Fixtures: root dispatch used 8 and 128 flat commands, a 24-level path with
+siblings at each parent, and a leaf with 24 flags. Dispatch covered success,
+local and explicit help, parser and handler usage, cold construction plus first
+execution, and repeated execution. Registration used 8 and 128 prebuilt
+declarations with fresh `New` in individual and batch paths. Adapter parsing
+used 128 shallow siblings and the same depth-24 path. All repeated app and
+adapter fixtures were built outside the timed loops; benchmark outcomes were
+checked for their intended classes.
+
+Representative five-sample ranges (ns/op; B/op; allocs/op): small repeated
+success 31,066–41,041; 33,209–33,210; 287. Small local help 103,022–116,183;
+78,176–78,177; 810. Parser usage 62,596–74,475; 66,363–66,364; 591. Wide
+success 218,073–253,010; 269,063–269,070; 1,898. Deep success 119,958–139,625;
+139,787–139,791; 1,074. Wide help 491,172–622,188; 549,923–549,940; 4,033.
+Explicit help measured 101,998–111,895 ns/op, 78,152–78,153 B/op, and 815
+allocs/op; handler usage measured 61,878–72,264 ns/op, 66,211–66,212 B/op, and
+583 allocs/op. Cold construction plus first execution measured 38,152–50,235
+ns/op, 38,275–38,276 B/op, and 339 allocs/op for the small app, and
+251,945–315,276 ns/op, 291,822–291,827 B/op, and 2,184 allocs/op for the wide
+app. The 24-flag leaf focused repeat was 43,662–56,492 ns/op, 40,027–40,028
+B/op, and 401 allocs/op.
+Deep help was noisy in the full run (429,873–1,634,363 ns/op); a focused repeat
+was 300,155–326,094 ns/op, 294,882–294,896 B/op, and 2,408 allocs/op. The initial
+flag-heavy timing also varied widely before the focused range reported above.
+Registration at 8 declarations measured individual 7,141–7,645 ns/op,
+5,497 B/op, 64 allocs/op versus batch 4,159–5,190 ns/op, 3,456 B/op,
+48 allocs/op. At 128 declarations, individual measured 414,618–492,261 ns/op,
+405,011–405,018 B/op, 784 allocs/op versus batch 33,985–36,207 ns/op,
+23,013–23,014 B/op, 288 allocs/op. Adapter parse measured shallow/wide
+150,918–171,598 ns/op, 242,936 B/op, 1,705 allocs/op, and deep
+86,282–109,697 ns/op, 119,920 B/op, 827 allocs/op. These are baselines, not
+speedup claims; the large initial deep-help and flag-heavy timing variation
+shows why later comparisons should report run variation.

 ## Stage 2: establish report and scope measurements

--- a/internal/cobra/performance_test.go
+++ b/internal/cobra/performance_test.go
@@ -0,0 +1,62 @@
+package cobra
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"testing"
+)
+
+func BenchmarkPerformanceParse(b *testing.B) {
+	wide := performanceFlatTree(128)
+	wideArgs := []string{"wide127"}
+	deep, deepArgs := performanceNestedTree(24)
+	options := ParseOptions{Context: context.Background(), Out: io.Discard, Err: io.Discard}
+
+	for _, benchmark := range []struct {
+		name string
+		root *Command
+		args []string
+	}{
+		{name: "shallow_wide", root: wide, args: wideArgs},
+		{name: "deep", root: deep, args: deepArgs},
+	} {
+		b.Run(benchmark.name, func(b *testing.B) {
+			result := Parse(benchmark.root, benchmark.args, options)
+			if result.Failure != nil {
+				b.Fatalf("fixture parse failed: %s", result.Failure.Message)
+			}
+			b.ReportAllocs()
+			b.ResetTimer()
+			for i := 0; i < b.N; i++ {
+				result := Parse(benchmark.root, benchmark.args, options)
+				if result.Failure != nil {
+					b.Fatalf("parse failed: %s", result.Failure.Message)
+				}
+			}
+		})
+	}
+}
+
+func performanceFlatTree(count int) *Command {
+	root := &Command{Name: "app", Children: make([]*Command, count)}
+	for index := range root.Children {
+		root.Children[index] = &Command{Name: fmt.Sprintf("wide%03d", index), Runnable: true}
+	}
+	return root
+}
+
+func performanceNestedTree(depth int) (*Command, []string) {
+	path := make([]string, depth)
+	for index := range path {
+		path[index] = fmt.Sprintf("level%02d", index)
+	}
+	current := &Command{Name: path[depth-1], Runnable: true}
+	for index := depth - 2; index >= 0; index-- {
+		current = &Command{Name: path[index], Children: []*Command{
+			current,
+			{Name: fmt.Sprintf("sibling%02d", index), Runnable: true},
+		}}
+	}
+	return &Command{Name: "app", Children: []*Command{current}}, path
+}
--- a/performance_dispatch_test.go
+++ b/performance_dispatch_test.go
@@ -0,0 +1,184 @@
+package commandkit_test
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"testing"
+
+	commandkit "gitea.maximumdirect.net/eric/CommandKit"
+)
+
+var performanceUsageCause = errors.New("benchmark usage result")
+
+func BenchmarkPerformanceDispatch(b *testing.B) {
+	small := performanceFlatCommands(b, 8, "cmd")
+	wide := performanceFlatCommands(b, 128, "wide")
+	deep, deepPath := performanceNestedCommand(b)
+	smallApp := performanceApp(b, small)
+	wideApp := performanceApp(b, wide)
+	deepApp := performanceApp(b, []commandkit.Command{deep})
+	streams := commandkit.Streams{Out: io.Discard, Err: io.Discard}
+
+	benchmarks := []struct {
+		name string
+		app  *commandkit.App
+		args []string
+		kind commandkit.OutcomeKind
+	}{
+		{name: "success", app: smallApp, args: []string{"cmd00"}, kind: commandkit.OutcomeSuccess},
+		{name: "local_help", app: smallApp, args: []string{"cmd01", "--help"}, kind: commandkit.OutcomeSuccess},
+		{name: "explicit_help", app: smallApp, args: []string{"help", "cmd02"}, kind: commandkit.OutcomeSuccess},
+		{name: "parser_usage", app: smallApp, args: []string{"cmd03", "--unknown"}, kind: commandkit.OutcomeUsage},
+		{name: "handler_usage", app: smallApp, args: []string{"cmd04"}, kind: commandkit.OutcomeUsage},
+		{name: "wide_success", app: wideApp, args: []string{"wide127"}, kind: commandkit.OutcomeSuccess},
+		{name: "wide_help", app: wideApp, args: []string{"wide127", "--help"}, kind: commandkit.OutcomeSuccess},
+		{name: "deep_success", app: deepApp, args: deepPath, kind: commandkit.OutcomeSuccess},
+		{name: "deep_help", app: deepApp, args: append(append([]string(nil), deepPath...), "--help"), kind: commandkit.OutcomeSuccess},
+	}
+	for _, benchmark := range benchmarks {
+		b.Run(benchmark.name, func(b *testing.B) {
+			verifyPerformanceOutcome(b, benchmark.app.Execute(context.Background(), benchmark.args, streams), benchmark.kind)
+			b.ReportAllocs()
+			b.ResetTimer()
+			for i := 0; i < b.N; i++ {
+				verifyPerformanceOutcome(b, benchmark.app.Execute(context.Background(), benchmark.args, streams), benchmark.kind)
+			}
+		})
+	}
+
+	flagArgs := []string{"cmd07"}
+	for index := 0; index < 24; index++ {
+		flagArgs = append(flagArgs, fmt.Sprintf("--flag%02d=value", index))
+	}
+	b.Run("flag_heavy_success", func(b *testing.B) {
+		verifyPerformanceOutcome(b, smallApp.Execute(context.Background(), flagArgs, streams), commandkit.OutcomeSuccess)
+		b.ReportAllocs()
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			verifyPerformanceOutcome(b, smallApp.Execute(context.Background(), flagArgs, streams), commandkit.OutcomeSuccess)
+		}
+	})
+
+	for _, fixture := range []struct {
+		name     string
+		commands []commandkit.Command
+	}{
+		{name: "small", commands: small},
+		{name: "wide", commands: wide},
+	} {
+		b.Run("construction_first_execution/"+fixture.name, func(b *testing.B) {
+			b.ReportAllocs()
+			for i := 0; i < b.N; i++ {
+				app := performanceApp(b, fixture.commands)
+				verifyPerformanceOutcome(b, app.Execute(context.Background(), []string{performanceCommandName(fixture.name)}, streams), commandkit.OutcomeSuccess)
+			}
+		})
+	}
+}
+
+func BenchmarkPerformanceRegistration(b *testing.B) {
+	for _, count := range []int{8, 128} {
+		commands := performanceFlatCommands(b, count, "register")
+		b.Run(fmt.Sprintf("individual/%d", count), func(b *testing.B) {
+			b.ReportAllocs()
+			for i := 0; i < b.N; i++ {
+				app, err := commandkit.New(commandkit.Identity{Name: "bench"})
+				if err != nil {
+					b.Fatal(err)
+				}
+				for _, command := range commands {
+					if err := app.Register(command); err != nil {
+						b.Fatal(err)
+					}
+				}
+			}
+		})
+		b.Run(fmt.Sprintf("batch/%d", count), func(b *testing.B) {
+			b.ReportAllocs()
+			for i := 0; i < b.N; i++ {
+				app, err := commandkit.New(commandkit.Identity{Name: "bench"})
+				if err != nil {
+					b.Fatal(err)
+				}
+				if err := app.Register(commands...); err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}
+
+func performanceFlatCommands(tb testing.TB, count int, prefix string) []commandkit.Command {
+	tb.Helper()
+	commands := make([]commandkit.Command, 0, count)
+	for index := 0; index < count; index++ {
+		name := fmt.Sprintf("%s%02d", prefix, index)
+		flags := []commandkit.Flag[struct{}](nil)
+		if prefix == "cmd" && index == 7 {
+			for flagIndex := 0; flagIndex < 24; flagIndex++ {
+				flags = append(flags, commandkit.ValueFlag(commandkit.FlagMeta{Name: fmt.Sprintf("flag%02d", flagIndex)}, (*string)(nil), func(value string) (string, error) { return value, nil }, func(*struct{}, commandkit.Supplied[string]) {}))
+			}
+		}
+		run := func(commandkit.Invocation, struct{}) commandkit.Outcome { return commandkit.Success() }
+		if prefix == "cmd" && index == 4 {
+			run = func(commandkit.Invocation, struct{}) commandkit.Outcome {
+				return commandkit.Usage(performanceUsageCause)
+			}
+		}
+		command, err := commandkit.NewCommand(commandkit.CommandSpec[struct{}]{Name: name, Summary: "Benchmark command.", Flags: flags, Run: run})
+		if err != nil {
+			tb.Fatal(err)
+		}
+		commands = append(commands, command)
+	}
+	return commands
+}
+
+func performanceNestedCommand(tb testing.TB) (commandkit.Command, []string) {
+	tb.Helper()
+	path := make([]string, 24)
+	for index := range path {
+		path[index] = fmt.Sprintf("level%02d", index)
+	}
+	leaf, err := commandkit.NewCommand(commandkit.CommandSpec[struct{}]{Name: path[len(path)-1], Summary: "Nested benchmark command.", Run: func(commandkit.Invocation, struct{}) commandkit.Outcome { return commandkit.Success() }})
+	if err != nil {
+		tb.Fatal(err)
+	}
+	current := leaf
+	for index := len(path) - 2; index >= 0; index-- {
+		sibling, err := commandkit.NewCommand(commandkit.CommandSpec[struct{}]{Name: fmt.Sprintf("sibling%02d", index), Summary: "Sibling command.", Run: func(commandkit.Invocation, struct{}) commandkit.Outcome { return commandkit.Success() }})
+		if err != nil {
+			tb.Fatal(err)
+		}
+		current, err = commandkit.Group(path[index], "Nested benchmark group.", current, sibling)
+		if err != nil {
+			tb.Fatal(err)
+		}
+	}
+	return current, path
+}
+
+func performanceApp(tb testing.TB, commands []commandkit.Command) *commandkit.App {
+	tb.Helper()
+	app, err := commandkit.New(commandkit.Identity{Name: "bench"}, commands...)
+	if err != nil {
+		tb.Fatal(err)
+	}
+	return app
+}
+
+func performanceCommandName(fixture string) string {
+	if fixture == "wide" {
+		return "wide00"
+	}
+	return "cmd00"
+}
+
+func verifyPerformanceOutcome(tb testing.TB, outcome commandkit.Outcome, expected commandkit.OutcomeKind) {
+	tb.Helper()
+	if outcome.Kind != expected {
+		tb.Fatalf("outcome kind = %v, want %v (cause %v)", outcome.Kind, expected, outcome.Cause)
+	}
+}