commandkit/integrations/ckconfig/performance_test.go

package ckconfig

import (
	"context"
	"fmt"
	"io"
	"strings"
	"testing"
)

var performanceFramedBytes []byte

func BenchmarkPerformanceYAMLFraming(b *testing.B) {
	fixtures := []struct {
		name   string
		scope  string
		native []byte
	}{
		{name: "4k_ordinary", scope: "app", native: performanceNativeDocument(4<<10, false)},
		{name: "1m_directive", scope: "promptkit", native: performanceNativeDocument(1<<20, true)},
	}
	for _, fixture := range fixtures {
		b.Run("framing/"+fixture.name, func(b *testing.B) {
			framed, hasDirective := frameEffectiveYAML(fixture.scope, fixture.native)
			if len(framed) <= len(fixture.native) || hasDirective != strings.Contains(fixture.name, "directive") {
				b.Fatalf("framing fixture result has unexpected size or directive classification: native=%d framed=%d", len(fixture.native), len(framed))
			}
			b.ReportAllocs()
			b.ResetTimer()
			for i := 0; i < b.N; i++ {
				performanceFramedBytes, _ = frameEffectiveYAML(fixture.scope, fixture.native)
			}
		})
		b.Run("aggregate/"+fixture.name, func(b *testing.B) {
			app := performanceShowApp(b, fixture.native, fixture.native)
			args := []string{"config", "show", "--scope", "all"}
			verifyConfigReport(b, app, args, len(fixture.native)*2)
			streams := Streams{Out: io.Discard, Err: io.Discard}
			b.ReportAllocs()
			b.ResetTimer()
			for i := 0; i < b.N; i++ {
				if outcome := app.Execute(context.Background(), args, streams); outcome.Kind != OutcomeSuccess {
					b.Fatalf("aggregate config show outcome = %#v", outcome)
				}
			}
		})
	}

	single := performanceNativeDocument(4<<10, false)
	app := performanceShowApp(b, single)
	args := []string{"config", "show"}
	b.Run("single_scope/4k_ordinary", func(b *testing.B) {
		verifyConfigReport(b, app, args, len(single))
		streams := Streams{Out: io.Discard, Err: io.Discard}
		b.ReportAllocs()
		b.ResetTimer()
		for i := 0; i < b.N; i++ {
			if outcome := app.Execute(context.Background(), args, streams); outcome.Kind != OutcomeSuccess {
				b.Fatalf("single-scope config show outcome = %#v", outcome)
			}
		}
	})
}

func BenchmarkPerformanceConfigScopes(b *testing.B) {
	for _, scopeCount := range []int{2, 32} {
		suite := performanceConfigSuite(scopeCount, []ConfigOperation{ConfigValidate, ConfigShow})
		app := performanceConfigApp(b, suite)
		selected := fmt.Sprintf("scope%02d", scopeCount-1)
		for _, operation := range []struct {
			name string
			args []string
		}{
			{name: "discovery", args: []string{"config", "scopes"}},
			{name: "named_selection", args: []string{"config", "validate", "--scope", selected}},
			{name: "aggregate_validation", args: []string{"config", "validate", "--scope", "all"}},
		} {
			b.Run(fmt.Sprintf("%s/scopes_%d", operation.name, scopeCount), func(b *testing.B) {
				verifyConfigReport(b, app, operation.args, 1)
				streams := Streams{Out: io.Discard, Err: io.Discard}
				b.ReportAllocs()
				b.ResetTimer()
				for i := 0; i < b.N; i++ {
					if outcome := app.Execute(context.Background(), operation.args, streams); outcome.Kind != OutcomeSuccess {
						b.Fatalf("configuration operation outcome = %#v", outcome)
					}
				}
			})
		}
	}
}

func BenchmarkPerformanceConfigRegistration(b *testing.B) {
	for _, selectorCount := range []int{4, 64} {
		suite := performanceSelectorSuite(selectorCount)
		performanceConfigApp(b, suite)
		b.Run(fmt.Sprintf("selectors_%d", selectorCount), func(b *testing.B) {
			b.ReportAllocs()
			for i := 0; i < b.N; i++ {
				app, err := New(Identity{Name: "bench"})
				if err != nil {
					b.Fatal(err)
				}
				if err := RegisterConfig(app, suite); err != nil {
					b.Fatal(err)
				}
			}
		})
	}
}

func performanceNativeDocument(targetBytes int, directive bool) []byte {
	preamble := "# generated native configuration\n"
	if directive {
		preamble = "%YAML 1.2\n---\n"
	}
	var body strings.Builder
	body.Grow(targetBytes)
	for index := 0; len(preamble)+body.Len() < targetBytes; index++ {
		fmt.Fprintf(&body, "key%06d: value\n", index)
	}
	return []byte(preamble + body.String())
}

type performanceEffectiveWriter []byte

func (document performanceEffectiveWriter) WriteEffective(writer io.Writer) error {
	_, err := writer.Write(document)
	return err
}

func performanceShowApp(tb testing.TB, documents ...[]byte) *App {
	tb.Helper()
	scopes := make([]ConfigScope[struct{}], len(documents))
	for index, document := range documents {
		name := "app"
		if index > 0 {
			name = fmt.Sprintf("scope%02d", index)
		}
		writer := performanceEffectiveWriter(document)
		scopes[index] = ConfigScope[struct{}]{
			Name:       name,
			Operations: []ConfigOperation{ConfigShow},
			Acquire: func(context.Context, ConfigRequest, struct{}) (Acquired[ConfigView], error) {
				return Acquired[ConfigView]{Value: ConfigView{Effective: writer}}, nil
			},
		}
	}
	suite := ConfigSuite[struct{}]{Scopes: scopes}
	return performanceConfigApp(tb, suite)
}

func performanceConfigSuite(scopeCount int, operations []ConfigOperation) ConfigSuite[struct{}] {
	scopes := make([]ConfigScope[struct{}], scopeCount)
	for index := range scopes {
		name := "app"
		if index > 0 {
			name = fmt.Sprintf("scope%02d", index)
		}
		scopes[index] = ConfigScope[struct{}]{
			Name:       name,
			Operations: append([]ConfigOperation(nil), operations...),
			Acquire: func(context.Context, ConfigRequest, struct{}) (Acquired[ConfigView], error) {
				return Acquired[ConfigView]{}, nil
			},
		}
	}
	return ConfigSuite[struct{}]{Scopes: scopes}
}

func performanceSelectorSuite(selectorCount int) ConfigSuite[struct{}] {
	suite := performanceConfigSuite(1, []ConfigOperation{ConfigValidate})
	suite.Selectors = make([]Selector[struct{}], selectorCount)
	for index := range suite.Selectors {
		flag := ValueFlag(FlagMeta{Name: fmt.Sprintf("selector%02d", index)}, (*string)(nil), parseString, func(*struct{}, Supplied[string]) {})
		suite.Selectors[index] = Selector[struct{}]{Binding: flag, Scopes: []string{"app"}}
	}
	return suite
}

func performanceConfigApp(tb testing.TB, suite ConfigSuite[struct{}]) *App {
	tb.Helper()
	app, err := New(Identity{Name: "bench"})
	if err != nil {
		tb.Fatal(err)
	}
	if err := RegisterConfig(app, suite); err != nil {
		tb.Fatal(err)
	}
	return app
}

func verifyConfigReport(tb testing.TB, app *App, args []string, minimumBytes int) {
	tb.Helper()
	var output bytes.Buffer
	outcome := app.Execute(context.Background(), args, Streams{Out: &output, Err: io.Discard})
	if outcome.Kind != OutcomeSuccess || output.Len() < minimumBytes {
		tb.Fatalf("configuration report outcome = %#v, output bytes %d, want at least %d", outcome, output.Len(), minimumBytes)
	}
}