commandkit/performance_reports_test.go

package commandkit

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"strings"
	"testing"

	promptkit "gitea.maximumdirect.net/eric/promptkit"
)

var performanceFramedBytes []byte

func BenchmarkPerformancePromptReports(b *testing.B) {
	for _, inputCount := range []int{4, 128} {
		inspection := performancePromptInspection(inputCount)
		app := performancePromptApp(b, inspection, performanceProfileInspection(nil))
		for _, format := range []string{"text", "yaml"} {
			name := fmt.Sprintf("prompt/inputs_%d/%s", inputCount, format)
			args := []string{"promptkit", "show", "prompt", "briefing", "--format", format}
			b.Run(name, func(b *testing.B) {
				verifyPromptReport(b, app, args, format, "prompt_id")
				streams := Streams{Out: io.Discard, Err: io.Discard}
				b.ReportAllocs()
				b.ResetTimer()
				for i := 0; i < b.N; i++ {
					if outcome := app.Execute(context.Background(), args, streams); outcome.Kind != OutcomeSuccess {
						b.Fatalf("prompt report outcome = %#v", outcome)
					}
				}
			})
		}
	}

	for _, fixture := range []struct {
		name  string
		extra map[string]any
	}{
		{name: "small", extra: performanceExtraParams(2, 3)},
		{name: "large", extra: performanceExtraParams(24, 8)},
	} {
		profile := performanceProfileInspection(fixture.extra)
		app := performancePromptApp(b, performancePromptInspection(4), profile)
		for _, format := range []string{"text", "yaml"} {
			name := fmt.Sprintf("profile/extra_%s/%s", fixture.name, format)
			args := []string{"promptkit", "show", "profile", "baseline", "--format", format}
			b.Run(name, func(b *testing.B) {
				verifyPromptReport(b, app, args, format, "profile_id")
				streams := Streams{Out: io.Discard, Err: io.Discard}
				b.ReportAllocs()
				b.ResetTimer()
				for i := 0; i < b.N; i++ {
					if outcome := app.Execute(context.Background(), args, streams); outcome.Kind != OutcomeSuccess {
						b.Fatalf("profile report outcome = %#v", outcome)
					}
				}
			})
		}
	}
}

func BenchmarkPerformanceYAMLFraming(b *testing.B) {
	fixtures := []struct {
		name   string
		scope  string
		native []byte
	}{
		{name: "4k_ordinary", scope: "app", native: performanceNativeDocument(4<<10, false)},
		{name: "1m_directive", scope: "promptkit", native: performanceNativeDocument(1<<20, true)},
	}
	for _, fixture := range fixtures {
		b.Run("framing/"+fixture.name, func(b *testing.B) {
			framed, hasDirective := frameEffectiveYAML(fixture.scope, fixture.native)
			if len(framed) <= len(fixture.native) || hasDirective != strings.Contains(fixture.name, "directive") {
				b.Fatalf("framing fixture result has unexpected size or directive classification: native=%d framed=%d", len(fixture.native), len(framed))
			}
			b.ReportAllocs()
			b.ResetTimer()
			for i := 0; i < b.N; i++ {
				performanceFramedBytes, _ = frameEffectiveYAML(fixture.scope, fixture.native)
			}
		})
		b.Run("aggregate/"+fixture.name, func(b *testing.B) {
			app := performanceShowApp(b, fixture.native, fixture.native)
			args := []string{"config", "show", "--scope", "all"}
			verifyConfigReport(b, app, args, len(fixture.native)*2)
			streams := Streams{Out: io.Discard, Err: io.Discard}
			b.ReportAllocs()
			b.ResetTimer()
			for i := 0; i < b.N; i++ {
				if outcome := app.Execute(context.Background(), args, streams); outcome.Kind != OutcomeSuccess {
					b.Fatalf("aggregate config show outcome = %#v", outcome)
				}
			}
		})
	}

	single := performanceNativeDocument(4<<10, false)
	app := performanceShowApp(b, single)
	args := []string{"config", "show"}
	b.Run("single_scope/4k_ordinary", func(b *testing.B) {
		verifyConfigReport(b, app, args, len(single))
		streams := Streams{Out: io.Discard, Err: io.Discard}
		b.ReportAllocs()
		b.ResetTimer()
		for i := 0; i < b.N; i++ {
			if outcome := app.Execute(context.Background(), args, streams); outcome.Kind != OutcomeSuccess {
				b.Fatalf("single-scope config show outcome = %#v", outcome)
			}
		}
	})
}

func BenchmarkPerformanceConfigScopes(b *testing.B) {
	for _, scopeCount := range []int{2, 32} {
		suite := performanceConfigSuite(scopeCount, []ConfigOperation{ConfigValidate, ConfigShow})
		app := performanceConfigApp(b, suite)
		selected := fmt.Sprintf("scope%02d", scopeCount-1)
		for _, operation := range []struct {
			name string
			args []string
		}{
			{name: "discovery", args: []string{"config", "scopes"}},
			{name: "named_selection", args: []string{"config", "validate", "--scope", selected}},
			{name: "aggregate_validation", args: []string{"config", "validate", "--scope", "all"}},
		} {
			b.Run(fmt.Sprintf("%s/scopes_%d", operation.name, scopeCount), func(b *testing.B) {
				verifyConfigReport(b, app, operation.args, 1)
				streams := Streams{Out: io.Discard, Err: io.Discard}
				b.ReportAllocs()
				b.ResetTimer()
				for i := 0; i < b.N; i++ {
					if outcome := app.Execute(context.Background(), operation.args, streams); outcome.Kind != OutcomeSuccess {
						b.Fatalf("configuration operation outcome = %#v", outcome)
					}
				}
			})
		}
	}
}

func BenchmarkPerformanceConfigRegistration(b *testing.B) {
	for _, selectorCount := range []int{4, 64} {
		suite := performanceSelectorSuite(selectorCount)
		performanceConfigApp(b, suite)
		b.Run(fmt.Sprintf("selectors_%d", selectorCount), func(b *testing.B) {
			b.ReportAllocs()
			for i := 0; i < b.N; i++ {
				app, err := New(Identity{Name: "bench"})
				if err != nil {
					b.Fatal(err)
				}
				if err := RegisterConfig(app, suite); err != nil {
					b.Fatal(err)
				}
			}
		})
	}
}

func performancePromptInspection(inputCount int) *promptkit.PromptInspection {
	inputs := make([]promptkit.PromptInputDefinition, inputCount)
	for index := range inputs {
		inputs[index] = promptkit.PromptInputDefinition{
			Name:        fmt.Sprintf("input%03d", index),
			Required:    index%2 == 0,
			ContentType: "text/plain",
			Description: "A representative prompt input.",
		}
	}
	return &promptkit.PromptInspection{
		PromptID:         "briefing",
		PromptVersion:    "1",
		PromptHash:       "fixed-hash",
		DefaultProfileID: "baseline",
		Inputs:           inputs,
		OutputContract: promptkit.OutputContract{
			Format:         promptkit.FormatText,
			ValidationMode: promptkit.ValidationBasic,
			RepairAttempts: 1,
		},
	}
}

func performanceProfileInspection(extra map[string]any) *promptkit.ProfileInspection {
	return &promptkit.ProfileInspection{
		ProfileID: "baseline",
		EffectiveModelParams: promptkit.ExecutionTarget{
			BackendID:   "benchmark-backend",
			Endpoint:    "https://example.invalid/v1",
			Model:       "benchmark-model",
			Temperature: 0.5,
			MaxTokens:   512,
			ExtraParams: extra,
		},
	}
}

func performanceExtraParams(keys, width int) map[string]any {
	values := make(map[string]any, keys)
	for keyIndex := 0; keyIndex < keys; keyIndex++ {
		nested := make(map[string]any, width)
		for valueIndex := 0; valueIndex < width; valueIndex++ {
			nested[fmt.Sprintf("value%02d", valueIndex)] = fmt.Sprintf("sample-%02d-%02d", keyIndex, valueIndex)
		}
		values[fmt.Sprintf("group%02d", keyIndex)] = map[string]any{
			"enabled": keyIndex%2 == 0,
			"nested":  nested,
			"sequence": []any{
				"first",
				keyIndex,
				map[string]any{"leaf": true},
			},
		}
	}
	return values
}

func performancePromptApp(tb testing.TB, prompt *promptkit.PromptInspection, profile *promptkit.ProfileInspection) *App {
	tb.Helper()
	app, err := New(Identity{Name: "bench"})
	if err != nil {
		tb.Fatal(err)
	}
	suite := PromptKitSuite[struct{}]{
		InspectPrompt: func(context.Context, Selection, struct{}, string, string) (Acquired[*promptkit.PromptInspection], error) {
			return Acquired[*promptkit.PromptInspection]{Value: prompt}, nil
		},
		InspectProfile: func(context.Context, Selection, struct{}, string) (Acquired[*promptkit.ProfileInspection], error) {
			return Acquired[*promptkit.ProfileInspection]{Value: profile}, nil
		},
	}
	if err := RegisterPromptKit(app, suite); err != nil {
		tb.Fatal(err)
	}
	return app
}

func performanceNativeDocument(targetBytes int, directive bool) []byte {
	preamble := "# generated native configuration\n"
	if directive {
		preamble = "%YAML 1.2\n---\n"
	}
	var body strings.Builder
	body.Grow(targetBytes)
	for index := 0; len(preamble)+body.Len() < targetBytes; index++ {
		fmt.Fprintf(&body, "key%06d: value\n", index)
	}
	return []byte(preamble + body.String())
}

type performanceEffectiveWriter []byte

func (document performanceEffectiveWriter) WriteEffective(writer io.Writer) error {
	_, err := writer.Write(document)
	return err
}

func performanceShowApp(tb testing.TB, documents ...[]byte) *App {
	tb.Helper()
	scopes := make([]ConfigScope[struct{}], len(documents))
	for index, document := range documents {
		name := "app"
		if index > 0 {
			name = fmt.Sprintf("scope%02d", index)
		}
		writer := performanceEffectiveWriter(document)
		scopes[index] = ConfigScope[struct{}]{
			Name:       name,
			Operations: []ConfigOperation{ConfigShow},
			Acquire: func(context.Context, ConfigRequest, struct{}) (Acquired[ConfigView], error) {
				return Acquired[ConfigView]{Value: ConfigView{Effective: writer}}, nil
			},
		}
	}
	suite := ConfigSuite[struct{}]{Scopes: scopes}
	return performanceConfigApp(tb, suite)
}

func performanceConfigSuite(scopeCount int, operations []ConfigOperation) ConfigSuite[struct{}] {
	scopes := make([]ConfigScope[struct{}], scopeCount)
	for index := range scopes {
		name := "app"
		if index > 0 {
			name = fmt.Sprintf("scope%02d", index)
		}
		scopes[index] = ConfigScope[struct{}]{
			Name:       name,
			Operations: append([]ConfigOperation(nil), operations...),
			Acquire: func(context.Context, ConfigRequest, struct{}) (Acquired[ConfigView], error) {
				return Acquired[ConfigView]{}, nil
			},
		}
	}
	return ConfigSuite[struct{}]{Scopes: scopes}
}

func performanceSelectorSuite(selectorCount int) ConfigSuite[struct{}] {
	suite := performanceConfigSuite(1, []ConfigOperation{ConfigValidate})
	suite.Selectors = make([]Selector[struct{}], selectorCount)
	for index := range suite.Selectors {
		flag := ValueFlag(FlagMeta{Name: fmt.Sprintf("selector%02d", index)}, (*string)(nil), parseString, func(*struct{}, Supplied[string]) {})
		suite.Selectors[index] = Selector[struct{}]{Binding: flag, Scopes: []string{"app"}}
	}
	return suite
}

func performanceConfigApp(tb testing.TB, suite ConfigSuite[struct{}]) *App {
	tb.Helper()
	app, err := New(Identity{Name: "bench"})
	if err != nil {
		tb.Fatal(err)
	}
	if err := RegisterConfig(app, suite); err != nil {
		tb.Fatal(err)
	}
	return app
}

func verifyPromptReport(tb testing.TB, app *App, args []string, format, marker string) {
	tb.Helper()
	var output bytes.Buffer
	outcome := app.Execute(context.Background(), args, Streams{Out: &output, Err: io.Discard})
	wantMarker := marker
	if format == "text" {
		if marker == "prompt_id" {
			wantMarker = "Prompt ID:"
		} else {
			wantMarker = "Profile ID:"
		}
	}
	if outcome.Kind != OutcomeSuccess || !strings.Contains(output.String(), wantMarker) {
		tb.Fatalf("%s report outcome = %#v, output length %d", format, outcome, output.Len())
	}
	if format == "yaml" && !strings.Contains(output.String(), ":") {
		tb.Fatalf("YAML report lacks mapping entries: %q", output.String())
	}
	if format == "text" && !strings.Contains(output.String(), " ID:") {
		tb.Fatalf("text report has unexpected shape: %q", output.String()[:min(output.Len(), 80)])
	}
}

func verifyConfigReport(tb testing.TB, app *App, args []string, minimumBytes int) {
	tb.Helper()
	var output bytes.Buffer
	outcome := app.Execute(context.Background(), args, Streams{Out: &output, Err: io.Discard})
	if outcome.Kind != OutcomeSuccess || output.Len() < minimumBytes {
		tb.Fatalf("configuration report outcome = %#v, output bytes %d, want at least %d", outcome, output.Len(), minimumBytes)
	}
}