commandkit/integrations/ckprompt/performance_test.go

package ckprompt

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"strings"
	"testing"

	promptkit "gitea.maximumdirect.net/eric/promptkit"
)

func BenchmarkPerformancePromptReports(b *testing.B) {
	for _, inputCount := range []int{4, 128} {
		inspection := performancePromptInspection(inputCount)
		app := performancePromptApp(b, inspection, performanceProfileInspection(nil))
		for _, format := range []string{"text", "yaml"} {
			name := fmt.Sprintf("prompt/inputs_%d/%s", inputCount, format)
			args := []string{"promptkit", "show", "prompt", "briefing", "--format", format}
			b.Run(name, func(b *testing.B) {
				verifyPromptReport(b, app, args, format, "prompt_id")
				streams := Streams{Out: io.Discard, Err: io.Discard}
				b.ReportAllocs()
				b.ResetTimer()
				for i := 0; i < b.N; i++ {
					if outcome := app.Execute(context.Background(), args, streams); outcome.Kind != OutcomeSuccess {
						b.Fatalf("prompt report outcome = %#v", outcome)
					}
				}
			})
		}
	}

	for _, fixture := range []struct {
		name  string
		extra map[string]any
	}{
		{name: "small", extra: performanceExtraParams(2, 3)},
		{name: "large", extra: performanceExtraParams(24, 8)},
	} {
		profile := performanceProfileInspection(fixture.extra)
		app := performancePromptApp(b, performancePromptInspection(4), profile)
		for _, format := range []string{"text", "yaml"} {
			name := fmt.Sprintf("profile/extra_%s/%s", fixture.name, format)
			args := []string{"promptkit", "show", "profile", "baseline", "--format", format}
			b.Run(name, func(b *testing.B) {
				verifyPromptReport(b, app, args, format, "profile_id")
				streams := Streams{Out: io.Discard, Err: io.Discard}
				b.ReportAllocs()
				b.ResetTimer()
				for i := 0; i < b.N; i++ {
					if outcome := app.Execute(context.Background(), args, streams); outcome.Kind != OutcomeSuccess {
						b.Fatalf("profile report outcome = %#v", outcome)
					}
				}
			})
		}
	}
}

func performancePromptInspection(inputCount int) *promptkit.PromptInspection {
	inputs := make([]promptkit.PromptInputDefinition, inputCount)
	for index := range inputs {
		inputs[index] = promptkit.PromptInputDefinition{
			Name:        fmt.Sprintf("input%03d", index),
			Required:    index%2 == 0,
			ContentType: "text/plain",
			Description: "A representative prompt input.",
		}
	}
	return &promptkit.PromptInspection{
		PromptID:         "briefing",
		PromptVersion:    "1",
		PromptHash:       "fixed-hash",
		DefaultProfileID: "baseline",
		Inputs:           inputs,
		OutputContract: promptkit.OutputContract{
			Format:         promptkit.FormatText,
			ValidationMode: promptkit.ValidationBasic,
			RepairAttempts: 1,
		},
	}
}

func performanceProfileInspection(extra map[string]any) *promptkit.ProfileInspection {
	return &promptkit.ProfileInspection{
		ProfileID: "baseline",
		EffectiveModelParams: promptkit.ExecutionTarget{
			BackendID:   "benchmark-backend",
			Endpoint:    "https://example.invalid/v1",
			Model:       "benchmark-model",
			Temperature: 0.5,
			MaxTokens:   512,
			ExtraParams: extra,
		},
	}
}

func performanceExtraParams(keys, width int) map[string]any {
	values := make(map[string]any, keys)
	for keyIndex := 0; keyIndex < keys; keyIndex++ {
		nested := make(map[string]any, width)
		for valueIndex := 0; valueIndex < width; valueIndex++ {
			nested[fmt.Sprintf("value%02d", valueIndex)] = fmt.Sprintf("sample-%02d-%02d", keyIndex, valueIndex)
		}
		values[fmt.Sprintf("group%02d", keyIndex)] = map[string]any{
			"enabled": keyIndex%2 == 0,
			"nested":  nested,
			"sequence": []any{
				"first",
				keyIndex,
				map[string]any{"leaf": true},
			},
		}
	}
	return values
}

func performancePromptApp(tb testing.TB, prompt *promptkit.PromptInspection, profile *promptkit.ProfileInspection) *App {
	tb.Helper()
	app, err := New(Identity{Name: "bench"})
	if err != nil {
		tb.Fatal(err)
	}
	suite := PromptKitSuite[struct{}]{
		InspectPrompt: func(context.Context, Selection, struct{}, string, string) (Acquired[*promptkit.PromptInspection], error) {
			return Acquired[*promptkit.PromptInspection]{Value: prompt}, nil
		},
		InspectProfile: func(context.Context, Selection, struct{}, string) (Acquired[*promptkit.ProfileInspection], error) {
			return Acquired[*promptkit.ProfileInspection]{Value: profile}, nil
		},
	}
	if err := RegisterPromptKit(app, suite); err != nil {
		tb.Fatal(err)
	}
	return app
}

func verifyPromptReport(tb testing.TB, app *App, args []string, format, marker string) {
	tb.Helper()
	var output bytes.Buffer
	outcome := app.Execute(context.Background(), args, Streams{Out: &output, Err: io.Discard})
	wantMarker := marker
	if format == "text" {
		if marker == "prompt_id" {
			wantMarker = "Prompt ID:"
		} else {
			wantMarker = "Profile ID:"
		}
	}
	if outcome.Kind != OutcomeSuccess || !strings.Contains(output.String(), wantMarker) {
		tb.Fatalf("%s report outcome = %#v, output length %d", format, outcome, output.Len())
	}
	if format == "yaml" && !strings.Contains(output.String(), ":") {
		tb.Fatalf("YAML report lacks mapping entries: %q", output.String())
	}
	if format == "text" && !strings.Contains(output.String(), " ID:") {
		tb.Fatalf("text report has unexpected shape: %q", output.String()[:min(output.Len(), 80)])
	}
}