Move integration code into optional modules

integrations/ckprompt/performance_test.go

@@ -0,0 +1,161 @@
+package ckprompt
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"strings"
+	"testing"
+
+	promptkit "gitea.maximumdirect.net/eric/promptkit"
+)
+
+func BenchmarkPerformancePromptReports(b *testing.B) {
+	for _, inputCount := range []int{4, 128} {
+		inspection := performancePromptInspection(inputCount)
+		app := performancePromptApp(b, inspection, performanceProfileInspection(nil))
+		for _, format := range []string{"text", "yaml"} {
+			name := fmt.Sprintf("prompt/inputs_%d/%s", inputCount, format)
+			args := []string{"promptkit", "show", "prompt", "briefing", "--format", format}
+			b.Run(name, func(b *testing.B) {
+				verifyPromptReport(b, app, args, format, "prompt_id")
+				streams := Streams{Out: io.Discard, Err: io.Discard}
+				b.ReportAllocs()
+				b.ResetTimer()
+				for i := 0; i < b.N; i++ {
+					if outcome := app.Execute(context.Background(), args, streams); outcome.Kind != OutcomeSuccess {
+						b.Fatalf("prompt report outcome = %#v", outcome)
+					}
+				}
+			})
+		}
+	}
+
+	for _, fixture := range []struct {
+		name  string
+		extra map[string]any
+	}{
+		{name: "small", extra: performanceExtraParams(2, 3)},
+		{name: "large", extra: performanceExtraParams(24, 8)},
+	} {
+		profile := performanceProfileInspection(fixture.extra)
+		app := performancePromptApp(b, performancePromptInspection(4), profile)
+		for _, format := range []string{"text", "yaml"} {
+			name := fmt.Sprintf("profile/extra_%s/%s", fixture.name, format)
+			args := []string{"promptkit", "show", "profile", "baseline", "--format", format}
+			b.Run(name, func(b *testing.B) {
+				verifyPromptReport(b, app, args, format, "profile_id")
+				streams := Streams{Out: io.Discard, Err: io.Discard}
+				b.ReportAllocs()
+				b.ResetTimer()
+				for i := 0; i < b.N; i++ {
+					if outcome := app.Execute(context.Background(), args, streams); outcome.Kind != OutcomeSuccess {
+						b.Fatalf("profile report outcome = %#v", outcome)
+					}
+				}
+			})
+		}
+	}
+}
+
+func performancePromptInspection(inputCount int) *promptkit.PromptInspection {
+	inputs := make([]promptkit.PromptInputDefinition, inputCount)
+	for index := range inputs {
+		inputs[index] = promptkit.PromptInputDefinition{
+			Name:        fmt.Sprintf("input%03d", index),
+			Required:    index%2 == 0,
+			ContentType: "text/plain",
+			Description: "A representative prompt input.",
+		}
+	}
+	return &promptkit.PromptInspection{
+		PromptID:         "briefing",
+		PromptVersion:    "1",
+		PromptHash:       "fixed-hash",
+		DefaultProfileID: "baseline",
+		Inputs:           inputs,
+		OutputContract: promptkit.OutputContract{
+			Format:         promptkit.FormatText,
+			ValidationMode: promptkit.ValidationBasic,
+			RepairAttempts: 1,
+		},
+	}
+}
+
+func performanceProfileInspection(extra map[string]any) *promptkit.ProfileInspection {
+	return &promptkit.ProfileInspection{
+		ProfileID: "baseline",
+		EffectiveModelParams: promptkit.ExecutionTarget{
+			BackendID:   "benchmark-backend",
+			Endpoint:    "https://example.invalid/v1",
+			Model:       "benchmark-model",
+			Temperature: 0.5,
+			MaxTokens:   512,
+			ExtraParams: extra,
+		},
+	}
+}
+
+func performanceExtraParams(keys, width int) map[string]any {
+	values := make(map[string]any, keys)
+	for keyIndex := 0; keyIndex < keys; keyIndex++ {
+		nested := make(map[string]any, width)
+		for valueIndex := 0; valueIndex < width; valueIndex++ {
+			nested[fmt.Sprintf("value%02d", valueIndex)] = fmt.Sprintf("sample-%02d-%02d", keyIndex, valueIndex)
+		}
+		values[fmt.Sprintf("group%02d", keyIndex)] = map[string]any{
+			"enabled": keyIndex%2 == 0,
+			"nested":  nested,
+			"sequence": []any{
+				"first",
+				keyIndex,
+				map[string]any{"leaf": true},
+			},
+		}
+	}
+	return values
+}
+
+func performancePromptApp(tb testing.TB, prompt *promptkit.PromptInspection, profile *promptkit.ProfileInspection) *App {
+	tb.Helper()
+	app, err := New(Identity{Name: "bench"})
+	if err != nil {
+		tb.Fatal(err)
+	}
+	suite := PromptKitSuite[struct{}]{
+		InspectPrompt: func(context.Context, Selection, struct{}, string, string) (Acquired[*promptkit.PromptInspection], error) {
+			return Acquired[*promptkit.PromptInspection]{Value: prompt}, nil
+		},
+		InspectProfile: func(context.Context, Selection, struct{}, string) (Acquired[*promptkit.ProfileInspection], error) {
+			return Acquired[*promptkit.ProfileInspection]{Value: profile}, nil
+		},
+	}
+	if err := RegisterPromptKit(app, suite); err != nil {
+		tb.Fatal(err)
+	}
+	return app
+}
+
+func verifyPromptReport(tb testing.TB, app *App, args []string, format, marker string) {
+	tb.Helper()
+	var output bytes.Buffer
+	outcome := app.Execute(context.Background(), args, Streams{Out: &output, Err: io.Discard})
+	wantMarker := marker
+	if format == "text" {
+		if marker == "prompt_id" {
+			wantMarker = "Prompt ID:"
+		} else {
+			wantMarker = "Profile ID:"
+		}
+	}
+	if outcome.Kind != OutcomeSuccess || !strings.Contains(output.String(), wantMarker) {
+		tb.Fatalf("%s report outcome = %#v, output length %d", format, outcome, output.Len())
+	}
+	if format == "yaml" && !strings.Contains(output.String(), ":") {
+		tb.Fatalf("YAML report lacks mapping entries: %q", output.String())
+	}
+	if format == "text" && !strings.Contains(output.String(), " ID:") {
+		tb.Fatalf("text report has unexpected shape: %q", output.String()[:min(output.Len(), 80)])
+	}
+}