package ckprompt import ( "bytes" "context" "fmt" "io" "strings" "testing" promptkit "gitea.maximumdirect.net/eric/promptkit" ) func BenchmarkPerformancePromptReports(b *testing.B) { for _, inputCount := range []int{4, 128} { inspection := performancePromptInspection(inputCount) app := performancePromptApp(b, inspection, performanceProfileInspection(nil)) for _, format := range []string{"text", "yaml"} { name := fmt.Sprintf("prompt/inputs_%d/%s", inputCount, format) args := []string{"promptkit", "show", "prompt", "briefing", "--format", format} b.Run(name, func(b *testing.B) { verifyPromptReport(b, app, args, format, "prompt_id") streams := Streams{Out: io.Discard, Err: io.Discard} b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { if outcome := app.Execute(context.Background(), args, streams); outcome.Kind != OutcomeSuccess { b.Fatalf("prompt report outcome = %#v", outcome) } } }) } } for _, fixture := range []struct { name string extra map[string]any }{ {name: "small", extra: performanceExtraParams(2, 3)}, {name: "large", extra: performanceExtraParams(24, 8)}, } { profile := performanceProfileInspection(fixture.extra) app := performancePromptApp(b, performancePromptInspection(4), profile) for _, format := range []string{"text", "yaml"} { name := fmt.Sprintf("profile/extra_%s/%s", fixture.name, format) args := []string{"promptkit", "show", "profile", "baseline", "--format", format} b.Run(name, func(b *testing.B) { verifyPromptReport(b, app, args, format, "profile_id") streams := Streams{Out: io.Discard, Err: io.Discard} b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { if outcome := app.Execute(context.Background(), args, streams); outcome.Kind != OutcomeSuccess { b.Fatalf("profile report outcome = %#v", outcome) } } }) } } } func performancePromptInspection(inputCount int) *promptkit.PromptInspection { inputs := make([]promptkit.PromptInputDefinition, inputCount) for index := range inputs { inputs[index] = promptkit.PromptInputDefinition{ Name: fmt.Sprintf("input%03d", index), Required: index%2 == 0, ContentType: "text/plain", Description: "A representative prompt input.", } } return &promptkit.PromptInspection{ PromptID: "briefing", PromptVersion: "1", PromptHash: "fixed-hash", DefaultProfileID: "baseline", Inputs: inputs, OutputContract: promptkit.OutputContract{ Format: promptkit.FormatText, ValidationMode: promptkit.ValidationBasic, RepairAttempts: 1, }, } } func performanceProfileInspection(extra map[string]any) *promptkit.ProfileInspection { return &promptkit.ProfileInspection{ ProfileID: "baseline", EffectiveModelParams: promptkit.ExecutionTarget{ BackendID: "benchmark-backend", Endpoint: "https://example.invalid/v1", Model: "benchmark-model", Temperature: 0.5, MaxTokens: 512, ExtraParams: extra, }, } } func performanceExtraParams(keys, width int) map[string]any { values := make(map[string]any, keys) for keyIndex := 0; keyIndex < keys; keyIndex++ { nested := make(map[string]any, width) for valueIndex := 0; valueIndex < width; valueIndex++ { nested[fmt.Sprintf("value%02d", valueIndex)] = fmt.Sprintf("sample-%02d-%02d", keyIndex, valueIndex) } values[fmt.Sprintf("group%02d", keyIndex)] = map[string]any{ "enabled": keyIndex%2 == 0, "nested": nested, "sequence": []any{ "first", keyIndex, map[string]any{"leaf": true}, }, } } return values } func performancePromptApp(tb testing.TB, prompt *promptkit.PromptInspection, profile *promptkit.ProfileInspection) *App { tb.Helper() app, err := New(Identity{Name: "bench"}) if err != nil { tb.Fatal(err) } suite := PromptKitSuite[struct{}]{ InspectPrompt: func(context.Context, Selection, struct{}, string, string) (Acquired[*promptkit.PromptInspection], error) { return Acquired[*promptkit.PromptInspection]{Value: prompt}, nil }, InspectProfile: func(context.Context, Selection, struct{}, string) (Acquired[*promptkit.ProfileInspection], error) { return Acquired[*promptkit.ProfileInspection]{Value: profile}, nil }, } if err := RegisterPromptKit(app, suite); err != nil { tb.Fatal(err) } return app } func verifyPromptReport(tb testing.TB, app *App, args []string, format, marker string) { tb.Helper() var output bytes.Buffer outcome := app.Execute(context.Background(), args, Streams{Out: &output, Err: io.Discard}) wantMarker := marker if format == "text" { if marker == "prompt_id" { wantMarker = "Prompt ID:" } else { wantMarker = "Profile ID:" } } if outcome.Kind != OutcomeSuccess || !strings.Contains(output.String(), wantMarker) { tb.Fatalf("%s report outcome = %#v, output length %d", format, outcome, output.Len()) } if format == "yaml" && !strings.Contains(output.String(), ":") { tb.Fatalf("YAML report lacks mapping entries: %q", output.String()) } if format == "text" && !strings.Contains(output.String(), " ID:") { tb.Fatalf("text report has unexpected shape: %q", output.String()[:min(output.Len(), 80)]) } }