package commandkit_test import ( "bytes" "context" "errors" "io" "reflect" "strings" "testing" "testing/fstest" commandkit "gitea.maximumdirect.net/eric/CommandKit" configkit "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/promptkit" "go.yaml.in/yaml/v4" ) const promptKitInspectionFixture = `prompts: "team/summary": "2": default_profile: child inputs: - name: topic required: true content_type: text/plain description: "Topic to summarize" - name: audience required: false content_type: text/plain description: "" messages: - role: user content: "Summarize {{.topic}}" output: format: json validation_mode: basic repair_attempts: 2 "3": messages: - role: user content: "Version three" output: format: text validation_mode: none empty: "1": messages: - role: user content: "No inputs" output: format: text validation_mode: none ` func promptKitInspectionEngine(t *testing.T) *promptkit.Engine { t.Helper() inputs, err := promptkit.FilesystemInputs("prompt fixture", fstest.MapFS{ "config/prompts.yml": &fstest.MapFile{Data: []byte(promptKitInspectionFixture)}, }, "config") if err != nil { t.Fatal(err) } engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ Underlays: []configkit.Underlay{ {Inputs: inputs}, {Inputs: []configkit.Input{{Source: configkit.Mapping("profile fixture", map[string]any{ "backends": map[string]any{"local": map[string]any{"endpoint": "https://example.invalid/v1"}}, "profiles": map[string]any{ "parent": map[string]any{ "backend": "local", "model": "base-model", "temperature": 0.7, "max_tokens": 100, "top_p": 0.8, "extra_params": map[string]any{ "token": "provider-private-sentinel", "nested": map[string]any{"null_value": nil, "visible": "kept"}, "sequence": []any{"first", nil, 3}, }, }, "child": map[string]any{"extends": "parent", "temperature": 0, "max_tokens": 0, "api_key_env": "PROMPTKIT_TEST_KEY"}, "requires-key": map[string]any{"backend": "local", "model": "key-model", "api_key_required": true}, "no-extra": map[string]any{"backend": "local", "model": "empty-params"}, }, })}}}, }, }}) if err != nil { t.Fatalf("construct PromptKit fixture engine: %v", err) } return engine } func TestPromptKitInspectionCommandsReportRealCapturedDefinitions(t *testing.T) { t.Setenv("PROMPTKIT_TEST_KEY", "credential-private-sentinel") engine := promptKitInspectionEngine(t) loadCalls := 0 suite, err := commandkit.NewPromptKitSuite(func(context.Context, commandkit.Selection, struct{}) (*promptkit.Engine, error) { loadCalls++ return engine, nil }) if err != nil { t.Fatal(err) } app, err := commandkit.New(commandkit.Identity{Name: "example"}) if err != nil { t.Fatal(err) } if err := commandkit.RegisterPromptKit(app, suite); err != nil { t.Fatal(err) } var promptYAML bytes.Buffer outcome := app.Execute(context.Background(), []string{"promptkit", "show", "prompt", "team/summary", "--version", "2", "--format", "yaml"}, commandkit.Streams{Out: &promptYAML, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess || loadCalls != 1 { t.Fatalf("prompt inspection outcome=%#v calls=%d report=%q", outcome, loadCalls, promptYAML.String()) } var promptReport map[string]any if err := yaml.Unmarshal(promptYAML.Bytes(), &promptReport); err != nil { t.Fatalf("decode prompt YAML: %v", err) } if !reflect.DeepEqual(sortedKeys(promptReport), []string{"default_profile_id", "inputs", "output_contract", "prompt_hash", "prompt_id", "prompt_version"}) { t.Fatalf("prompt report keys = %v", sortedKeys(promptReport)) } if promptReport["prompt_id"] != "team/summary" || promptReport["prompt_version"] != "2" || promptReport["default_profile_id"] != "child" || promptReport["prompt_hash"] == "" { t.Fatalf("prompt report identity = %#v", promptReport) } inputs := promptReport["inputs"].([]any) if len(inputs) != 2 || inputs[0].(map[string]any)["name"] != "topic" || inputs[1].(map[string]any)["name"] != "audience" { t.Fatalf("prompt inputs/order = %#v", inputs) } outputContract := promptReport["output_contract"].(map[string]any) if outputContract["format"] != "json" || outputContract["validation_mode"] != "basic" || outputContract["repair_attempts"] != 2 { t.Fatalf("output contract = %#v", outputContract) } var promptText bytes.Buffer outcome = app.Execute(context.Background(), []string{"promptkit", "show", "prompt", "team/summary", "--version", "2"}, commandkit.Streams{Out: &promptText, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(promptText.String(), `Prompt hash: "`) || !strings.Contains(promptText.String(), `- name: "topic"`) || !strings.Contains(promptText.String(), `validation_mode: "basic"`) { t.Fatalf("prompt text = (%#v, %q)", outcome, promptText.String()) } var omittedVersion bytes.Buffer outcome = app.Execute(context.Background(), []string{"promptkit", "show", "prompt", "empty", "--format", "yaml"}, commandkit.Streams{Out: &omittedVersion, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(omittedVersion.String(), "prompt_version: \"1\"") { t.Fatalf("omitted singleton prompt version = (%#v, %q)", outcome, omittedVersion.String()) } var profileYAML bytes.Buffer outcome = app.Execute(context.Background(), []string{"promptkit", "show", "profile", "child", "--format", "yaml"}, commandkit.Streams{Out: &profileYAML, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess { t.Fatalf("profile inspection outcome=%#v report=%q", outcome, profileYAML.String()) } var profileReport map[string]any if err := yaml.Unmarshal(profileYAML.Bytes(), &profileReport); err != nil { t.Fatalf("decode profile YAML: %v", err) } if !reflect.DeepEqual(sortedKeys(profileReport), []string{"api_key_required", "effective_model_params", "profile_id"}) || profileReport["profile_id"] != "child" || profileReport["api_key_required"] != false { t.Fatalf("profile report = %#v", profileReport) } params := profileReport["effective_model_params"].(map[string]any) if !reflect.DeepEqual(sortedKeys(params), []string{"api_key_env", "backend_id", "endpoint", "extra_params", "max_tokens", "model", "reasoning_effort", "service_tier", "temperature", "timeout_seconds", "top_p"}) { t.Fatalf("model parameter keys = %v", sortedKeys(params)) } if params["temperature"] != 0 || params["max_tokens"] != 0 || params["api_key_env"] != "PROMPTKIT_TEST_KEY" { t.Fatalf("effective owning values = %#v", params) } extra := params["extra_params"].(map[string]any) nested := extra["nested"].(map[string]any) sequence := extra["sequence"].([]any) if nested["null_value"] != nil || nested["visible"] != "kept" || len(sequence) != 3 || sequence[1] != nil || sequence[2] != 3 { t.Fatalf("nested provider parameters changed: %#v", extra) } if extra["token"] != "provider-private-sentinel" || strings.Contains(profileYAML.String(), "credential-private-sentinel") { t.Fatalf("provider parameters or credential reference changed: %q", profileYAML.String()) } var profileText bytes.Buffer outcome = app.Execute(context.Background(), []string{"promptkit", "show", "profile", "child"}, commandkit.Streams{Out: &profileText, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(profileText.String(), "temperature: 0\n") || !strings.Contains(profileText.String(), "api_key_env: \"PROMPTKIT_TEST_KEY\"") || !strings.Contains(profileText.String(), "null_value: null") { t.Fatalf("profile text = (%#v, %q)", outcome, profileText.String()) } var requiredKey bytes.Buffer outcome = app.Execute(context.Background(), []string{"promptkit", "show", "profile", "requires-key", "--format", "yaml"}, commandkit.Streams{Out: &requiredKey, Err: io.Discard}) var requiredReport map[string]any if outcome.Kind != commandkit.OutcomeSuccess || yaml.Unmarshal(requiredKey.Bytes(), &requiredReport) != nil || requiredReport["api_key_required"] != true { t.Fatalf("credential requirement report = (%#v, %q)", outcome, requiredKey.String()) } var emptyPrompt bytes.Buffer outcome = app.Execute(context.Background(), []string{"promptkit", "show", "prompt", "empty", "--version", "1", "--format", "yaml"}, commandkit.Streams{Out: &emptyPrompt, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(emptyPrompt.String(), "inputs: []") { t.Fatalf("empty inputs report = (%#v, %q)", outcome, emptyPrompt.String()) } var zeroExtra bytes.Buffer outcome = app.Execute(context.Background(), []string{"promptkit", "show", "profile", "no-extra", "--format", "yaml"}, commandkit.Streams{Out: &zeroExtra, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(zeroExtra.String(), "extra_params: {}") { t.Fatalf("nil extra params report = (%#v, %q)", outcome, zeroExtra.String()) } } func TestPromptKitInspectionSelectionErrorsHelpAndOptionalCapabilities(t *testing.T) { engine := promptKitInspectionEngine(t) loaderCalls := 0 suite, err := commandkit.NewPromptKitSuite(func(context.Context, commandkit.Selection, struct{}) (*promptkit.Engine, error) { loaderCalls++ return engine, nil }) if err != nil { t.Fatal(err) } app, _ := commandkit.New(commandkit.Identity{Name: "example"}) if err := commandkit.RegisterPromptKit(app, suite); err != nil { t.Fatal(err) } badTarget := suite badTarget.Selectors = []commandkit.Selector[struct{}]{{ Binding: commandkit.ValueFlag[struct{}](commandkit.FlagMeta{Name: "campaign"}, nil, func(value string) (string, error) { return value, nil }, func(*struct{}, commandkit.Supplied[string]) {}), Scopes: []string{"app"}, }} invalidApp, _ := commandkit.New(commandkit.Identity{Name: "invalid"}) if err := commandkit.RegisterPromptKit(invalidApp, badTarget); err == nil { t.Fatal("selector for another scope was accepted") } for _, tc := range []struct { args []string want error }{ {args: []string{"promptkit", "show", "prompt", "team/summary"}, want: promptkit.ErrPromptLoad}, {args: []string{"promptkit", "show", "prompt", "missing", "--version", "1"}, want: promptkit.ErrPromptNotFound}, {args: []string{"promptkit", "show", "profile", "missing"}, want: promptkit.ErrProfileNotFound}, {args: []string{"promptkit", "show", "prompt", "", "--version", "2"}, want: promptkit.ErrInvalidRequest}, } { outcome := app.Execute(context.Background(), tc.args, commandkit.Streams{Out: io.Discard, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, tc.want) { t.Errorf("execute %q = (%#v), want operational error matching %v", tc.args, outcome, tc.want) } } cancelled, cancel := context.WithCancel(context.Background()) cancel() outcome := app.Execute(cancelled, []string{"promptkit", "show", "prompt", "empty", "--version", "1"}, commandkit.Streams{Out: io.Discard, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, context.Canceled) { t.Fatalf("cancelled prompt inspection = %#v", outcome) } beforeHelp := loaderCalls outcome = app.Execute(context.Background(), []string{"promptkit", "show", "prompt", "--help"}, commandkit.Streams{Out: io.Discard, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess || loaderCalls != beforeHelp { t.Fatalf("help acquired engine: outcome=%#v calls=%d", outcome, loaderCalls) } beforeMissing := loaderCalls outcome = app.Execute(context.Background(), []string{"promptkit", "show", "prompt"}, commandkit.Streams{Out: io.Discard, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeUsage || loaderCalls != beforeMissing { t.Fatalf("missing ID = (%#v, calls=%d), want usage before acquisition", outcome, loaderCalls) } beforeScopeFlag := loaderCalls outcome = app.Execute(context.Background(), []string{"promptkit", "show", "profile", "child", "--scope", "promptkit"}, commandkit.Streams{Out: io.Discard, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeUsage || loaderCalls != beforeScopeFlag { t.Fatalf("definition command accepted --scope: outcome=%#v calls=%d", outcome, loaderCalls) } profileOnlyApp, _ := commandkit.New(commandkit.Identity{Name: "profile-only"}) if err := commandkit.RegisterPromptKit(profileOnlyApp, commandkit.PromptKitSuite[struct{}]{InspectProfile: func(ctx context.Context, _ commandkit.Selection, _ struct{}, id string) (commandkit.Acquired[*promptkit.ProfileInspection], error) { value, err := engine.InspectProfile(ctx, id) return commandkit.Acquired[*promptkit.ProfileInspection]{Value: value}, err }}); err != nil { t.Fatal(err) } outcome = profileOnlyApp.Execute(context.Background(), []string{"promptkit", "show", "profile", "child"}, commandkit.Streams{Out: io.Discard, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess { t.Fatalf("profile-only capability was not installed: %#v", outcome) } outcome = profileOnlyApp.Execute(context.Background(), []string{"promptkit", "show", "prompt", "empty"}, commandkit.Streams{Out: io.Discard, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeUsage { t.Fatalf("profile-only suite installed prompt inspection: %#v", outcome) } promptOnlyApp, _ := commandkit.New(commandkit.Identity{Name: "prompt-only"}) if err := commandkit.RegisterPromptKit(promptOnlyApp, commandkit.PromptKitSuite[struct{}]{InspectPrompt: func(ctx context.Context, _ commandkit.Selection, _ struct{}, id, version string) (commandkit.Acquired[*promptkit.PromptInspection], error) { value, err := engine.InspectPrompt(ctx, id, version) return commandkit.Acquired[*promptkit.PromptInspection]{Value: value}, err }}); err != nil { t.Fatal(err) } outcome = promptOnlyApp.Execute(context.Background(), []string{"promptkit", "show", "prompt", "empty"}, commandkit.Streams{Out: io.Discard, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess { t.Fatalf("prompt-only capability was not installed: %#v", outcome) } outcome = promptOnlyApp.Execute(context.Background(), []string{"promptkit", "show", "profile", "child"}, commandkit.Streams{Out: io.Discard, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeUsage { t.Fatalf("prompt-only suite installed profile inspection: %#v", outcome) } emptyApp, _ := commandkit.New(commandkit.Identity{Name: "empty"}) if err := commandkit.RegisterPromptKit(emptyApp, commandkit.PromptKitSuite[struct{}]{}); err != nil { t.Fatalf("empty PromptKit suite = %v", err) } outcome = emptyApp.Execute(context.Background(), []string{"help"}, commandkit.Streams{Out: io.Discard, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess { t.Fatalf("empty PromptKit suite changed command family: %#v", outcome) } outcome = emptyApp.Execute(context.Background(), []string{"help", "promptkit"}, commandkit.Streams{Out: io.Discard, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeUsage { t.Fatalf("empty PromptKit suite installed its namespace: %#v", outcome) } } func TestPromptKitSuitePassesSelectionAndProjectedContext(t *testing.T) { type appContext struct { Campaign commandkit.Supplied[string] Runtime string } type contextKey struct{} engine := promptKitInspectionEngine(t) ctx := context.WithValue(context.Background(), contextKey{}, "caller") wantSelection := commandkit.Selection{ Config: commandkit.Supplied[string]{Value: "main.yml", Set: true}, Overlay: commandkit.Supplied[string]{Value: "dev", Set: true}, } loaderCalls, contexts, checks := 0, 0, 0 var gotSelection commandkit.Selection var gotContext appContext suite, err := commandkit.NewPromptKitSuite(func(gotCtx context.Context, selection commandkit.Selection, input appContext) (*promptkit.Engine, error) { loaderCalls++ gotSelection, gotContext = selection, input if gotCtx != ctx || gotCtx.Value(contextKey{}) != "caller" { t.Fatal("engine loader did not receive invocation context") } return engine, nil }) if err != nil { t.Fatal(err) } suite.NewContext = func() appContext { contexts++; return appContext{Runtime: "invocation cache"} } suite.Selectors = []commandkit.Selector[appContext]{{ Binding: commandkit.ValueFlag[appContext](commandkit.FlagMeta{Name: "campaign"}, nil, func(value string) (string, error) { return value, nil }, func(input *appContext, value commandkit.Supplied[string]) { input.Campaign = value }), Scopes: []string{"promptkit"}, }} suite.CheckInput = func(selection commandkit.Selection, input appContext) error { checks++ if selection != wantSelection || input.Campaign != (commandkit.Supplied[string]{Value: "spring", Set: true}) { t.Errorf("preflight inputs = (%+v, %+v)", selection, input) } return nil } app, _ := commandkit.New(commandkit.Identity{Name: "example"}) if err := commandkit.RegisterPromptKit(app, suite); err != nil { t.Fatal(err) } outcome := app.Execute(ctx, []string{"promptkit", "show", "profile", "child", "--config", "main.yml", "--overlay", "dev", "--campaign", "spring"}, commandkit.Streams{Out: io.Discard, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess || loaderCalls != 1 || contexts != 1 || checks != 1 { t.Fatalf("inspection = (%#v), loader/context/check calls = %d/%d/%d", outcome, loaderCalls, contexts, checks) } if gotSelection != wantSelection || gotContext.Campaign != (commandkit.Supplied[string]{Value: "spring", Set: true}) || gotContext.Runtime != "invocation cache" { t.Fatalf("loader selection/context = (%+v, %+v)", gotSelection, gotContext) } } func TestPromptKitDirectSuiteRetainsDiagnosticsOnNamedLookupFailure(t *testing.T) { warning := commandkit.Diagnostic{Severity: configkit.SeverityWarning, Write: func(w io.Writer) error { _, err := io.WriteString(w, "configuration warning\n"); return err }} app, _ := commandkit.New(commandkit.Identity{Name: "example"}) err := commandkit.RegisterPromptKit(app, commandkit.PromptKitSuite[struct{}]{InspectPrompt: func(context.Context, commandkit.Selection, struct{}, string, string) (commandkit.Acquired[*promptkit.PromptInspection], error) { return commandkit.Acquired[*promptkit.PromptInspection]{Diagnostics: []commandkit.Diagnostic{warning}}, promptkit.ErrPromptNotFound }}) if err != nil { t.Fatal(err) } outcome := app.Execute(context.Background(), []string{"promptkit", "show", "prompt", "missing"}, commandkit.Streams{Out: io.Discard, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, promptkit.ErrPromptNotFound) || len(outcome.Diagnostics) < 1 || outcome.Diagnostics[0].Severity != configkit.SeverityWarning { t.Fatalf("named lookup outcome did not retain acquisition diagnostic: %#v", outcome) } } func TestPromptKitInspectionReportsOutputDeliveryFailure(t *testing.T) { engine := promptKitInspectionEngine(t) suite, err := commandkit.NewPromptKitSuite(func(context.Context, commandkit.Selection, struct{}) (*promptkit.Engine, error) { return engine, nil }) if err != nil { t.Fatal(err) } app, _ := commandkit.New(commandkit.Identity{Name: "example"}) if err := commandkit.RegisterPromptKit(app, suite); err != nil { t.Fatal(err) } writeErr := errors.New("report output failed") outcome := app.Execute(context.Background(), []string{"promptkit", "show", "profile", "child", "--format", "yaml"}, commandkit.Streams{Out: promptKitFailingWriter{err: writeErr}, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeOperational || !outcome.ReportIncomplete || !errors.Is(outcome.DeliveryError, writeErr) { t.Fatalf("promptkit output failure = %#v", outcome) } } type promptKitFailingWriter struct{ err error } func (writer promptKitFailingWriter) Write([]byte) (int, error) { return 0, writer.err } func sortedKeys(values map[string]any) []string { keys := make([]string, 0, len(values)) for key := range values { keys = append(keys, key) } for i := 0; i < len(keys); i++ { for j := i + 1; j < len(keys); j++ { if keys[j] < keys[i] { keys[i], keys[j] = keys[j], keys[i] } } } return keys }