package ckprompt import ( "bytes" "context" "errors" "io" "strings" "testing" commandkit "gitea.maximumdirect.net/eric/CommandKit" "gitea.maximumdirect.net/eric/CommandKit/integrations/ckconfig" promptkit "gitea.maximumdirect.net/eric/promptkit" "go.yaml.in/yaml/v4" ) func newPromptTestApp(t *testing.T, commands ...commandkit.Command) *commandkit.App { t.Helper() app, err := commandkit.New(commandkit.Identity{Name: "example"}, commands...) if err != nil { t.Fatal(err) } return app } func TestCommandBuildsIndependentOptionalTrees(t *testing.T) { promptOnly := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{InspectPrompt: func(context.Context, PromptRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.PromptInspection], error) { return commandkit.Acquired[*promptkit.PromptInspection]{}, errors.New("unused") }})) var output bytes.Buffer outcome := promptOnly.Execute(context.Background(), []string{"help", "promptkit", "show"}, commandkit.Streams{Out: &output, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(output.String(), "\n prompt ") || strings.Contains(output.String(), "\n profile ") || strings.Contains(output.String(), "config") { t.Fatalf("prompt-only help = (%#v, %q)", outcome, output.String()) } profileOnly := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{InspectProfile: func(context.Context, ProfileRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.ProfileInspection], error) { return commandkit.Acquired[*promptkit.ProfileInspection]{}, errors.New("unused") }})) output.Reset() outcome = profileOnly.Execute(context.Background(), []string{"help", "promptkit", "show"}, commandkit.Streams{Out: &output, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(output.String(), "\n profile ") || strings.Contains(output.String(), "\n prompt ") { t.Fatalf("profile-only help = (%#v, %q)", outcome, output.String()) } both := Command(Suite[struct{}, struct{}]{ InspectPrompt: func(context.Context, PromptRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.PromptInspection], error) { return commandkit.Acquired[*promptkit.PromptInspection]{}, nil }, InspectProfile: func(context.Context, ProfileRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.ProfileInspection], error) { return commandkit.Acquired[*promptkit.ProfileInspection]{}, nil }, }) if err := newPromptTestApp(t).Register(both); err != nil { t.Fatalf("both-capability tree failed to register: %v", err) } app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{Selectors: []commandkit.Flag[struct{}]{{}}})) output.Reset() outcome = app.Execute(context.Background(), []string{"help"}, commandkit.Streams{Out: &output, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess || strings.Contains(output.String(), "promptkit") { t.Fatalf("empty suite installed a namespace: (%#v, %q)", outcome, output.String()) } invalid := FromEngine[int, int](nil) invalid.InspectPrompt, invalid.InspectProfile = nil, nil if err := newPromptTestApp(t).Register(Command(invalid)); err == nil || !strings.Contains(err.Error(), "PromptKit engine loader is nil") { t.Fatalf("empty invalid adapter lost its construction issue: %v", err) } } func TestCommandUsesInstalledNamespaceCollisionChecks(t *testing.T) { command := Command(Suite[struct{}, struct{}]{InspectPrompt: func(context.Context, PromptRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.PromptInspection], error) { return commandkit.Acquired[*promptkit.PromptInspection]{}, nil }}) other := commandkit.Group("promptkit", "Another command tree.", commandkit.NewCommand(commandkit.CommandSpec[struct{}]{Name: "child", Summary: "A child."})) first, err := commandkit.New(commandkit.Identity{Name: "example"}, other) if err != nil { t.Fatal(err) } if err := first.Register(command); err == nil { t.Fatal("PromptKit tree installed over an existing namespace") } second, err := commandkit.New(commandkit.Identity{Name: "example"}, command) if err != nil { t.Fatal(err) } if err := second.Register(other); err == nil { t.Fatal("later command tree installed over the PromptKit namespace") } } func TestCommandValidatesOnlyGeneratedCapturedBindings(t *testing.T) { unused := Command(Suite[struct{}, struct{}]{Selectors: []commandkit.Flag[struct{}]{{}}}) if err := newPromptTestApp(t).Register(unused); err != nil { t.Fatalf("unused malformed selector affected empty suite: %v", err) } generated := Command(Suite[struct{}, struct{}]{Selectors: []commandkit.Flag[struct{}]{{}}, InspectProfile: func(context.Context, ProfileRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.ProfileInspection], error) { return commandkit.Acquired[*promptkit.ProfileInspection]{}, nil }}) if err := newPromptTestApp(t).Register(generated); err == nil { t.Fatal("generated command accepted an invalid captured selector") } } func TestDefinitionInputAndStateLifetimesFollowChecks(t *testing.T) { type inspectInput struct{ Campaign commandkit.Supplied[string] } type runState struct{} conversions := 0 flag := commandkit.ValueFlag[*inspectInput, string](commandkit.FlagMeta{Name: "campaign", Required: true}, (*string)(nil), func(value string) (string, error) { conversions++ return value, nil }, func(input **inspectInput, value commandkit.Supplied[string]) { (*input).Campaign = value }) newInputs, newStates, checks, callbacks := 0, 0, 0, 0 rejectInput := false var checked *inspectInput var wantContext context.Context suite := Suite[*inspectInput, *runState]{ NewInput: func() *inspectInput { newInputs++; return &inspectInput{} }, NewState: func() *runState { newStates++ if checks != 1 { t.Errorf("state initialized after %d input checks", checks) } return &runState{} }, Selectors: []commandkit.Flag[*inspectInput]{flag}, CheckInput: func(selection ckconfig.Selection, input *inspectInput) error { checks++ checked = input if rejectInput { return errors.New("rejected definition input") } if !selection.Config.Set || selection.Config.Value != "base.yml" || !selection.Overlay.Set || selection.Overlay.Value != "local.yml" || !input.Campaign.Set || input.Campaign.Value != "spring" { return errors.New("definition input was incomplete") } return nil }, InspectPrompt: func(ctx context.Context, request PromptRequest, input *inspectInput, state *runState) (commandkit.Acquired[*promptkit.PromptInspection], error) { callbacks++ if ctx != wantContext || input != checked || request.ID != "briefing" || !request.Version.Set || request.Version.Value != "2" || state == nil { t.Errorf("prompt callback arguments = ctx=%v request=%#v input=%p state=%p", ctx, request, input, state) } return commandkit.Acquired[*promptkit.PromptInspection]{Value: &promptkit.PromptInspection{PromptID: request.ID, PromptVersion: request.Version.Value}}, nil }, } app := newPromptTestApp(t, Command(suite)) wantContext = context.WithValue(context.Background(), struct{}{}, "context") var output bytes.Buffer outcome := app.Execute(wantContext, []string{"promptkit", "show", "prompt", "briefing", "--campaign", "spring", "--config", "base.yml", "--overlay", "local.yml", "--version", "2"}, commandkit.Streams{Out: &output, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess || newInputs != 1 || newStates != 1 || checks != 1 || callbacks != 1 || conversions != 1 { t.Fatalf("execution outcome=%#v counts conversion/input/state/check/callback=%d/%d/%d/%d/%d report=%q", outcome, conversions, newInputs, newStates, checks, callbacks, output.String()) } rejectInput = true output.Reset() outcome = app.Execute(wantContext, []string{"promptkit", "show", "prompt", "briefing", "--campaign", "spring", "--config", "base.yml", "--overlay", "local.yml", "--version", "2"}, commandkit.Streams{Out: &output, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeUsage || newInputs != 2 || newStates != 1 || checks != 2 || callbacks != 1 || conversions != 2 { t.Fatalf("rejected input reached state/callback: outcome=%#v counts=%d/%d/%d/%d/%d", outcome, conversions, newInputs, newStates, checks, callbacks) } } func TestDefinitionReportsRetainNativeFieldsAndSafeCredentialReferences(t *testing.T) { t.Setenv("PROMPTKIT_ADAPTER_TEST_KEY", "credential-value-must-not-appear") engine := adapterEngine(t) loadCalls := 0 suite := FromEngine(func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) { loadCalls++ return engine, nil }) app := newPromptTestApp(t, Command(suite)) var output bytes.Buffer outcome := app.Execute(context.Background(), []string{"promptkit", "show", "prompt", "briefing", "--version", "1", "--format", "yaml"}, commandkit.Streams{Out: &output, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess || loadCalls != 1 { t.Fatalf("prompt report outcome=%#v load calls=%d report=%s", outcome, loadCalls, output.String()) } var prompt map[string]any if err := yaml.Unmarshal(output.Bytes(), &prompt); err != nil { t.Fatal(err) } inputs := prompt["inputs"].([]any) if len(inputs) != 2 || inputs[0].(map[string]any)["name"] != "topic" || inputs[1].(map[string]any)["name"] != "audience" || prompt["prompt_hash"] == "" { t.Fatalf("prompt fields/order = %#v", prompt) } contract := prompt["output_contract"].(map[string]any) if contract["repair_attempts"] != 0 { t.Fatalf("zero-valued contract field was lost: %#v", contract) } output.Reset() outcome = app.Execute(context.Background(), []string{"promptkit", "show", "prompt", "briefing"}, commandkit.Streams{Out: &output, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess || loadCalls != 2 || !strings.Contains(output.String(), "Prompt version: \"1\"") || !strings.Contains(output.String(), `- name: "topic"`) { t.Fatalf("default text prompt report = (%#v, calls=%d, %s)", outcome, loadCalls, output.String()) } output.Reset() outcome = app.Execute(context.Background(), []string{"promptkit", "show", "profile", "default", "--format", "yaml"}, commandkit.Streams{Out: &output, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess || loadCalls != 3 { t.Fatalf("profile report outcome=%#v calls=%d report=%s", outcome, loadCalls, output.String()) } var profile map[string]any if err := yaml.Unmarshal(output.Bytes(), &profile); err != nil { t.Fatal(err) } params := profile["effective_model_params"].(map[string]any) if params["api_key_env"] != "PROMPTKIT_ADAPTER_TEST_KEY" || params["temperature"] != 0 || params["max_tokens"] != 0 || strings.Contains(output.String(), "credential-value-must-not-appear") { t.Fatalf("credential or zero model fields = %#v / %s", params, output.String()) } extra := params["extra_params"].(map[string]any) nested := extra["nested"].(map[string]any) sequence := extra["sequence"].([]any) if extra["token"] != "caller-owned-value" || nested["null_value"] != nil || nested["visible"] != true || len(sequence) != 3 || sequence[1] != nil || sequence[2] != 3 { t.Fatalf("nested ExtraParams changed: %#v", extra) } output.Reset() outcome = app.Execute(context.Background(), []string{"promptkit", "show", "profile", "default"}, commandkit.Streams{Out: &output, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess || loadCalls != 4 || !strings.Contains(output.String(), "API key required: false") || !strings.Contains(output.String(), `api_key_env: "PROMPTKIT_ADAPTER_TEST_KEY"`) { t.Fatalf("default text profile report = (%#v, calls=%d, %s)", outcome, loadCalls, output.String()) } output.Reset() outcome = app.Execute(context.Background(), []string{"promptkit", "show", "profile", "requires-key", "--format", "yaml"}, commandkit.Streams{Out: &output, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(output.String(), "api_key_required: true") { t.Fatalf("credential requirement report = (%#v, %s)", outcome, output.String()) } output.Reset() outcome = app.Execute(context.Background(), []string{"promptkit", "show", "prompt", "empty", "--format", "yaml"}, commandkit.Streams{Out: &output, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(output.String(), "inputs: []") { t.Fatalf("empty prompt input list = (%#v, %s)", outcome, output.String()) } } func TestNativeLookupErrorsAndBlankIDsReachCallbacks(t *testing.T) { nativeErr := errors.New("native lookup failure") calls := 0 diagnostic := commandkit.MessageDiagnostic(commandkit.SeverityWarning, "lookup context") app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{InspectPrompt: func(_ context.Context, request PromptRequest, _ struct{}, _ struct{}) (commandkit.Acquired[*promptkit.PromptInspection], error) { calls++ if request.ID != "" { t.Errorf("blank ID was rewritten to %q", request.ID) } return commandkit.Acquired[*promptkit.PromptInspection]{Diagnostics: []commandkit.Diagnostic{diagnostic}}, nativeErr }})) var output, diagnosticOutput bytes.Buffer outcome := app.Execute(context.Background(), []string{"promptkit", "show", "prompt"}, commandkit.Streams{Out: &output, Err: &diagnosticOutput}) if outcome.Kind != commandkit.OutcomeUsage || calls != 0 || output.Len() != 0 { t.Fatalf("missing ID was not a usage failure: (%#v, calls=%d, out=%q)", outcome, calls, output.String()) } output.Reset() diagnosticOutput.Reset() outcome = app.Execute(context.Background(), []string{"promptkit", "show", "prompt", ""}, commandkit.Streams{Out: &output, Err: &diagnosticOutput}) if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, nativeErr) || calls != 1 || output.Len() != 0 || !strings.Contains(diagnosticOutput.String(), "lookup context") { t.Fatalf("native lookup failure = (%#v, calls=%d, out=%q, err=%q)", outcome, calls, output.String(), diagnosticOutput.String()) } } func TestCancelledInspectionDoesNotRender(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) called := 0 app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{InspectProfile: func(context.Context, ProfileRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.ProfileInspection], error) { called++ cancel() return commandkit.Acquired[*promptkit.ProfileInspection]{Value: &promptkit.ProfileInspection{ProfileID: "default"}}, nil }})) var output bytes.Buffer outcome := app.Execute(ctx, []string{"promptkit", "show", "profile", "default"}, commandkit.Streams{Out: &output, Err: io.Discard}) if !errors.Is(outcome.Cause, context.Canceled) || outcome.Kind != commandkit.OutcomeOperational || called != 1 || output.Len() != 0 { t.Fatalf("cancelled inspection = (%#v, called=%d, output=%q)", outcome, called, output.String()) } } func TestDefinitionFormatPreflightAndReportDeliveryFailure(t *testing.T) { calls := 0 app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{InspectProfile: func(context.Context, ProfileRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.ProfileInspection], error) { calls++ return commandkit.Acquired[*promptkit.ProfileInspection]{Value: &promptkit.ProfileInspection{ProfileID: "default"}}, nil }})) var output bytes.Buffer outcome := app.Execute(context.Background(), []string{"promptkit", "show", "profile", "default", "--format", "json"}, commandkit.Streams{Out: &output, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeUsage || calls != 0 || output.Len() != 0 { t.Fatalf("bad format was not rejected before callback: outcome=%#v calls=%d output=%q", outcome, calls, output.String()) } writeErr := errors.New("report writer failed") outcome = app.Execute(context.Background(), []string{"promptkit", "show", "profile", "default"}, commandkit.Streams{Out: partialReportWriter{err: writeErr}, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, writeErr) || outcome.DeliveryError != writeErr || !outcome.ReportIncomplete || calls != 1 { t.Fatalf("report delivery outcome = %#v calls=%d", outcome, calls) } } type partialReportWriter struct{ err error } func (writer partialReportWriter) Write(content []byte) (int, error) { if len(content) == 0 { return 0, writer.err } return 1, writer.err } var _ io.Writer = partialReportWriter{} func TestProfileTextUsesCredentialNamesAndExplicitEmptyExtraParams(t *testing.T) { report := profileReportFromInspection(&promptkit.ProfileInspection{ProfileID: "default", EffectiveModelParams: promptkit.ExecutionTarget{APIKeyEnv: "APP_KEY", ExtraParams: nil}}) text, err := formatProfileText(report) if err != nil || !strings.Contains(text, "api_key_env: \"APP_KEY\"") || !strings.Contains(text, "extra_params:\n {}\n") || !strings.Contains(text, "API key required: false") { t.Fatalf("profile text = (%q, %v)", text, err) } if strings.Contains(text, "APP_KEY_VALUE") { t.Fatalf("report contains a credential value: %q", text) } } func TestPromptCatalogYAMLPreservesSchemaStringsAndOrder(t *testing.T) { want := []promptkit.PromptIdentity{ {ID: " briefing ", Version: " 03 "}, {ID: "001", Version: "1.10"}, {ID: " briefing ", Version: " 03 "}, } output, err := encodePromptCatalog(want) if err != nil { t.Fatal(err) } var decoded []map[string]any if err := yaml.Unmarshal(output, &decoded); err != nil { t.Fatalf("decode prompt catalog %q: %v", output, err) } if len(decoded) != len(want) { t.Fatalf("catalog entry count = %d, want %d: %q", len(decoded), len(want), output) } for index, entry := range decoded { if len(entry) != 2 { t.Errorf("entry %d fields = %#v, want only id and version", index, entry) } id, idOK := entry["id"].(string) version, versionOK := entry["version"].(string) if !idOK || !versionOK || id != want[index].ID || version != want[index].Version { t.Errorf("entry %d = %#v, want id/version %q/%q as strings", index, entry, want[index].ID, want[index].Version) } } } func TestCatalogYAMLEncodersRenderNilAndEmptyInventoriesAsSequences(t *testing.T) { for _, test := range []struct { name string data func() ([]byte, error) }{ {name: "nil prompts", data: func() ([]byte, error) { return encodePromptCatalog(nil) }}, {name: "empty prompts", data: func() ([]byte, error) { return encodePromptCatalog(make([]promptkit.PromptIdentity, 0)) }}, {name: "nil profiles", data: func() ([]byte, error) { return encodeProfileCatalog(nil) }}, {name: "empty profiles", data: func() ([]byte, error) { return encodeProfileCatalog(make([]string, 0)) }}, } { t.Run(test.name, func(t *testing.T) { output, err := test.data() if err != nil { t.Fatal(err) } if strings.TrimSpace(string(output)) != "[]" { t.Fatalf("empty catalog YAML = %q, want empty sequence", output) } }) } } func TestProfileCatalogYAMLPreservesStringValuesAndOrder(t *testing.T) { want := []string{" default ", "001", "same", "same"} output, err := encodeProfileCatalog(want) if err != nil { t.Fatal(err) } var decoded []any if err := yaml.Unmarshal(output, &decoded); err != nil { t.Fatalf("decode profile catalog %q: %v", output, err) } if len(decoded) != len(want) { t.Fatalf("catalog entry count = %d, want %d: %q", len(decoded), len(want), output) } for index, item := range decoded { value, ok := item.(string) if !ok || value != want[index] { t.Errorf("profile %d = %#v, want string %q", index, item, want[index]) } } }