commandkit/promptkit_test.go

package commandkit_test

import (
	"bytes"
	"context"
	"errors"
	"io"
	"reflect"
	"strings"
	"testing"
	"testing/fstest"

	commandkit "gitea.maximumdirect.net/eric/CommandKit"
	configkit "gitea.maximumdirect.net/eric/ConfigKit"
	"gitea.maximumdirect.net/eric/promptkit"
	"go.yaml.in/yaml/v4"
)

const promptKitInspectionFixture = `prompts:
  "team/summary":
    "2":
      default_profile: child
      inputs:
        - name: topic
          required: true
          content_type: text/plain
          description: "Topic to summarize"
        - name: audience
          required: false
          content_type: text/plain
          description: ""
      messages:
        - role: user
          content: "Summarize {{.topic}}"
      output:
        format: json
        validation_mode: basic
        repair_attempts: 2
    "3":
      messages:
        - role: user
          content: "Version three"
      output:
        format: text
        validation_mode: none
  empty:
    "1":
      messages:
        - role: user
          content: "No inputs"
      output:
        format: text
        validation_mode: none
`

func promptKitInspectionEngine(t *testing.T) *promptkit.Engine {
	t.Helper()
	inputs, err := promptkit.FilesystemInputs("prompt fixture", fstest.MapFS{
		"config/prompts.yml": &fstest.MapFile{Data: []byte(promptKitInspectionFixture)},
	}, "config")
	if err != nil {
		t.Fatal(err)
	}
	engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
		Underlays: []configkit.Underlay{
			{Inputs: inputs},
			{Inputs: []configkit.Input{{Source: configkit.Mapping("profile fixture", map[string]any{
				"backends": map[string]any{"local": map[string]any{"endpoint": "https://example.invalid/v1"}},
				"profiles": map[string]any{
					"parent": map[string]any{
						"backend": "local", "model": "base-model", "temperature": 0.7,
						"max_tokens": 100, "top_p": 0.8,
						"extra_params": map[string]any{
							"token":    "provider-private-sentinel",
							"nested":   map[string]any{"null_value": nil, "visible": "kept"},
							"sequence": []any{"first", nil, 3},
						},
					},
					"child":        map[string]any{"extends": "parent", "temperature": 0, "max_tokens": 0, "api_key_env": "PROMPTKIT_TEST_KEY"},
					"requires-key": map[string]any{"backend": "local", "model": "key-model", "api_key_required": true},
					"no-extra":     map[string]any{"backend": "local", "model": "empty-params"},
				},
			})}}},
		},
	}})
	if err != nil {
		t.Fatalf("construct PromptKit fixture engine: %v", err)
	}
	return engine
}

func TestPromptKitInspectionCommandsReportRealCapturedDefinitions(t *testing.T) {
	t.Setenv("PROMPTKIT_TEST_KEY", "credential-private-sentinel")
	engine := promptKitInspectionEngine(t)
	loadCalls := 0
	suite, err := commandkit.NewPromptKitSuite(func(context.Context, commandkit.Selection, struct{}) (*promptkit.Engine, error) {
		loadCalls++
		return engine, nil
	})
	if err != nil {
		t.Fatal(err)
	}
	app, err := commandkit.New(commandkit.Identity{Name: "example"})
	if err != nil {
		t.Fatal(err)
	}
	if err := commandkit.RegisterPromptKit(app, suite); err != nil {
		t.Fatal(err)
	}

	var promptYAML bytes.Buffer
	outcome := app.Execute(context.Background(), []string{"promptkit", "show", "prompt", "team/summary", "--version", "2", "--format", "yaml"}, commandkit.Streams{Out: &promptYAML, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeSuccess || loadCalls != 1 {
		t.Fatalf("prompt inspection outcome=%#v calls=%d report=%q", outcome, loadCalls, promptYAML.String())
	}
	var promptReport map[string]any
	if err := yaml.Unmarshal(promptYAML.Bytes(), &promptReport); err != nil {
		t.Fatalf("decode prompt YAML: %v", err)
	}
	if !reflect.DeepEqual(sortedKeys(promptReport), []string{"default_profile_id", "inputs", "output_contract", "prompt_hash", "prompt_id", "prompt_version"}) {
		t.Fatalf("prompt report keys = %v", sortedKeys(promptReport))
	}
	if promptReport["prompt_id"] != "team/summary" || promptReport["prompt_version"] != "2" || promptReport["default_profile_id"] != "child" || promptReport["prompt_hash"] == "" {
		t.Fatalf("prompt report identity = %#v", promptReport)
	}
	inputs := promptReport["inputs"].([]any)
	if len(inputs) != 2 || inputs[0].(map[string]any)["name"] != "topic" || inputs[1].(map[string]any)["name"] != "audience" {
		t.Fatalf("prompt inputs/order = %#v", inputs)
	}
	outputContract := promptReport["output_contract"].(map[string]any)
	if outputContract["format"] != "json" || outputContract["validation_mode"] != "basic" || outputContract["repair_attempts"] != 2 {
		t.Fatalf("output contract = %#v", outputContract)
	}

	var promptText bytes.Buffer
	outcome = app.Execute(context.Background(), []string{"promptkit", "show", "prompt", "team/summary", "--version", "2"}, commandkit.Streams{Out: &promptText, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(promptText.String(), `Prompt hash: "`) || !strings.Contains(promptText.String(), `- name: "topic"`) || !strings.Contains(promptText.String(), `validation_mode: "basic"`) {
		t.Fatalf("prompt text = (%#v, %q)", outcome, promptText.String())
	}
	var omittedVersion bytes.Buffer
	outcome = app.Execute(context.Background(), []string{"promptkit", "show", "prompt", "empty", "--format", "yaml"}, commandkit.Streams{Out: &omittedVersion, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(omittedVersion.String(), "prompt_version: \"1\"") {
		t.Fatalf("omitted singleton prompt version = (%#v, %q)", outcome, omittedVersion.String())
	}

	var profileYAML bytes.Buffer
	outcome = app.Execute(context.Background(), []string{"promptkit", "show", "profile", "child", "--format", "yaml"}, commandkit.Streams{Out: &profileYAML, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeSuccess {
		t.Fatalf("profile inspection outcome=%#v report=%q", outcome, profileYAML.String())
	}
	var profileReport map[string]any
	if err := yaml.Unmarshal(profileYAML.Bytes(), &profileReport); err != nil {
		t.Fatalf("decode profile YAML: %v", err)
	}
	if !reflect.DeepEqual(sortedKeys(profileReport), []string{"api_key_required", "effective_model_params", "profile_id"}) || profileReport["profile_id"] != "child" || profileReport["api_key_required"] != false {
		t.Fatalf("profile report = %#v", profileReport)
	}
	params := profileReport["effective_model_params"].(map[string]any)
	if !reflect.DeepEqual(sortedKeys(params), []string{"api_key_env", "backend_id", "endpoint", "extra_params", "max_tokens", "model", "reasoning_effort", "service_tier", "temperature", "timeout_seconds", "top_p"}) {
		t.Fatalf("model parameter keys = %v", sortedKeys(params))
	}
	if params["temperature"] != 0 || params["max_tokens"] != 0 || params["api_key_env"] != "PROMPTKIT_TEST_KEY" {
		t.Fatalf("effective owning values = %#v", params)
	}
	extra := params["extra_params"].(map[string]any)
	nested := extra["nested"].(map[string]any)
	sequence := extra["sequence"].([]any)
	if nested["null_value"] != nil || nested["visible"] != "kept" || len(sequence) != 3 || sequence[1] != nil || sequence[2] != 3 {
		t.Fatalf("nested provider parameters changed: %#v", extra)
	}
	if extra["token"] != "provider-private-sentinel" || strings.Contains(profileYAML.String(), "credential-private-sentinel") {
		t.Fatalf("provider parameters or credential reference changed: %q", profileYAML.String())
	}

	var profileText bytes.Buffer
	outcome = app.Execute(context.Background(), []string{"promptkit", "show", "profile", "child"}, commandkit.Streams{Out: &profileText, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(profileText.String(), "temperature: 0\n") || !strings.Contains(profileText.String(), "api_key_env: \"PROMPTKIT_TEST_KEY\"") || !strings.Contains(profileText.String(), "null_value: null") {
		t.Fatalf("profile text = (%#v, %q)", outcome, profileText.String())
	}

	var requiredKey bytes.Buffer
	outcome = app.Execute(context.Background(), []string{"promptkit", "show", "profile", "requires-key", "--format", "yaml"}, commandkit.Streams{Out: &requiredKey, Err: io.Discard})
	var requiredReport map[string]any
	if outcome.Kind != commandkit.OutcomeSuccess || yaml.Unmarshal(requiredKey.Bytes(), &requiredReport) != nil || requiredReport["api_key_required"] != true {
		t.Fatalf("credential requirement report = (%#v, %q)", outcome, requiredKey.String())
	}

	var emptyPrompt bytes.Buffer
	outcome = app.Execute(context.Background(), []string{"promptkit", "show", "prompt", "empty", "--version", "1", "--format", "yaml"}, commandkit.Streams{Out: &emptyPrompt, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(emptyPrompt.String(), "inputs: []") {
		t.Fatalf("empty inputs report = (%#v, %q)", outcome, emptyPrompt.String())
	}
	var zeroExtra bytes.Buffer
	outcome = app.Execute(context.Background(), []string{"promptkit", "show", "profile", "no-extra", "--format", "yaml"}, commandkit.Streams{Out: &zeroExtra, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(zeroExtra.String(), "extra_params: {}") {
		t.Fatalf("nil extra params report = (%#v, %q)", outcome, zeroExtra.String())
	}
}

func TestPromptKitInspectionSelectionErrorsHelpAndOptionalCapabilities(t *testing.T) {
	engine := promptKitInspectionEngine(t)
	loaderCalls := 0
	suite, err := commandkit.NewPromptKitSuite(func(context.Context, commandkit.Selection, struct{}) (*promptkit.Engine, error) {
		loaderCalls++
		return engine, nil
	})
	if err != nil {
		t.Fatal(err)
	}
	app, _ := commandkit.New(commandkit.Identity{Name: "example"})
	if err := commandkit.RegisterPromptKit(app, suite); err != nil {
		t.Fatal(err)
	}
	badTarget := suite
	badTarget.Selectors = []commandkit.Selector[struct{}]{{
		Binding: commandkit.ValueFlag[struct{}](commandkit.FlagMeta{Name: "campaign"}, nil, func(value string) (string, error) { return value, nil }, func(*struct{}, commandkit.Supplied[string]) {}),
		Scopes:  []string{"app"},
	}}
	invalidApp, _ := commandkit.New(commandkit.Identity{Name: "invalid"})
	if err := commandkit.RegisterPromptKit(invalidApp, badTarget); err == nil {
		t.Fatal("selector for another scope was accepted")
	}
	for _, tc := range []struct {
		args []string
		want error
	}{
		{args: []string{"promptkit", "show", "prompt", "team/summary"}, want: promptkit.ErrPromptLoad},
		{args: []string{"promptkit", "show", "prompt", "missing", "--version", "1"}, want: promptkit.ErrPromptNotFound},
		{args: []string{"promptkit", "show", "profile", "missing"}, want: promptkit.ErrProfileNotFound},
		{args: []string{"promptkit", "show", "prompt", "", "--version", "2"}, want: promptkit.ErrInvalidRequest},
	} {
		outcome := app.Execute(context.Background(), tc.args, commandkit.Streams{Out: io.Discard, Err: io.Discard})
		if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, tc.want) {
			t.Errorf("execute %q = (%#v), want operational error matching %v", tc.args, outcome, tc.want)
		}
	}
	cancelled, cancel := context.WithCancel(context.Background())
	cancel()
	outcome := app.Execute(cancelled, []string{"promptkit", "show", "prompt", "empty", "--version", "1"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, context.Canceled) {
		t.Fatalf("cancelled prompt inspection = %#v", outcome)
	}
	beforeHelp := loaderCalls
	outcome = app.Execute(context.Background(), []string{"promptkit", "show", "prompt", "--help"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeSuccess || loaderCalls != beforeHelp {
		t.Fatalf("help acquired engine: outcome=%#v calls=%d", outcome, loaderCalls)
	}
	beforeMissing := loaderCalls
	outcome = app.Execute(context.Background(), []string{"promptkit", "show", "prompt"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeUsage || loaderCalls != beforeMissing {
		t.Fatalf("missing ID = (%#v, calls=%d), want usage before acquisition", outcome, loaderCalls)
	}
	beforeScopeFlag := loaderCalls
	outcome = app.Execute(context.Background(), []string{"promptkit", "show", "profile", "child", "--scope", "promptkit"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeUsage || loaderCalls != beforeScopeFlag {
		t.Fatalf("definition command accepted --scope: outcome=%#v calls=%d", outcome, loaderCalls)
	}

	profileOnlyApp, _ := commandkit.New(commandkit.Identity{Name: "profile-only"})
	if err := commandkit.RegisterPromptKit(profileOnlyApp, commandkit.PromptKitSuite[struct{}]{InspectProfile: func(ctx context.Context, _ commandkit.Selection, _ struct{}, id string) (commandkit.Acquired[*promptkit.ProfileInspection], error) {
		value, err := engine.InspectProfile(ctx, id)
		return commandkit.Acquired[*promptkit.ProfileInspection]{Value: value}, err
	}}); err != nil {
		t.Fatal(err)
	}
	outcome = profileOnlyApp.Execute(context.Background(), []string{"promptkit", "show", "profile", "child"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeSuccess {
		t.Fatalf("profile-only capability was not installed: %#v", outcome)
	}
	outcome = profileOnlyApp.Execute(context.Background(), []string{"promptkit", "show", "prompt", "empty"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeUsage {
		t.Fatalf("profile-only suite installed prompt inspection: %#v", outcome)
	}
	promptOnlyApp, _ := commandkit.New(commandkit.Identity{Name: "prompt-only"})
	if err := commandkit.RegisterPromptKit(promptOnlyApp, commandkit.PromptKitSuite[struct{}]{InspectPrompt: func(ctx context.Context, _ commandkit.Selection, _ struct{}, id, version string) (commandkit.Acquired[*promptkit.PromptInspection], error) {
		value, err := engine.InspectPrompt(ctx, id, version)
		return commandkit.Acquired[*promptkit.PromptInspection]{Value: value}, err
	}}); err != nil {
		t.Fatal(err)
	}
	outcome = promptOnlyApp.Execute(context.Background(), []string{"promptkit", "show", "prompt", "empty"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeSuccess {
		t.Fatalf("prompt-only capability was not installed: %#v", outcome)
	}
	outcome = promptOnlyApp.Execute(context.Background(), []string{"promptkit", "show", "profile", "child"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeUsage {
		t.Fatalf("prompt-only suite installed profile inspection: %#v", outcome)
	}

	emptyApp, _ := commandkit.New(commandkit.Identity{Name: "empty"})
	if err := commandkit.RegisterPromptKit(emptyApp, commandkit.PromptKitSuite[struct{}]{}); err != nil {
		t.Fatalf("empty PromptKit suite = %v", err)
	}
	outcome = emptyApp.Execute(context.Background(), []string{"help"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeSuccess {
		t.Fatalf("empty PromptKit suite changed command family: %#v", outcome)
	}
	outcome = emptyApp.Execute(context.Background(), []string{"help", "promptkit"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeUsage {
		t.Fatalf("empty PromptKit suite installed its namespace: %#v", outcome)
	}
}

func TestPromptKitSuitePassesSelectionAndProjectedContext(t *testing.T) {
	type appContext struct {
		Campaign commandkit.Supplied[string]
		Runtime  string
	}
	type contextKey struct{}
	engine := promptKitInspectionEngine(t)
	ctx := context.WithValue(context.Background(), contextKey{}, "caller")
	wantSelection := commandkit.Selection{
		Config:  commandkit.Supplied[string]{Value: "main.yml", Set: true},
		Overlay: commandkit.Supplied[string]{Value: "dev", Set: true},
	}
	loaderCalls, contexts, checks := 0, 0, 0
	var gotSelection commandkit.Selection
	var gotContext appContext
	suite, err := commandkit.NewPromptKitSuite(func(gotCtx context.Context, selection commandkit.Selection, input appContext) (*promptkit.Engine, error) {
		loaderCalls++
		gotSelection, gotContext = selection, input
		if gotCtx != ctx || gotCtx.Value(contextKey{}) != "caller" {
			t.Fatal("engine loader did not receive invocation context")
		}
		return engine, nil
	})
	if err != nil {
		t.Fatal(err)
	}
	suite.NewContext = func() appContext { contexts++; return appContext{Runtime: "invocation cache"} }
	suite.Selectors = []commandkit.Selector[appContext]{{
		Binding: commandkit.ValueFlag[appContext](commandkit.FlagMeta{Name: "campaign"}, nil, func(value string) (string, error) { return value, nil }, func(input *appContext, value commandkit.Supplied[string]) { input.Campaign = value }),
		Scopes:  []string{"promptkit"},
	}}
	suite.CheckInput = func(selection commandkit.Selection, input appContext) error {
		checks++
		if selection != wantSelection || input.Campaign != (commandkit.Supplied[string]{Value: "spring", Set: true}) {
			t.Errorf("preflight inputs = (%+v, %+v)", selection, input)
		}
		return nil
	}
	app, _ := commandkit.New(commandkit.Identity{Name: "example"})
	if err := commandkit.RegisterPromptKit(app, suite); err != nil {
		t.Fatal(err)
	}
	outcome := app.Execute(ctx, []string{"promptkit", "show", "profile", "child", "--config", "main.yml", "--overlay", "dev", "--campaign", "spring"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeSuccess || loaderCalls != 1 || contexts != 1 || checks != 1 {
		t.Fatalf("inspection = (%#v), loader/context/check calls = %d/%d/%d", outcome, loaderCalls, contexts, checks)
	}
	if gotSelection != wantSelection || gotContext.Campaign != (commandkit.Supplied[string]{Value: "spring", Set: true}) || gotContext.Runtime != "invocation cache" {
		t.Fatalf("loader selection/context = (%+v, %+v)", gotSelection, gotContext)
	}
}

func TestPromptKitDirectSuiteRetainsDiagnosticsOnNamedLookupFailure(t *testing.T) {
	warning := commandkit.Diagnostic{Severity: configkit.SeverityWarning, Write: func(w io.Writer) error { _, err := io.WriteString(w, "configuration warning\n"); return err }}
	app, _ := commandkit.New(commandkit.Identity{Name: "example"})
	err := commandkit.RegisterPromptKit(app, commandkit.PromptKitSuite[struct{}]{InspectPrompt: func(context.Context, commandkit.Selection, struct{}, string, string) (commandkit.Acquired[*promptkit.PromptInspection], error) {
		return commandkit.Acquired[*promptkit.PromptInspection]{Diagnostics: []commandkit.Diagnostic{warning}}, promptkit.ErrPromptNotFound
	}})
	if err != nil {
		t.Fatal(err)
	}
	outcome := app.Execute(context.Background(), []string{"promptkit", "show", "prompt", "missing"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, promptkit.ErrPromptNotFound) || len(outcome.Diagnostics) < 1 || outcome.Diagnostics[0].Severity != configkit.SeverityWarning {
		t.Fatalf("named lookup outcome did not retain acquisition diagnostic: %#v", outcome)
	}
}

func TestPromptKitInspectionReportsOutputDeliveryFailure(t *testing.T) {
	engine := promptKitInspectionEngine(t)
	suite, err := commandkit.NewPromptKitSuite(func(context.Context, commandkit.Selection, struct{}) (*promptkit.Engine, error) {
		return engine, nil
	})
	if err != nil {
		t.Fatal(err)
	}
	app, _ := commandkit.New(commandkit.Identity{Name: "example"})
	if err := commandkit.RegisterPromptKit(app, suite); err != nil {
		t.Fatal(err)
	}
	writeErr := errors.New("report output failed")
	outcome := app.Execute(context.Background(), []string{"promptkit", "show", "profile", "child", "--format", "yaml"}, commandkit.Streams{Out: promptKitFailingWriter{err: writeErr}, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeOperational || !outcome.ReportIncomplete || !errors.Is(outcome.DeliveryError, writeErr) {
		t.Fatalf("promptkit output failure = %#v", outcome)
	}
}

type promptKitFailingWriter struct{ err error }

func (writer promptKitFailingWriter) Write([]byte) (int, error) { return 0, writer.err }

func sortedKeys(values map[string]any) []string {
	keys := make([]string, 0, len(values))
	for key := range values {
		keys = append(keys, key)
	}
	for i := 0; i < len(keys); i++ {
		for j := i + 1; j < len(keys); j++ {
			if keys[j] < keys[i] {
				keys[i], keys[j] = keys[j], keys[i]
			}
		}
	}
	return keys
}