Add PromptKit definition inspection

2026-10-07 14:40:41 +00:00
parent 6ad723cde1
commit 0af44a1172
7 changed files with 924 additions and 23 deletions
--- a/promptkit_test.go
+++ b/promptkit_test.go
@@ -0,0 +1,378 @@
+package commandkit_test
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"io"
+	"reflect"
+	"strings"
+	"testing"
+	"testing/fstest"
+
+	commandkit "gitea.maximumdirect.net/eric/CommandKit"
+	configkit "gitea.maximumdirect.net/eric/ConfigKit"
+	"gitea.maximumdirect.net/eric/promptkit"
+	"go.yaml.in/yaml/v4"
+)
+
+const promptKitInspectionFixture = `prompts:
+  "team/summary":
+    "2":
+      default_profile: child
+      inputs:
+        - name: topic
+          required: true
+          content_type: text/plain
+          description: "Topic to summarize"
+        - name: audience
+          required: false
+          content_type: text/plain
+          description: ""
+      messages:
+        - role: user
+          content: "Summarize {{.topic}}"
+      output:
+        format: json
+        validation_mode: basic
+        repair_attempts: 2
+    "3":
+      messages:
+        - role: user
+          content: "Version three"
+      output:
+        format: text
+        validation_mode: none
+  empty:
+    "1":
+      messages:
+        - role: user
+          content: "No inputs"
+      output:
+        format: text
+        validation_mode: none
+`
+
+func promptKitInspectionEngine(t *testing.T) *promptkit.Engine {
+	t.Helper()
+	inputs, err := promptkit.FilesystemInputs("prompt fixture", fstest.MapFS{
+		"config/prompts.yml": &fstest.MapFile{Data: []byte(promptKitInspectionFixture)},
+	}, "config")
+	if err != nil {
+		t.Fatal(err)
+	}
+	engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
+		Underlays: []configkit.Underlay{
+			{Inputs: inputs},
+			{Inputs: []configkit.Input{{Source: configkit.Mapping("profile fixture", map[string]any{
+				"backends": map[string]any{"local": map[string]any{"endpoint": "https://example.invalid/v1"}},
+				"profiles": map[string]any{
+					"parent": map[string]any{
+						"backend": "local", "model": "base-model", "temperature": 0.7,
+						"max_tokens": 100, "top_p": 0.8,
+						"extra_params": map[string]any{
+							"token":    "provider-private-sentinel",
+							"nested":   map[string]any{"null_value": nil, "visible": "kept"},
+							"sequence": []any{"first", nil, 3},
+						},
+					},
+					"child":        map[string]any{"extends": "parent", "temperature": 0, "max_tokens": 0, "api_key_env": "PROMPTKIT_TEST_KEY"},
+					"requires-key": map[string]any{"backend": "local", "model": "key-model", "api_key_required": true},
+					"no-extra":     map[string]any{"backend": "local", "model": "empty-params"},
+				},
+			})}}},
+		},
+	}})
+	if err != nil {
+		t.Fatalf("construct PromptKit fixture engine: %v", err)
+	}
+	return engine
+}
+
+func TestPromptKitInspectionCommandsReportRealCapturedDefinitions(t *testing.T) {
+	t.Setenv("PROMPTKIT_TEST_KEY", "credential-private-sentinel")
+	engine := promptKitInspectionEngine(t)
+	loadCalls := 0
+	suite, err := commandkit.NewPromptKitSuite(func(context.Context, commandkit.Selection, struct{}) (*promptkit.Engine, error) {
+		loadCalls++
+		return engine, nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	app, err := commandkit.New(commandkit.Identity{Name: "example"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := commandkit.RegisterPromptKit(app, suite); err != nil {
+		t.Fatal(err)
+	}
+
+	var promptYAML bytes.Buffer
+	outcome := app.Execute(context.Background(), []string{"promptkit", "show", "prompt", "team/summary", "--version", "2", "--format", "yaml"}, commandkit.Streams{Out: &promptYAML, Err: io.Discard})
+	if outcome.Kind != commandkit.OutcomeSuccess || loadCalls != 1 {
+		t.Fatalf("prompt inspection outcome=%#v calls=%d report=%q", outcome, loadCalls, promptYAML.String())
+	}
+	var promptReport map[string]any
+	if err := yaml.Unmarshal(promptYAML.Bytes(), &promptReport); err != nil {
+		t.Fatalf("decode prompt YAML: %v", err)
+	}
+	if !reflect.DeepEqual(sortedKeys(promptReport), []string{"default_profile_id", "inputs", "output_contract", "prompt_hash", "prompt_id", "prompt_version"}) {
+		t.Fatalf("prompt report keys = %v", sortedKeys(promptReport))
+	}
+	if promptReport["prompt_id"] != "team/summary" || promptReport["prompt_version"] != "2" || promptReport["default_profile_id"] != "child" || promptReport["prompt_hash"] == "" {
+		t.Fatalf("prompt report identity = %#v", promptReport)
+	}
+	inputs := promptReport["inputs"].([]any)
+	if len(inputs) != 2 || inputs[0].(map[string]any)["name"] != "topic" || inputs[1].(map[string]any)["name"] != "audience" {
+		t.Fatalf("prompt inputs/order = %#v", inputs)
+	}
+	outputContract := promptReport["output_contract"].(map[string]any)
+	if outputContract["format"] != "json" || outputContract["validation_mode"] != "basic" || outputContract["repair_attempts"] != 2 {
+		t.Fatalf("output contract = %#v", outputContract)
+	}
+
+	var promptText bytes.Buffer
+	outcome = app.Execute(context.Background(), []string{"promptkit", "show", "prompt", "team/summary", "--version", "2"}, commandkit.Streams{Out: &promptText, Err: io.Discard})
+	if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(promptText.String(), `Prompt hash: "`) || !strings.Contains(promptText.String(), `- name: "topic"`) || !strings.Contains(promptText.String(), `validation_mode: "basic"`) {
+		t.Fatalf("prompt text = (%#v, %q)", outcome, promptText.String())
+	}
+	var omittedVersion bytes.Buffer
+	outcome = app.Execute(context.Background(), []string{"promptkit", "show", "prompt", "empty", "--format", "yaml"}, commandkit.Streams{Out: &omittedVersion, Err: io.Discard})
+	if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(omittedVersion.String(), "prompt_version: \"1\"") {
+		t.Fatalf("omitted singleton prompt version = (%#v, %q)", outcome, omittedVersion.String())
+	}
+
+	var profileYAML bytes.Buffer
+	outcome = app.Execute(context.Background(), []string{"promptkit", "show", "profile", "child", "--format", "yaml"}, commandkit.Streams{Out: &profileYAML, Err: io.Discard})
+	if outcome.Kind != commandkit.OutcomeSuccess {
+		t.Fatalf("profile inspection outcome=%#v report=%q", outcome, profileYAML.String())
+	}
+	var profileReport map[string]any
+	if err := yaml.Unmarshal(profileYAML.Bytes(), &profileReport); err != nil {
+		t.Fatalf("decode profile YAML: %v", err)
+	}
+	if !reflect.DeepEqual(sortedKeys(profileReport), []string{"api_key_required", "effective_model_params", "profile_id"}) || profileReport["profile_id"] != "child" || profileReport["api_key_required"] != false {
+		t.Fatalf("profile report = %#v", profileReport)
+	}
+	params := profileReport["effective_model_params"].(map[string]any)
+	if !reflect.DeepEqual(sortedKeys(params), []string{"api_key_env", "backend_id", "endpoint", "extra_params", "max_tokens", "model", "reasoning_effort", "service_tier", "temperature", "timeout_seconds", "top_p"}) {
+		t.Fatalf("model parameter keys = %v", sortedKeys(params))
+	}
+	if params["temperature"] != 0 || params["max_tokens"] != 0 || params["api_key_env"] != "PROMPTKIT_TEST_KEY" {
+		t.Fatalf("effective owning values = %#v", params)
+	}
+	extra := params["extra_params"].(map[string]any)
+	nested := extra["nested"].(map[string]any)
+	sequence := extra["sequence"].([]any)
+	if nested["null_value"] != nil || nested["visible"] != "kept" || len(sequence) != 3 || sequence[1] != nil || sequence[2] != 3 {
+		t.Fatalf("nested provider parameters changed: %#v", extra)
+	}
+	if extra["token"] != "provider-private-sentinel" || strings.Contains(profileYAML.String(), "credential-private-sentinel") {
+		t.Fatalf("provider parameters or credential reference changed: %q", profileYAML.String())
+	}
+
+	var profileText bytes.Buffer
+	outcome = app.Execute(context.Background(), []string{"promptkit", "show", "profile", "child"}, commandkit.Streams{Out: &profileText, Err: io.Discard})
+	if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(profileText.String(), "temperature: 0\n") || !strings.Contains(profileText.String(), "api_key_env: \"PROMPTKIT_TEST_KEY\"") || !strings.Contains(profileText.String(), "null_value: null") {
+		t.Fatalf("profile text = (%#v, %q)", outcome, profileText.String())
+	}
+
+	var requiredKey bytes.Buffer
+	outcome = app.Execute(context.Background(), []string{"promptkit", "show", "profile", "requires-key", "--format", "yaml"}, commandkit.Streams{Out: &requiredKey, Err: io.Discard})
+	var requiredReport map[string]any
+	if outcome.Kind != commandkit.OutcomeSuccess || yaml.Unmarshal(requiredKey.Bytes(), &requiredReport) != nil || requiredReport["api_key_required"] != true {
+		t.Fatalf("credential requirement report = (%#v, %q)", outcome, requiredKey.String())
+	}
+
+	var emptyPrompt bytes.Buffer
+	outcome = app.Execute(context.Background(), []string{"promptkit", "show", "prompt", "empty", "--version", "1", "--format", "yaml"}, commandkit.Streams{Out: &emptyPrompt, Err: io.Discard})
+	if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(emptyPrompt.String(), "inputs: []") {
+		t.Fatalf("empty inputs report = (%#v, %q)", outcome, emptyPrompt.String())
+	}
+	var zeroExtra bytes.Buffer
+	outcome = app.Execute(context.Background(), []string{"promptkit", "show", "profile", "no-extra", "--format", "yaml"}, commandkit.Streams{Out: &zeroExtra, Err: io.Discard})
+	if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(zeroExtra.String(), "extra_params: {}") {
+		t.Fatalf("nil extra params report = (%#v, %q)", outcome, zeroExtra.String())
+	}
+}
+
+func TestPromptKitInspectionSelectionErrorsHelpAndOptionalCapabilities(t *testing.T) {
+	engine := promptKitInspectionEngine(t)
+	loaderCalls := 0
+	suite, err := commandkit.NewPromptKitSuite(func(context.Context, commandkit.Selection, struct{}) (*promptkit.Engine, error) {
+		loaderCalls++
+		return engine, nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	app, _ := commandkit.New(commandkit.Identity{Name: "example"})
+	if err := commandkit.RegisterPromptKit(app, suite); err != nil {
+		t.Fatal(err)
+	}
+	badTarget := suite
+	badTarget.Selectors = []commandkit.Selector[struct{}]{{
+		Binding: commandkit.ValueFlag[struct{}](commandkit.FlagMeta{Name: "campaign"}, nil, func(value string) (string, error) { return value, nil }, func(*struct{}, commandkit.Supplied[string]) {}),
+		Scopes:  []string{"app"},
+	}}
+	invalidApp, _ := commandkit.New(commandkit.Identity{Name: "invalid"})
+	if err := commandkit.RegisterPromptKit(invalidApp, badTarget); err == nil {
+		t.Fatal("selector for another scope was accepted")
+	}
+	for _, tc := range []struct {
+		args []string
+		want error
+	}{
+		{args: []string{"promptkit", "show", "prompt", "team/summary"}, want: promptkit.ErrPromptLoad},
+		{args: []string{"promptkit", "show", "prompt", "missing", "--version", "1"}, want: promptkit.ErrPromptNotFound},
+		{args: []string{"promptkit", "show", "profile", "missing"}, want: promptkit.ErrProfileNotFound},
+		{args: []string{"promptkit", "show", "prompt", "", "--version", "2"}, want: promptkit.ErrInvalidRequest},
+	} {
+		outcome := app.Execute(context.Background(), tc.args, commandkit.Streams{Out: io.Discard, Err: io.Discard})
+		if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, tc.want) {
+			t.Errorf("execute %q = (%#v), want operational error matching %v", tc.args, outcome, tc.want)
+		}
+	}
+	cancelled, cancel := context.WithCancel(context.Background())
+	cancel()
+	outcome := app.Execute(cancelled, []string{"promptkit", "show", "prompt", "empty", "--version", "1"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
+	if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, context.Canceled) {
+		t.Fatalf("cancelled prompt inspection = %#v", outcome)
+	}
+	beforeHelp := loaderCalls
+	outcome = app.Execute(context.Background(), []string{"promptkit", "show", "prompt", "--help"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
+	if outcome.Kind != commandkit.OutcomeSuccess || loaderCalls != beforeHelp {
+		t.Fatalf("help acquired engine: outcome=%#v calls=%d", outcome, loaderCalls)
+	}
+	beforeMissing := loaderCalls
+	outcome = app.Execute(context.Background(), []string{"promptkit", "show", "prompt"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
+	if outcome.Kind != commandkit.OutcomeUsage || loaderCalls != beforeMissing {
+		t.Fatalf("missing ID = (%#v, calls=%d), want usage before acquisition", outcome, loaderCalls)
+	}
+
+	profileOnlyApp, _ := commandkit.New(commandkit.Identity{Name: "profile-only"})
+	if err := commandkit.RegisterPromptKit(profileOnlyApp, commandkit.PromptKitSuite[struct{}]{InspectProfile: func(ctx context.Context, _ commandkit.Selection, _ struct{}, id string) (commandkit.Acquired[*promptkit.ProfileInspection], error) {
+		value, err := engine.InspectProfile(ctx, id)
+		return commandkit.Acquired[*promptkit.ProfileInspection]{Value: value}, err
+	}}); err != nil {
+		t.Fatal(err)
+	}
+	outcome = profileOnlyApp.Execute(context.Background(), []string{"promptkit", "show", "profile", "child"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
+	if outcome.Kind != commandkit.OutcomeSuccess {
+		t.Fatalf("profile-only capability was not installed: %#v", outcome)
+	}
+	outcome = profileOnlyApp.Execute(context.Background(), []string{"promptkit", "show", "prompt", "empty"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
+	if outcome.Kind != commandkit.OutcomeUsage {
+		t.Fatalf("profile-only suite installed prompt inspection: %#v", outcome)
+	}
+	promptOnlyApp, _ := commandkit.New(commandkit.Identity{Name: "prompt-only"})
+	if err := commandkit.RegisterPromptKit(promptOnlyApp, commandkit.PromptKitSuite[struct{}]{InspectPrompt: func(ctx context.Context, _ commandkit.Selection, _ struct{}, id, version string) (commandkit.Acquired[*promptkit.PromptInspection], error) {
+		value, err := engine.InspectPrompt(ctx, id, version)
+		return commandkit.Acquired[*promptkit.PromptInspection]{Value: value}, err
+	}}); err != nil {
+		t.Fatal(err)
+	}
+	outcome = promptOnlyApp.Execute(context.Background(), []string{"promptkit", "show", "prompt", "empty"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
+	if outcome.Kind != commandkit.OutcomeSuccess {
+		t.Fatalf("prompt-only capability was not installed: %#v", outcome)
+	}
+	outcome = promptOnlyApp.Execute(context.Background(), []string{"promptkit", "show", "profile", "child"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
+	if outcome.Kind != commandkit.OutcomeUsage {
+		t.Fatalf("prompt-only suite installed profile inspection: %#v", outcome)
+	}
+
+	emptyApp, _ := commandkit.New(commandkit.Identity{Name: "empty"})
+	if err := commandkit.RegisterPromptKit(emptyApp, commandkit.PromptKitSuite[struct{}]{}); err != nil {
+		t.Fatalf("empty PromptKit suite = %v", err)
+	}
+	outcome = emptyApp.Execute(context.Background(), []string{"help"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
+	if outcome.Kind != commandkit.OutcomeSuccess {
+		t.Fatalf("empty PromptKit suite changed command family: %#v", outcome)
+	}
+	outcome = emptyApp.Execute(context.Background(), []string{"help", "promptkit"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
+	if outcome.Kind != commandkit.OutcomeUsage {
+		t.Fatalf("empty PromptKit suite installed its namespace: %#v", outcome)
+	}
+}
+
+func TestPromptKitSuitePassesSelectionAndProjectedContext(t *testing.T) {
+	type appContext struct {
+		Campaign commandkit.Supplied[string]
+		Runtime  string
+	}
+	type contextKey struct{}
+	engine := promptKitInspectionEngine(t)
+	ctx := context.WithValue(context.Background(), contextKey{}, "caller")
+	wantSelection := commandkit.Selection{
+		Config:  commandkit.Supplied[string]{Value: "main.yml", Set: true},
+		Overlay: commandkit.Supplied[string]{Value: "dev", Set: true},
+	}
+	loaderCalls, contexts, checks := 0, 0, 0
+	var gotSelection commandkit.Selection
+	var gotContext appContext
+	suite, err := commandkit.NewPromptKitSuite(func(gotCtx context.Context, selection commandkit.Selection, input appContext) (*promptkit.Engine, error) {
+		loaderCalls++
+		gotSelection, gotContext = selection, input
+		if gotCtx != ctx || gotCtx.Value(contextKey{}) != "caller" {
+			t.Fatal("engine loader did not receive invocation context")
+		}
+		return engine, nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	suite.NewContext = func() appContext { contexts++; return appContext{Runtime: "invocation cache"} }
+	suite.Selectors = []commandkit.Selector[appContext]{{
+		Binding: commandkit.ValueFlag[appContext](commandkit.FlagMeta{Name: "campaign"}, nil, func(value string) (string, error) { return value, nil }, func(input *appContext, value commandkit.Supplied[string]) { input.Campaign = value }),
+		Scopes:  []string{"promptkit"},
+	}}
+	suite.CheckInput = func(selection commandkit.Selection, input appContext) error {
+		checks++
+		if selection != wantSelection || input.Campaign != (commandkit.Supplied[string]{Value: "spring", Set: true}) {
+			t.Errorf("preflight inputs = (%+v, %+v)", selection, input)
+		}
+		return nil
+	}
+	app, _ := commandkit.New(commandkit.Identity{Name: "example"})
+	if err := commandkit.RegisterPromptKit(app, suite); err != nil {
+		t.Fatal(err)
+	}
+	outcome := app.Execute(ctx, []string{"promptkit", "show", "profile", "child", "--config", "main.yml", "--overlay", "dev", "--campaign", "spring"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
+	if outcome.Kind != commandkit.OutcomeSuccess || loaderCalls != 1 || contexts != 1 || checks != 1 {
+		t.Fatalf("inspection = (%#v), loader/context/check calls = %d/%d/%d", outcome, loaderCalls, contexts, checks)
+	}
+	if gotSelection != wantSelection || gotContext.Campaign != (commandkit.Supplied[string]{Value: "spring", Set: true}) || gotContext.Runtime != "invocation cache" {
+		t.Fatalf("loader selection/context = (%+v, %+v)", gotSelection, gotContext)
+	}
+}
+
+func TestPromptKitDirectSuiteRetainsDiagnosticsOnNamedLookupFailure(t *testing.T) {
+	warning := commandkit.Diagnostic{Severity: configkit.SeverityWarning, Write: func(w io.Writer) error { _, err := io.WriteString(w, "configuration warning\n"); return err }}
+	app, _ := commandkit.New(commandkit.Identity{Name: "example"})
+	err := commandkit.RegisterPromptKit(app, commandkit.PromptKitSuite[struct{}]{InspectPrompt: func(context.Context, commandkit.Selection, struct{}, string, string) (commandkit.Acquired[*promptkit.PromptInspection], error) {
+		return commandkit.Acquired[*promptkit.PromptInspection]{Diagnostics: []commandkit.Diagnostic{warning}}, promptkit.ErrPromptNotFound
+	}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	outcome := app.Execute(context.Background(), []string{"promptkit", "show", "prompt", "missing"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
+	if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, promptkit.ErrPromptNotFound) || len(outcome.Diagnostics) < 1 || outcome.Diagnostics[0].Severity != configkit.SeverityWarning {
+		t.Fatalf("named lookup outcome did not retain acquisition diagnostic: %#v", outcome)
+	}
+}
+
+func sortedKeys(values map[string]any) []string {
+	keys := make([]string, 0, len(values))
+	for key := range values {
+		keys = append(keys, key)
+	}
+	for i := 0; i < len(keys); i++ {
+		for j := i + 1; j < len(keys); j++ {
+			if keys[j] < keys[i] {
+				keys[i], keys[j] = keys[j], keys[i]
+			}
+		}
+	}
+	return keys
+}