commandkit/integrations/ckprompt/examples/composed/main_test.go

package main

import (
	"bytes"
	"context"
	"errors"
	"io"
	"reflect"
	"strings"
	"testing"

	commandkit "gitea.maximumdirect.net/eric/CommandKit"
	"gitea.maximumdirect.net/eric/CommandKit/integrations/ckconfig"
	"gitea.maximumdirect.net/eric/CommandKit/integrations/ckprompt"
	configkit "gitea.maximumdirect.net/eric/ConfigKit"
	promptkit "gitea.maximumdirect.net/eric/promptkit"
	"go.yaml.in/yaml/v4"
)

func TestComposedApplicationDiffUsesFixturesAndCampaignOverride(t *testing.T) {
	app, err := newApp()
	if err != nil {
		t.Fatal(err)
	}
	args := []string{"config", "diff", "--scope", "app", "--config", "fixtures/before.yml", "--against-config", "fixtures/after.yml"}
	var output bytes.Buffer
	outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeSuccess {
		t.Fatalf("fixture diff outcome = %#v", outcome)
	}
	value := composedReportValue(t, output.Bytes(), "changes", "default_campaign")
	if value == nil || value.Value != "autumn" {
		t.Fatalf("fixture diff default_campaign = %#v; report %q", value, output.String())
	}

	output.Reset()
	args = append(args, "--campaign", "spring", "--limit=0")
	outcome = app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeSuccess {
		t.Fatalf("overridden fixture diff outcome = %#v", outcome)
	}
	value = composedReportValue(t, output.Bytes(), "changes", "campaigns", "spring", "model")
	if value == nil || value.Value != "spring-model-after" {
		t.Fatalf("campaign override spring model = %#v; report %q", value, output.String())
	}
}

func TestApplicationCacheSeparatesSelectionsCampaignValuesAndPresence(t *testing.T) {
	cache := &runCache{}
	beforeSelection := ckconfigSelection("fixtures/before.yml")
	afterSelection := ckconfigSelection("fixtures/after.yml")
	load := func(selection ckconfig.Selection, input *inspectionInput) *configkit.Result[appConfiguration] {
		t.Helper()
		result, err := loadApplicationConfig(context.Background(), selection, input, cache)
		if err != nil {
			t.Fatal(err)
		}
		return result
	}

	before := load(beforeSelection, nil)
	after := load(afterSelection, nil)
	if before.Config.DefaultCampaign != "spring" || after.Config.DefaultCampaign != "autumn" || before == after {
		t.Fatalf("file selections shared or returned wrong values: before=%#v after=%#v", before.Config, after.Config)
	}
	defaultAfter := load(afterSelection, nil)
	suppliedSpring := load(afterSelection, &inspectionInput{Campaign: commandkit.Supplied[string]{Value: "spring", Set: true}})
	suppliedAutumn := load(afterSelection, &inspectionInput{Campaign: commandkit.Supplied[string]{Value: "autumn", Set: true}})
	suppliedEmpty := load(afterSelection, &inspectionInput{Campaign: commandkit.Supplied[string]{Set: true}})
	if defaultAfter != after || suppliedSpring.Config.DefaultCampaign != "spring" || suppliedAutumn.Config.DefaultCampaign != "autumn" || suppliedEmpty.Config.DefaultCampaign != "" {
		t.Fatalf("campaign loading values = default:%q spring:%q autumn:%q empty:%q", defaultAfter.Config.DefaultCampaign, suppliedSpring.Config.DefaultCampaign, suppliedAutumn.Config.DefaultCampaign, suppliedEmpty.Config.DefaultCampaign)
	}
	if suppliedSpring == suppliedAutumn || suppliedAutumn == defaultAfter || suppliedEmpty == defaultAfter || len(cache.applications) != 5 {
		t.Fatalf("cache did not distinguish selection/input presence: entries=%d", len(cache.applications))
	}

	uncachedOne, err := loadApplicationConfig(context.Background(), afterSelection, nil, nil)
	if err != nil {
		t.Fatal(err)
	}
	uncachedTwo, err := loadApplicationConfig(context.Background(), afterSelection, nil, nil)
	if err != nil {
		t.Fatal(err)
	}
	if uncachedOne == uncachedTwo {
		t.Fatal("nil cache reused an application result")
	}
}

func TestApplicationCacheDoesNotRetainFailedLoads(t *testing.T) {
	cache := &runCache{}
	selection := ckconfig.Selection{Config: commandkit.Supplied[string]{Value: "fixtures/missing.yml", Set: true}}
	for range 2 {
		if _, err := loadApplicationConfig(context.Background(), selection, nil, cache); err == nil {
			t.Fatal("missing configuration unexpectedly loaded")
		}
	}
	if len(cache.applications) != 0 {
		t.Fatalf("failed result was cached: %#v", cache.applications)
	}
}

func TestComposedPromptKitAndDomainCommandsRemainAvailable(t *testing.T) {
	app, err := newApp()
	if err != nil {
		t.Fatal(err)
	}
	commands := [][]string{
		{"config", "validate", "--scope", "all", "--campaign", "spring", "--limit=0"},
		{"config", "show", "--scope", "promptkit", "--campaign", "spring"},
		{"config", "sources", "--scope", "promptkit", "--campaign", "spring"},
		{"config", "explain", "--scope", "promptkit", "--path", "/profiles/spring/model", "--campaign", "spring"},
		{"promptkit", "show", "prompt", "briefing", "--campaign", "spring", "--format", "yaml"},
		{"promptkit", "show", "profile", "spring", "--campaign", "spring"},
		{"session", "restore", "session-42", "--campaign", "spring"},
	}
	for _, args := range commands {
		outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: io.Discard, Err: io.Discard})
		if outcome.Kind != commandkit.OutcomeSuccess {
			t.Errorf("command %q outcome = %#v", args, outcome)
		}
	}
}

func TestComposedRequirementsDiscoveryUsesSelectedCampaign(t *testing.T) {
	app, err := newApp()
	if err != nil {
		t.Fatal(err)
	}

	for _, test := range []struct {
		name string
		args []string
	}{
		{name: "without variables", args: []string{"promptkit", "requirements", "show", "briefing", "--campaign", "spring"}},
		{name: "with caller variable", args: []string{"promptkit", "requirements", "show", "briefing", "--campaign", "spring", "--var", "mode=full", "--format", "yaml"}},
	} {
		t.Run(test.name, func(t *testing.T) {
			var output bytes.Buffer
			outcome := app.Execute(context.Background(), test.args, commandkit.Streams{Out: &output, Err: io.Discard})
			if outcome.Kind != commandkit.OutcomeSuccess || outcome.Status != 0 {
				t.Fatalf("requirements outcome = %#v, report %q", outcome, output.String())
			}

			var report map[string]any
			if err := yaml.Unmarshal(output.Bytes(), &report); err != nil {
				t.Fatalf("decode requirements report %q: %v", output.String(), err)
			}
			for _, key := range []string{"prompt_id", "version", "profile_id", "complete", "inputs", "variables", "issues"} {
				if _, found := report[key]; !found {
					t.Errorf("requirements report is missing %q: %#v", key, report)
				}
			}
			if report["prompt_id"] != "briefing" || report["version"] != "1" || report["profile_id"] != "spring" {
				t.Fatalf("requirements report metadata = %#v", report)
			}
		})
	}
}

func TestComposedRequestValidationUsesFileBindings(t *testing.T) {
	app, err := newApp()
	if err != nil {
		t.Fatal(err)
	}

	for _, test := range []struct {
		name string
		args []string
	}{
		{
			name: "default mode",
			args: []string{"promptkit", "requirements", "validate", "briefing", "--campaign", "spring", "--input", "topic=fixtures/topic.txt"},
		},
		{
			name: "conditional notes",
			args: []string{"promptkit", "requirements", "validate", "briefing", "--campaign", "spring", "--input", "topic=fixtures/topic.txt", "--var", "mode=full", "--input", "notes=fixtures/notes.txt", "--format", "yaml"},
		},
	} {
		t.Run(test.name, func(t *testing.T) {
			var output bytes.Buffer
			outcome := app.Execute(context.Background(), test.args, commandkit.Streams{Out: &output, Err: io.Discard})
			if outcome.Kind != commandkit.OutcomeSuccess || outcome.Status != 0 {
				t.Fatalf("request validation = %#v, report %q", outcome, output.String())
			}
			var report map[string]any
			if err := yaml.Unmarshal(output.Bytes(), &report); err != nil {
				t.Fatalf("decode validation confirmation %q: %v", output.String(), err)
			}
			if len(report) != 2 || report["prompt_id"] != "briefing" || report["valid"] != true {
				t.Fatalf("validation confirmation = %#v", report)
			}
		})
	}
}

func TestComposedRequestValidationRequiresTopicArtifact(t *testing.T) {
	app, err := newApp()
	if err != nil {
		t.Fatal(err)
	}
	var output bytes.Buffer
	outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing", "--campaign", "spring"}, commandkit.Streams{Out: &output, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || outcome.Cause == nil || output.Len() != 0 {
		t.Fatalf("missing topic artifact outcome = %#v, report %q", outcome, output.String())
	}
}

func TestBuildValidationRequestDefaultsOnlyAbsentVariables(t *testing.T) {
	input := &inspectionInput{Campaign: commandkit.Supplied[string]{Value: "autumn", Set: true}}
	cache := &runCache{}
	inputs := map[string]promptkit.ArtifactRef{"topic": promptkit.File("fixtures/topic.txt")}
	explicit := map[string]string{"topic": "", "mode": "", "audience": ""}
	seed := ckprompt.ValidationRequest{Request: promptkit.RunRequest{
		PromptID:  "briefing",
		ProfileID: "custom-profile",
		SessionID: "session-9",
		Vars:      explicit,
		Inputs:    inputs,
	}}

	got, err := buildValidationRequest(context.Background(), seed, input, cache)
	if err != nil {
		t.Fatal(err)
	}
	if got.PromptID != seed.Request.PromptID || got.ProfileID != seed.Request.ProfileID || got.SessionID != seed.Request.SessionID {
		t.Fatalf("request identity/context changed: %#v", got)
	}
	if got.Vars["topic"] != "" || got.Vars["mode"] != "" || got.Vars["audience"] != "" {
		t.Fatalf("explicit empty values were replaced: %#v", got.Vars)
	}
	if !reflect.DeepEqual(got.Inputs, inputs) || !reflect.DeepEqual(seed.Request.Vars, explicit) {
		t.Fatalf("seed bindings changed: built=%#v seed=%#v", got.Inputs, explicit)
	}
	key := applicationCacheKey{selection: seed.Selection, campaign: input.Campaign}
	cached := cache.applications[key]
	if cached == nil {
		t.Fatal("builder did not load application configuration into invocation state")
	}
	if _, err := loadPromptEngine(context.Background(), seed.Selection, input, cache); err != nil {
		t.Fatal(err)
	}
	if cache.applications[key] != cached {
		t.Fatal("engine loader did not reuse the builder's application configuration")
	}

	missing := ckprompt.ValidationRequest{Request: promptkit.RunRequest{PromptID: "briefing", Vars: map[string]string{"audience": "internal"}}}
	defaulted, err := buildValidationRequest(context.Background(), missing, input, cache)
	if err != nil {
		t.Fatal(err)
	}
	if defaulted.Vars["topic"] != "autumn campaign planning" || defaulted.Vars["mode"] != "" || defaulted.Vars["audience"] != "internal" {
		t.Fatalf("missing variables were not defaulted while retaining supplied values: %#v", defaulted.Vars)
	}
	if _, ok := missing.Request.Vars["topic"]; ok {
		t.Fatalf("builder modified the seed variable map: %#v", missing.Request.Vars)
	}
}

func TestPromptKitScopeDiffAndCatalogsUseSelectedCampaign(t *testing.T) {
	app, err := newApp()
	if err != nil {
		t.Fatal(err)
	}
	var output bytes.Buffer
	outcome := app.Execute(context.Background(), []string{"config", "diff", "--scope", "promptkit", "--config", "fixtures/before.yml", "--against-config", "fixtures/after.yml", "--campaign", "spring"}, commandkit.Streams{Out: &output, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeSuccess {
		t.Fatalf("PromptKit diff outcome = %#v", outcome)
	}
	value := composedReportValue(t, output.Bytes(), "changes", "profiles", "spring", "model")
	if value == nil || value.Value != "spring-model-after" {
		t.Fatalf("PromptKit diff spring model = %#v; report %q", value, output.String())
	}

	output.Reset()
	outcome = app.Execute(context.Background(), []string{"promptkit", "list", "prompts", "--config", "fixtures/before.yml", "--campaign", "spring"}, commandkit.Streams{Out: &output, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeSuccess {
		t.Fatalf("prompt catalog outcome = %#v", outcome)
	}
	var prompts []map[string]any
	if err := yaml.Unmarshal(output.Bytes(), &prompts); err != nil {
		t.Fatalf("decode prompt catalog %q: %v", output.String(), err)
	}
	foundPrompt := false
	for _, prompt := range prompts {
		if prompt["id"] == "briefing" && prompt["version"] == "1" {
			foundPrompt = true
			break
		}
	}
	if !foundPrompt {
		t.Fatalf("selected prompt identity missing from catalog: %#v", prompts)
	}

	output.Reset()
	outcome = app.Execute(context.Background(), []string{"promptkit", "list", "profiles", "--config", "fixtures/after.yml", "--campaign", "autumn", "--format", "yaml"}, commandkit.Streams{Out: &output, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeSuccess {
		t.Fatalf("profile catalog outcome = %#v", outcome)
	}
	var profiles []string
	if err := yaml.Unmarshal(output.Bytes(), &profiles); err != nil {
		t.Fatalf("decode profile catalog %q: %v", output.String(), err)
	}
	foundProfile := false
	for _, profile := range profiles {
		if profile == "autumn" {
			foundProfile = true
			break
		}
	}
	if !foundProfile {
		t.Fatalf("selected profile missing from catalog: %#v", profiles)
	}
}

func TestComposedCatalogsAndSmokeUseInjectedClientOnlyForInference(t *testing.T) {
	calls := 0
	app, err := newAppWithLLMClient(&exampleLLMClient{calls: &calls})
	if err != nil {
		t.Fatal(err)
	}

	commands := [][]string{
		{"promptkit", "list", "backends", "--campaign", "spring"},
		{"promptkit", "list", "namespaces", "--campaign", "spring"},
		{"promptkit", "list", "collections", "--campaign", "spring"},
		{"promptkit", "list", "prompts", "--namespace", "field-notes", "--campaign", "spring"},
		{"promptkit", "show", "prompt", "briefing", "--campaign", "spring"},
		{"promptkit", "requirements", "show", "briefing", "--campaign", "spring"},
	}
	for _, args := range commands {
		var output bytes.Buffer
		outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: io.Discard})
		if outcome.Kind != commandkit.OutcomeSuccess {
			t.Errorf("metadata command %q outcome = %#v; output %q", args, outcome, output.String())
		}
		if output.Len() == 0 {
			t.Errorf("metadata command %q produced no report", args)
		}
	}
	if calls != 0 {
		t.Fatalf("metadata and requirements commands invoked the LLM client %d times", calls)
	}

	var output bytes.Buffer
	outcome := app.Execute(context.Background(), []string{"promptkit", "test", "all", "--campaign", "spring", "--prompt", "briefing", "--profile", "spring"}, commandkit.Streams{Out: &output, Err: io.Discard})
	if output.Len() == 0 || !bytes.Contains(output.Bytes(), []byte("tests:")) {
		t.Fatalf("aggregate smoke report missing: outcome=%#v output=%q", outcome, output.String())
	}
	if calls == 0 {
		t.Fatalf("smoke command did not use the injected LLM client: outcome=%#v output=%q", outcome, output.String())
	}
}

func TestComposedSmokeReportSurvivesInferenceFailure(t *testing.T) {
	wantCause := errors.New("local example inference failed")
	smokeTest := func(_ context.Context, request ckprompt.SmokeRequest, _ *inspectionInput, _ *runCache) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
		return commandkit.Acquired[*promptkit.SmokeTestReport]{
			Value:       &promptkit.SmokeTestReport{Kind: request.Request.Kind, Completed: true},
			Diagnostics: []commandkit.Diagnostic{commandkit.MessageDiagnostic(commandkit.SeverityWarning, "independent smoke warning")},
			Explanation: commandkit.Explanation{Details: []commandkit.Detail{func(writer io.Writer) error {
				_, err := io.WriteString(writer, "Native text inference context.\n")
				return err
			}}},
		}, wantCause
	}
	app, err := newAppWithEngineLoader(&exampleLLMClient{}, loadPromptEngine, smokeTest)
	if err != nil {
		t.Fatal(err)
	}
	var output, diagnostics bytes.Buffer
	outcome := app.Execute(context.Background(), []string{"promptkit", "test", "text_inference", "--campaign", "spring", "--prompt", "briefing", "--profile", "spring", "--verbose"}, commandkit.Streams{Out: &output, Err: &diagnostics})
	if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || !errors.Is(outcome.Cause, wantCause) {
		t.Fatalf("smoke failure outcome = %#v; report %q; diagnostics %q", outcome, output.String(), diagnostics.String())
	}
	if !strings.Contains(output.String(), "kind: text_inference") || !strings.Contains(output.String(), "completed: true") {
		t.Fatalf("native smoke report was not delivered: %q", output.String())
	}
	warningAt := strings.Index(diagnostics.String(), "independent smoke warning")
	contextAt := strings.Index(diagnostics.String(), "Native text inference context")
	if warningAt < 0 || contextAt < 0 || warningAt >= contextAt || strings.Contains(diagnostics.String(), wantCause.Error()) {
		t.Fatalf("terminal explanation/diagnostics = %q / %q", outcome.Explanation.Message, diagnostics.String())
	}
	if outcome.Explanation.Message != "" {
		t.Errorf("unexpected synthesized terminal explanation %q", outcome.Explanation.Message)
	}
}

func TestComposedCustomRestoreUsesSharedReportingAndTerminalFailure(t *testing.T) {
	app, err := newApp()
	if err != nil {
		t.Fatal(err)
	}

	var output, diagnostics bytes.Buffer
	outcome := app.Execute(context.Background(), []string{"session", "restore", "session-42", "--campaign", "spring", "--quiet"}, commandkit.Streams{Out: &output, Err: &diagnostics})
	if outcome.Kind != commandkit.OutcomeSuccess || output.Len() != 0 || diagnostics.Len() != 0 {
		t.Fatalf("quiet custom command outcome/output/diagnostics = %#v / %q / %q", outcome, output.String(), diagnostics.String())
	}

	output.Reset()
	diagnostics.Reset()
	outcome = app.Execute(context.Background(), []string{"session", "restore", "missing", "--quiet"}, commandkit.Streams{Out: &output, Err: &diagnostics})
	if outcome.Kind != commandkit.OutcomeDomain || outcome.Status != 4 || outcome.Domain != "session_not_found" || !errors.Is(outcome.Cause, errSessionNotFound) {
		t.Fatalf("missing session outcome = %#v", outcome)
	}
	if output.Len() != 0 || outcome.Explanation.Message == "" || !strings.Contains(diagnostics.String(), outcome.Explanation.Message) {
		t.Fatalf("terminal failure output/explanation/diagnostics = %q / %q / %q", output.String(), outcome.Explanation.Message, diagnostics.String())
	}
	if strings.Contains(diagnostics.String(), errSessionNotFound.Error()) {
		t.Errorf("terminal presentation exposed the underlying cause: %q", diagnostics.String())
	}
}

func TestComposedAliasesReachCanonicalCommands(t *testing.T) {
	clientCalls := 0
	engineLoads := 0
	engineLoader := func(ctx context.Context, selection ckconfig.Selection, input *inspectionInput, cache *runCache) (*promptkit.Engine, error) {
		engineLoads++
		return loadPromptEngine(ctx, selection, input, cache)
	}
	app, err := newAppWithEngineLoader(&exampleLLMClient{calls: &clientCalls}, engineLoader, nil)
	if err != nil {
		t.Fatal(err)
	}

	var output bytes.Buffer
	outcome := app.Execute(context.Background(), []string{"pk", "--hlep"}, commandkit.Streams{Out: &output, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(output.String(), "--hlep → --help") || strings.Contains(output.String(), "field-notes") || engineLoads != 0 || clientCalls != 0 {
		t.Fatalf("group help alias outcome/output/engine loads/client calls = (%#v, %q, %d, %d)", outcome, output.String(), engineLoads, clientCalls)
	}
	output.Reset()
	outcome = app.Execute(context.Background(), []string{"pk", "--hlep", "--quiet"}, commandkit.Streams{Out: &output, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeSuccess || output.Len() != 0 || engineLoads != 0 || clientCalls != 0 {
		t.Fatalf("quiet aliased help outcome/output/engine loads/client calls = (%#v, %q, %d, %d)", outcome, output.String(), engineLoads, clientCalls)
	}

	for _, args := range [][]string{
		{"list", "collections", "--campaign", "spring"},
		{"pk", "list", "bundles", "--campaign", "spring"},
		{"promptkit", "list", "collections", "--campaign", "spring"},
	} {
		output.Reset()
		outcome = app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: io.Discard})
		if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(output.String(), "field-notes") {
			t.Fatalf("collection route %q outcome/report = (%#v, %q)", args, outcome, output.String())
		}
	}

	for _, args := range [][]string{
		{"pk", "show", "prompt", "briefing", "--campaign", "spring", "--fmt", "yaml"},
		{"promptkit", "show", "prompt", "briefing", "--campaign", "spring", "--format", "yaml"},
	} {
		output.Reset()
		outcome = app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: io.Discard})
		if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(output.String(), "prompt_id: briefing") {
			t.Fatalf("prompt inspection route %q outcome/report = (%#v, %q)", args, outcome, output.String())
		}
	}

	output.Reset()
	outcome = app.Execute(context.Background(), []string{"smoke", "text_inference", "--campaign", "spring", "--prompt", "briefing", "--profile", "spring"}, commandkit.Streams{Out: &output, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(output.String(), "kind: text_inference") || !strings.Contains(output.String(), "completed: true") || clientCalls != 1 {
		t.Fatalf("smoke shortcut outcome/report/client calls = (%#v, %q, %d)", outcome, output.String(), clientCalls)
	}
}

func composedReportValue(t *testing.T, data []byte, category string, fields ...string) *yaml.Node {
	t.Helper()
	var document yaml.Node
	if err := yaml.Unmarshal(data, &document); err != nil {
		t.Fatalf("decode composed report %q: %v", data, err)
	}
	node := composedField(document.Content[0], category)
	for _, field := range fields {
		node = composedField(node, field)
	}
	return node
}

func composedField(mapping *yaml.Node, name string) *yaml.Node {
	if mapping == nil || mapping.Kind != yaml.MappingNode {
		return nil
	}
	for index := 0; index+1 < len(mapping.Content); index += 2 {
		if mapping.Content[index].Value == name {
			return mapping.Content[index+1]
		}
	}
	return nil
}

func ckconfigSelection(path string) ckconfig.Selection {
	return ckconfig.Selection{Config: commandkit.Supplied[string]{Value: path, Set: true}}
}