Verify PromptKit comparison through config commands

docs/roadmap/implementation.md

@@ -1,6 +1,6 @@
 # Implement PromptKit comparison and catalogs
 
-Status: Stages 1–3 complete; remaining stages are planned.
+Status: Stages 1–4 complete; remaining stages are planned.
 
 ## Scope and authority
 
@@ -268,6 +268,26 @@ expectation is knowingly pending Stage 9; do not report the whole suite as passi
 while it remains. Keep assertions on semantic reports and supported outcomes,
 not exact diagnostic snapshots or private layout.
 
+**Completion evidence (2026-10-12)**
+
+- Added external-package integration coverage that installs `ckconfig.Command`
+  with the real `ckprompt.ConfigScope`, an app scope, and a public
+  `commandkit.App`. It compares two native PromptKit engines and decodes the
+  existing additions/changes/deletions YAML report, including a meaningful
+  change.
+- Verified explicit left/right selection, omitted-right inheritance, and a
+  supplied empty right override. The same captured campaign selector reaches
+  separately prepared left/right inputs; both checks finish before shared state
+  creation, and the provider receives those inputs and state.
+- Confirmed bare diff uses the app scope, scope `all` is rejected before engine
+  loading, installed ConfigKit failure preserves its `LoadError` identity and
+  left-side diagnostic heading, and help/version/scope discovery do not load an
+  engine. Narrowed discovery lists only enabled operations, and Show remains
+  usable after Diff is cleared.
+- `GOTOOLCHAIN=local go test -count=1 .` passed in `integrations/ckprompt`.
+  The composed example's obsolete negative expectation remains assigned to
+  Stage 9; no full-module pass is claimed here.
+
 ## Stage 5: Add optional catalog providers to the public suite
 
 **Outcome:** either catalog can be supplied independently, and `FromEngine`

integrations/ckprompt/comparison_composition_test.go

@@ -0,0 +1,255 @@
+package ckprompt_test
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"io"
+	"strings"
+	"testing"
+	"testing/fstest"
+
+	commandkit "gitea.maximumdirect.net/eric/CommandKit"
+	"gitea.maximumdirect.net/eric/CommandKit/integrations/ckconfig"
+	"gitea.maximumdirect.net/eric/CommandKit/integrations/ckprompt"
+	configkit "gitea.maximumdirect.net/eric/ConfigKit"
+	promptkit "gitea.maximumdirect.net/eric/promptkit"
+	"gopkg.in/yaml.v3"
+)
+
+func TestPublicConfigDiffUsesPromptKitPairAndPreparedInputs(t *testing.T) {
+	leftEngine := comparisonEngine(t, "Hello")
+	rightEngine := comparisonEngine(t, "Goodbye")
+	_, badConfigError := configkit.Load[map[string]any](configkit.Bytes("invalid PromptKit configuration", []byte("broken: [")))
+	if badConfigError == nil {
+		t.Fatal("invalid configuration unexpectedly loaded")
+	}
+	flag := commandkit.ValueFlag(commandkit.FlagMeta{Name: "campaign"}, (*string)(nil),
+		func(value string) (string, error) { return value, nil },
+		func(input **compositionInput, value commandkit.Supplied[string]) { (*input).Campaign = value })
+	var events []string
+	var checked []*compositionInput
+	var checkedSelections []ckconfig.Selection
+	var loadedInputs []*compositionInput
+	var loadedStates []*compositionState
+	var loadedSelections []ckconfig.Selection
+	loadCalls := 0
+	loadEngine := func(_ context.Context, selection ckconfig.Selection, input *compositionInput, state *compositionState) (*promptkit.Engine, error) {
+		loadCalls++
+		events = append(events, "load")
+		loadedInputs = append(loadedInputs, input)
+		loadedStates = append(loadedStates, state)
+		loadedSelections = append(loadedSelections, selection)
+		if !input.Campaign.Set || input.Campaign.Value != "spring" {
+			t.Errorf("loader campaign input = %#v", input.Campaign)
+		}
+		if strings.HasPrefix(selection.Config.Value, "bad") {
+			return nil, badConfigError
+		}
+		if loadCalls%2 == 1 {
+			return leftEngine, nil
+		}
+		return rightEngine, nil
+	}
+	appScope := ckconfig.Scope[*compositionInput, *compositionState]{
+		Name: "app", Operations: []ckconfig.Operation{ckconfig.Diff},
+		Compare: func(_ context.Context, _ ckconfig.ComparisonRequest[*compositionInput], _ *compositionState) (commandkit.Acquired[[]configkit.Difference], error) {
+			return commandkit.Acquired[[]configkit.Difference]{}, nil
+		},
+	}
+	promptScope := ckprompt.ConfigScope(loadEngine)
+	configCommand := ckconfig.Command(ckconfig.Suite[*compositionInput, *compositionState]{
+		NewInput: func() *compositionInput { return &compositionInput{} },
+		NewState: func() *compositionState {
+			events = append(events, "state")
+			return &compositionState{token: len(events)}
+		},
+		Selectors: []ckconfig.Selector[*compositionInput]{{Binding: flag, Scopes: []string{"app", "promptkit"}}},
+		CheckInput: func(scope string, selection ckconfig.Selection, input *compositionInput) error {
+			if scope == "promptkit" {
+				events = append(events, "check")
+				checked = append(checked, input)
+				checkedSelections = append(checkedSelections, selection)
+				if !input.Campaign.Set || input.Campaign.Value != "spring" {
+					t.Errorf("checked campaign input = %#v", input.Campaign)
+				}
+			}
+			return nil
+		},
+		Scopes: []ckconfig.Scope[*compositionInput, *compositionState]{appScope, promptScope},
+	})
+	app, err := commandkit.New(commandkit.Identity{Name: "comparison"}, configCommand)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, test := range []struct {
+		name       string
+		rightArgs  []string
+		rightValue string
+		rightSet   bool
+	}{
+		{name: "right selection is explicit", rightArgs: []string{"--against-config", "after.yml"}, rightValue: "after.yml", rightSet: true},
+		{name: "right selection inherits", rightValue: "before.yml", rightSet: true},
+		{name: "empty right selection is supplied", rightArgs: []string{"--against-config", ""}, rightSet: true},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			events = nil
+			checked = nil
+			checkedSelections = nil
+			loadedInputs = nil
+			loadedStates = nil
+			loadedSelections = nil
+			args := []string{"config", "diff", "--scope", "promptkit", "--config", "before.yml", "--campaign", "spring"}
+			args = append(args, test.rightArgs...)
+			var output bytes.Buffer
+			outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: io.Discard})
+			if outcome.Kind != commandkit.OutcomeSuccess {
+				t.Fatalf("comparison outcome = %#v, output %q", outcome, output.String())
+			}
+			var report struct {
+				Additions any            `yaml:"additions"`
+				Changes   map[string]any `yaml:"changes"`
+				Deletions any            `yaml:"deletions"`
+			}
+			if err := yaml.Unmarshal(output.Bytes(), &report); err != nil {
+				t.Fatalf("decode YAML report %q: %v", output.String(), err)
+			}
+			if len(report.Changes) == 0 {
+				t.Fatalf("PromptKit change missing from report: %q", output.String())
+			}
+			if len(checked) != 2 || checked[0] == checked[1] || len(loadedInputs) != 2 || loadedInputs[0] != checked[0] || loadedInputs[1] != checked[1] {
+				t.Fatalf("separately prepared side inputs were not passed through: checked=%#v loaded=%#v", checked, loadedInputs)
+			}
+			if len(loadedStates) != 2 || loadedStates[0] == nil || loadedStates[0] != loadedStates[1] {
+				t.Fatalf("comparison state was not shared: %#v", loadedStates)
+			}
+			if len(checkedSelections) != 2 || len(loadedSelections) != 2 {
+				t.Fatalf("side selections: checked=%#v loaded=%#v", checkedSelections, loadedSelections)
+			}
+			for _, selections := range [][]ckconfig.Selection{checkedSelections, loadedSelections} {
+				left, right := selections[0].Config, selections[1].Config
+				if !left.Set || left.Value != "before.yml" || right.Set != test.rightSet || right.Value != test.rightValue {
+					t.Errorf("left/right config selection = %#v / %#v", left, right)
+				}
+			}
+			if len(events) < 4 || events[0] != "check" || events[1] != "check" || events[2] != "state" || events[3] != "load" {
+				t.Fatalf("checks/state/load order = %#v", events)
+			}
+		})
+	}
+
+	if loadCalls != 6 {
+		t.Fatalf("PromptKit loader calls = %d, want six reached sides", loadCalls)
+	}
+	var appOutput bytes.Buffer
+	appOutcome := app.Execute(context.Background(), []string{"config", "diff"}, commandkit.Streams{Out: &appOutput, Err: io.Discard})
+	if appOutcome.Kind != commandkit.OutcomeSuccess || loadCalls != 6 {
+		t.Fatalf("bare diff should use app scope without loading PromptKit: outcome=%#v loads=%d", appOutcome, loadCalls)
+	}
+	unsupported := app.Execute(context.Background(), []string{"config", "diff", "--scope", "all"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
+	if unsupported.Kind != commandkit.OutcomeUsage || loadCalls != 6 {
+		t.Fatalf("comparison scope all should be rejected before PromptKit loading: outcome=%#v loads=%d", unsupported, loadCalls)
+	}
+
+	var diagnostics bytes.Buffer
+	failure := app.Execute(context.Background(), []string{"config", "diff", "--scope", "promptkit", "--config", "bad-left.yml", "--against-config", "right.yml", "--campaign", "spring"}, commandkit.Streams{Out: io.Discard, Err: &diagnostics})
+	if failure.Kind != commandkit.OutcomeOperational || !strings.Contains(diagnostics.String(), `Configuration scope "promptkit" (left):`) {
+		t.Fatalf("installed load failure = %#v, diagnostics %q", failure, diagnostics.String())
+	}
+	var loadFailure *configkit.LoadError
+	if !errors.As(failure.Cause, &loadFailure) || !errors.Is(failure.Cause, badConfigError) {
+		t.Fatalf("installed failure lost ConfigKit error identity: %#v", failure)
+	}
+}
+
+func TestPublicConfigDiffDiscoveryAndNarrowedCapabilities(t *testing.T) {
+	loads := 0
+	engine := comparisonEngine(t, "Hello")
+	loadEngine := func(_ context.Context, _ ckconfig.Selection, _ *compositionInput, _ *compositionState) (*promptkit.Engine, error) {
+		loads++
+		return engine, nil
+	}
+	scope := ckprompt.ConfigScope(loadEngine)
+	scope.Operations = []ckconfig.Operation{ckconfig.Diff, ckconfig.Show}
+	app, err := commandkit.New(commandkit.Identity{Name: "comparison"}, ckconfig.Command(ckconfig.Suite[*compositionInput, *compositionState]{Scopes: []ckconfig.Scope[*compositionInput, *compositionState]{scope}}))
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, args := range [][]string{
+		{"config", "diff", "--help"},
+		{"config", "scopes"},
+		{"version"},
+	} {
+		var output bytes.Buffer
+		outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: io.Discard})
+		if outcome.Kind != commandkit.OutcomeSuccess {
+			t.Errorf("%v outcome = %#v", args, outcome)
+		}
+		if strings.Join(args, " ") == "config scopes" {
+			for _, operation := range []string{"diff", "show"} {
+				if !strings.Contains(output.String(), operation) {
+					t.Errorf("scope discovery omitted %q: %s", operation, output.String())
+				}
+			}
+			for _, operation := range []string{"validate", "sources", "explain"} {
+				if strings.Contains(output.String(), operation) {
+					t.Errorf("scope discovery included narrowed operation %q: %s", operation, output.String())
+				}
+			}
+		}
+	}
+	if loads != 0 {
+		t.Fatalf("help, version, or scope discovery loaded %d engines", loads)
+	}
+
+	showOnly := ckprompt.ConfigScope(loadEngine)
+	showOnly.Operations = []ckconfig.Operation{ckconfig.Show}
+	showApp, err := commandkit.New(commandkit.Identity{Name: "comparison"}, ckconfig.Command(ckconfig.Suite[*compositionInput, *compositionState]{Scopes: []ckconfig.Scope[*compositionInput, *compositionState]{showOnly}}))
+	if err != nil {
+		t.Fatal(err)
+	}
+	var output bytes.Buffer
+	result := showApp.Execute(context.Background(), []string{"config", "show", "--scope", "promptkit"}, commandkit.Streams{Out: &output, Err: io.Discard})
+	if result.Kind != commandkit.OutcomeSuccess || loads != 1 || !strings.Contains(output.String(), "prompts:") {
+		t.Fatalf("Show after clearing Diff = %#v, loads=%d, output=%q", result, loads, output.String())
+	}
+	output.Reset()
+	result = showApp.Execute(context.Background(), []string{"config", "diff", "--scope", "promptkit"}, commandkit.Streams{Out: &output, Err: io.Discard})
+	if result.Kind != commandkit.OutcomeUsage || loads != 1 {
+		t.Fatalf("cleared Diff remained callable: %#v, loads=%d", result, loads)
+	}
+}
+
+func comparisonEngine(t *testing.T, greeting string) *promptkit.Engine {
+	t.Helper()
+	configuration := `prompts:
+  briefing:
+    "1":
+      messages:
+        - role: user
+          content: "Hello"
+      output:
+        format: text
+        validation_mode: none
+backends:
+  local:
+    endpoint: "https://example.invalid/v1"
+profiles:
+  default:
+    backend: local
+    model: example
+`
+	configuration = strings.Replace(configuration, `content: "Hello"`, `content: "`+greeting+`"`, 1)
+	inputs, err := promptkit.FilesystemInputs("comparison fixture", fstest.MapFS{
+		"config/prompts.yml": &fstest.MapFile{Data: []byte(configuration)},
+	}, "config")
+	if err != nil {
+		t.Fatal(err)
+	}
+	engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: inputs}}}})
+	if err != nil {
+		t.Fatalf("construct comparison engine: %v", err)
+	}
+	return engine
+}