diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 445cad6..71e20b4 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1,6 +1,6 @@ # Implement PromptKit comparison and catalogs -Status: Stages 1–3 complete; remaining stages are planned. +Status: Stages 1–4 complete; remaining stages are planned. ## Scope and authority @@ -268,6 +268,26 @@ expectation is knowingly pending Stage 9; do not report the whole suite as passi while it remains. Keep assertions on semantic reports and supported outcomes, not exact diagnostic snapshots or private layout. +**Completion evidence (2026-10-12)** + +- Added external-package integration coverage that installs `ckconfig.Command` + with the real `ckprompt.ConfigScope`, an app scope, and a public + `commandkit.App`. It compares two native PromptKit engines and decodes the + existing additions/changes/deletions YAML report, including a meaningful + change. +- Verified explicit left/right selection, omitted-right inheritance, and a + supplied empty right override. The same captured campaign selector reaches + separately prepared left/right inputs; both checks finish before shared state + creation, and the provider receives those inputs and state. +- Confirmed bare diff uses the app scope, scope `all` is rejected before engine + loading, installed ConfigKit failure preserves its `LoadError` identity and + left-side diagnostic heading, and help/version/scope discovery do not load an + engine. Narrowed discovery lists only enabled operations, and Show remains + usable after Diff is cleared. +- `GOTOOLCHAIN=local go test -count=1 .` passed in `integrations/ckprompt`. + The composed example's obsolete negative expectation remains assigned to + Stage 9; no full-module pass is claimed here. + ## Stage 5: Add optional catalog providers to the public suite **Outcome:** either catalog can be supplied independently, and `FromEngine` diff --git a/integrations/ckprompt/comparison_composition_test.go b/integrations/ckprompt/comparison_composition_test.go new file mode 100644 index 0000000..6a4b42e --- /dev/null +++ b/integrations/ckprompt/comparison_composition_test.go @@ -0,0 +1,255 @@ +package ckprompt_test + +import ( + "bytes" + "context" + "errors" + "io" + "strings" + "testing" + "testing/fstest" + + commandkit "gitea.maximumdirect.net/eric/CommandKit" + "gitea.maximumdirect.net/eric/CommandKit/integrations/ckconfig" + "gitea.maximumdirect.net/eric/CommandKit/integrations/ckprompt" + configkit "gitea.maximumdirect.net/eric/ConfigKit" + promptkit "gitea.maximumdirect.net/eric/promptkit" + "gopkg.in/yaml.v3" +) + +func TestPublicConfigDiffUsesPromptKitPairAndPreparedInputs(t *testing.T) { + leftEngine := comparisonEngine(t, "Hello") + rightEngine := comparisonEngine(t, "Goodbye") + _, badConfigError := configkit.Load[map[string]any](configkit.Bytes("invalid PromptKit configuration", []byte("broken: ["))) + if badConfigError == nil { + t.Fatal("invalid configuration unexpectedly loaded") + } + flag := commandkit.ValueFlag(commandkit.FlagMeta{Name: "campaign"}, (*string)(nil), + func(value string) (string, error) { return value, nil }, + func(input **compositionInput, value commandkit.Supplied[string]) { (*input).Campaign = value }) + var events []string + var checked []*compositionInput + var checkedSelections []ckconfig.Selection + var loadedInputs []*compositionInput + var loadedStates []*compositionState + var loadedSelections []ckconfig.Selection + loadCalls := 0 + loadEngine := func(_ context.Context, selection ckconfig.Selection, input *compositionInput, state *compositionState) (*promptkit.Engine, error) { + loadCalls++ + events = append(events, "load") + loadedInputs = append(loadedInputs, input) + loadedStates = append(loadedStates, state) + loadedSelections = append(loadedSelections, selection) + if !input.Campaign.Set || input.Campaign.Value != "spring" { + t.Errorf("loader campaign input = %#v", input.Campaign) + } + if strings.HasPrefix(selection.Config.Value, "bad") { + return nil, badConfigError + } + if loadCalls%2 == 1 { + return leftEngine, nil + } + return rightEngine, nil + } + appScope := ckconfig.Scope[*compositionInput, *compositionState]{ + Name: "app", Operations: []ckconfig.Operation{ckconfig.Diff}, + Compare: func(_ context.Context, _ ckconfig.ComparisonRequest[*compositionInput], _ *compositionState) (commandkit.Acquired[[]configkit.Difference], error) { + return commandkit.Acquired[[]configkit.Difference]{}, nil + }, + } + promptScope := ckprompt.ConfigScope(loadEngine) + configCommand := ckconfig.Command(ckconfig.Suite[*compositionInput, *compositionState]{ + NewInput: func() *compositionInput { return &compositionInput{} }, + NewState: func() *compositionState { + events = append(events, "state") + return &compositionState{token: len(events)} + }, + Selectors: []ckconfig.Selector[*compositionInput]{{Binding: flag, Scopes: []string{"app", "promptkit"}}}, + CheckInput: func(scope string, selection ckconfig.Selection, input *compositionInput) error { + if scope == "promptkit" { + events = append(events, "check") + checked = append(checked, input) + checkedSelections = append(checkedSelections, selection) + if !input.Campaign.Set || input.Campaign.Value != "spring" { + t.Errorf("checked campaign input = %#v", input.Campaign) + } + } + return nil + }, + Scopes: []ckconfig.Scope[*compositionInput, *compositionState]{appScope, promptScope}, + }) + app, err := commandkit.New(commandkit.Identity{Name: "comparison"}, configCommand) + if err != nil { + t.Fatal(err) + } + + for _, test := range []struct { + name string + rightArgs []string + rightValue string + rightSet bool + }{ + {name: "right selection is explicit", rightArgs: []string{"--against-config", "after.yml"}, rightValue: "after.yml", rightSet: true}, + {name: "right selection inherits", rightValue: "before.yml", rightSet: true}, + {name: "empty right selection is supplied", rightArgs: []string{"--against-config", ""}, rightSet: true}, + } { + t.Run(test.name, func(t *testing.T) { + events = nil + checked = nil + checkedSelections = nil + loadedInputs = nil + loadedStates = nil + loadedSelections = nil + args := []string{"config", "diff", "--scope", "promptkit", "--config", "before.yml", "--campaign", "spring"} + args = append(args, test.rightArgs...) + var output bytes.Buffer + outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeSuccess { + t.Fatalf("comparison outcome = %#v, output %q", outcome, output.String()) + } + var report struct { + Additions any `yaml:"additions"` + Changes map[string]any `yaml:"changes"` + Deletions any `yaml:"deletions"` + } + if err := yaml.Unmarshal(output.Bytes(), &report); err != nil { + t.Fatalf("decode YAML report %q: %v", output.String(), err) + } + if len(report.Changes) == 0 { + t.Fatalf("PromptKit change missing from report: %q", output.String()) + } + if len(checked) != 2 || checked[0] == checked[1] || len(loadedInputs) != 2 || loadedInputs[0] != checked[0] || loadedInputs[1] != checked[1] { + t.Fatalf("separately prepared side inputs were not passed through: checked=%#v loaded=%#v", checked, loadedInputs) + } + if len(loadedStates) != 2 || loadedStates[0] == nil || loadedStates[0] != loadedStates[1] { + t.Fatalf("comparison state was not shared: %#v", loadedStates) + } + if len(checkedSelections) != 2 || len(loadedSelections) != 2 { + t.Fatalf("side selections: checked=%#v loaded=%#v", checkedSelections, loadedSelections) + } + for _, selections := range [][]ckconfig.Selection{checkedSelections, loadedSelections} { + left, right := selections[0].Config, selections[1].Config + if !left.Set || left.Value != "before.yml" || right.Set != test.rightSet || right.Value != test.rightValue { + t.Errorf("left/right config selection = %#v / %#v", left, right) + } + } + if len(events) < 4 || events[0] != "check" || events[1] != "check" || events[2] != "state" || events[3] != "load" { + t.Fatalf("checks/state/load order = %#v", events) + } + }) + } + + if loadCalls != 6 { + t.Fatalf("PromptKit loader calls = %d, want six reached sides", loadCalls) + } + var appOutput bytes.Buffer + appOutcome := app.Execute(context.Background(), []string{"config", "diff"}, commandkit.Streams{Out: &appOutput, Err: io.Discard}) + if appOutcome.Kind != commandkit.OutcomeSuccess || loadCalls != 6 { + t.Fatalf("bare diff should use app scope without loading PromptKit: outcome=%#v loads=%d", appOutcome, loadCalls) + } + unsupported := app.Execute(context.Background(), []string{"config", "diff", "--scope", "all"}, commandkit.Streams{Out: io.Discard, Err: io.Discard}) + if unsupported.Kind != commandkit.OutcomeUsage || loadCalls != 6 { + t.Fatalf("comparison scope all should be rejected before PromptKit loading: outcome=%#v loads=%d", unsupported, loadCalls) + } + + var diagnostics bytes.Buffer + failure := app.Execute(context.Background(), []string{"config", "diff", "--scope", "promptkit", "--config", "bad-left.yml", "--against-config", "right.yml", "--campaign", "spring"}, commandkit.Streams{Out: io.Discard, Err: &diagnostics}) + if failure.Kind != commandkit.OutcomeOperational || !strings.Contains(diagnostics.String(), `Configuration scope "promptkit" (left):`) { + t.Fatalf("installed load failure = %#v, diagnostics %q", failure, diagnostics.String()) + } + var loadFailure *configkit.LoadError + if !errors.As(failure.Cause, &loadFailure) || !errors.Is(failure.Cause, badConfigError) { + t.Fatalf("installed failure lost ConfigKit error identity: %#v", failure) + } +} + +func TestPublicConfigDiffDiscoveryAndNarrowedCapabilities(t *testing.T) { + loads := 0 + engine := comparisonEngine(t, "Hello") + loadEngine := func(_ context.Context, _ ckconfig.Selection, _ *compositionInput, _ *compositionState) (*promptkit.Engine, error) { + loads++ + return engine, nil + } + scope := ckprompt.ConfigScope(loadEngine) + scope.Operations = []ckconfig.Operation{ckconfig.Diff, ckconfig.Show} + app, err := commandkit.New(commandkit.Identity{Name: "comparison"}, ckconfig.Command(ckconfig.Suite[*compositionInput, *compositionState]{Scopes: []ckconfig.Scope[*compositionInput, *compositionState]{scope}})) + if err != nil { + t.Fatal(err) + } + for _, args := range [][]string{ + {"config", "diff", "--help"}, + {"config", "scopes"}, + {"version"}, + } { + var output bytes.Buffer + outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeSuccess { + t.Errorf("%v outcome = %#v", args, outcome) + } + if strings.Join(args, " ") == "config scopes" { + for _, operation := range []string{"diff", "show"} { + if !strings.Contains(output.String(), operation) { + t.Errorf("scope discovery omitted %q: %s", operation, output.String()) + } + } + for _, operation := range []string{"validate", "sources", "explain"} { + if strings.Contains(output.String(), operation) { + t.Errorf("scope discovery included narrowed operation %q: %s", operation, output.String()) + } + } + } + } + if loads != 0 { + t.Fatalf("help, version, or scope discovery loaded %d engines", loads) + } + + showOnly := ckprompt.ConfigScope(loadEngine) + showOnly.Operations = []ckconfig.Operation{ckconfig.Show} + showApp, err := commandkit.New(commandkit.Identity{Name: "comparison"}, ckconfig.Command(ckconfig.Suite[*compositionInput, *compositionState]{Scopes: []ckconfig.Scope[*compositionInput, *compositionState]{showOnly}})) + if err != nil { + t.Fatal(err) + } + var output bytes.Buffer + result := showApp.Execute(context.Background(), []string{"config", "show", "--scope", "promptkit"}, commandkit.Streams{Out: &output, Err: io.Discard}) + if result.Kind != commandkit.OutcomeSuccess || loads != 1 || !strings.Contains(output.String(), "prompts:") { + t.Fatalf("Show after clearing Diff = %#v, loads=%d, output=%q", result, loads, output.String()) + } + output.Reset() + result = showApp.Execute(context.Background(), []string{"config", "diff", "--scope", "promptkit"}, commandkit.Streams{Out: &output, Err: io.Discard}) + if result.Kind != commandkit.OutcomeUsage || loads != 1 { + t.Fatalf("cleared Diff remained callable: %#v, loads=%d", result, loads) + } +} + +func comparisonEngine(t *testing.T, greeting string) *promptkit.Engine { + t.Helper() + configuration := `prompts: + briefing: + "1": + messages: + - role: user + content: "Hello" + output: + format: text + validation_mode: none +backends: + local: + endpoint: "https://example.invalid/v1" +profiles: + default: + backend: local + model: example +` + configuration = strings.Replace(configuration, `content: "Hello"`, `content: "`+greeting+`"`, 1) + inputs, err := promptkit.FilesystemInputs("comparison fixture", fstest.MapFS{ + "config/prompts.yml": &fstest.MapFile{Data: []byte(configuration)}, + }, "config") + if err != nil { + t.Fatal(err) + } + engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: inputs}}}}) + if err != nil { + t.Fatalf("construct comparison engine: %v", err) + } + return engine +}