Verify PromptKit comparison through config commands
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
# Implement PromptKit comparison and catalogs
|
||||
|
||||
Status: Stages 1–3 complete; remaining stages are planned.
|
||||
Status: Stages 1–4 complete; remaining stages are planned.
|
||||
|
||||
## Scope and authority
|
||||
|
||||
@@ -268,6 +268,26 @@ expectation is knowingly pending Stage 9; do not report the whole suite as passi
|
||||
while it remains. Keep assertions on semantic reports and supported outcomes,
|
||||
not exact diagnostic snapshots or private layout.
|
||||
|
||||
**Completion evidence (2026-10-12)**
|
||||
|
||||
- Added external-package integration coverage that installs `ckconfig.Command`
|
||||
with the real `ckprompt.ConfigScope`, an app scope, and a public
|
||||
`commandkit.App`. It compares two native PromptKit engines and decodes the
|
||||
existing additions/changes/deletions YAML report, including a meaningful
|
||||
change.
|
||||
- Verified explicit left/right selection, omitted-right inheritance, and a
|
||||
supplied empty right override. The same captured campaign selector reaches
|
||||
separately prepared left/right inputs; both checks finish before shared state
|
||||
creation, and the provider receives those inputs and state.
|
||||
- Confirmed bare diff uses the app scope, scope `all` is rejected before engine
|
||||
loading, installed ConfigKit failure preserves its `LoadError` identity and
|
||||
left-side diagnostic heading, and help/version/scope discovery do not load an
|
||||
engine. Narrowed discovery lists only enabled operations, and Show remains
|
||||
usable after Diff is cleared.
|
||||
- `GOTOOLCHAIN=local go test -count=1 .` passed in `integrations/ckprompt`.
|
||||
The composed example's obsolete negative expectation remains assigned to
|
||||
Stage 9; no full-module pass is claimed here.
|
||||
|
||||
## Stage 5: Add optional catalog providers to the public suite
|
||||
|
||||
**Outcome:** either catalog can be supplied independently, and `FromEngine`
|
||||
|
||||
255
integrations/ckprompt/comparison_composition_test.go
Normal file
255
integrations/ckprompt/comparison_composition_test.go
Normal file
@@ -0,0 +1,255 @@
|
||||
package ckprompt_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
commandkit "gitea.maximumdirect.net/eric/CommandKit"
|
||||
"gitea.maximumdirect.net/eric/CommandKit/integrations/ckconfig"
|
||||
"gitea.maximumdirect.net/eric/CommandKit/integrations/ckprompt"
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
promptkit "gitea.maximumdirect.net/eric/promptkit"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
func TestPublicConfigDiffUsesPromptKitPairAndPreparedInputs(t *testing.T) {
|
||||
leftEngine := comparisonEngine(t, "Hello")
|
||||
rightEngine := comparisonEngine(t, "Goodbye")
|
||||
_, badConfigError := configkit.Load[map[string]any](configkit.Bytes("invalid PromptKit configuration", []byte("broken: [")))
|
||||
if badConfigError == nil {
|
||||
t.Fatal("invalid configuration unexpectedly loaded")
|
||||
}
|
||||
flag := commandkit.ValueFlag(commandkit.FlagMeta{Name: "campaign"}, (*string)(nil),
|
||||
func(value string) (string, error) { return value, nil },
|
||||
func(input **compositionInput, value commandkit.Supplied[string]) { (*input).Campaign = value })
|
||||
var events []string
|
||||
var checked []*compositionInput
|
||||
var checkedSelections []ckconfig.Selection
|
||||
var loadedInputs []*compositionInput
|
||||
var loadedStates []*compositionState
|
||||
var loadedSelections []ckconfig.Selection
|
||||
loadCalls := 0
|
||||
loadEngine := func(_ context.Context, selection ckconfig.Selection, input *compositionInput, state *compositionState) (*promptkit.Engine, error) {
|
||||
loadCalls++
|
||||
events = append(events, "load")
|
||||
loadedInputs = append(loadedInputs, input)
|
||||
loadedStates = append(loadedStates, state)
|
||||
loadedSelections = append(loadedSelections, selection)
|
||||
if !input.Campaign.Set || input.Campaign.Value != "spring" {
|
||||
t.Errorf("loader campaign input = %#v", input.Campaign)
|
||||
}
|
||||
if strings.HasPrefix(selection.Config.Value, "bad") {
|
||||
return nil, badConfigError
|
||||
}
|
||||
if loadCalls%2 == 1 {
|
||||
return leftEngine, nil
|
||||
}
|
||||
return rightEngine, nil
|
||||
}
|
||||
appScope := ckconfig.Scope[*compositionInput, *compositionState]{
|
||||
Name: "app", Operations: []ckconfig.Operation{ckconfig.Diff},
|
||||
Compare: func(_ context.Context, _ ckconfig.ComparisonRequest[*compositionInput], _ *compositionState) (commandkit.Acquired[[]configkit.Difference], error) {
|
||||
return commandkit.Acquired[[]configkit.Difference]{}, nil
|
||||
},
|
||||
}
|
||||
promptScope := ckprompt.ConfigScope(loadEngine)
|
||||
configCommand := ckconfig.Command(ckconfig.Suite[*compositionInput, *compositionState]{
|
||||
NewInput: func() *compositionInput { return &compositionInput{} },
|
||||
NewState: func() *compositionState {
|
||||
events = append(events, "state")
|
||||
return &compositionState{token: len(events)}
|
||||
},
|
||||
Selectors: []ckconfig.Selector[*compositionInput]{{Binding: flag, Scopes: []string{"app", "promptkit"}}},
|
||||
CheckInput: func(scope string, selection ckconfig.Selection, input *compositionInput) error {
|
||||
if scope == "promptkit" {
|
||||
events = append(events, "check")
|
||||
checked = append(checked, input)
|
||||
checkedSelections = append(checkedSelections, selection)
|
||||
if !input.Campaign.Set || input.Campaign.Value != "spring" {
|
||||
t.Errorf("checked campaign input = %#v", input.Campaign)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Scopes: []ckconfig.Scope[*compositionInput, *compositionState]{appScope, promptScope},
|
||||
})
|
||||
app, err := commandkit.New(commandkit.Identity{Name: "comparison"}, configCommand)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
rightArgs []string
|
||||
rightValue string
|
||||
rightSet bool
|
||||
}{
|
||||
{name: "right selection is explicit", rightArgs: []string{"--against-config", "after.yml"}, rightValue: "after.yml", rightSet: true},
|
||||
{name: "right selection inherits", rightValue: "before.yml", rightSet: true},
|
||||
{name: "empty right selection is supplied", rightArgs: []string{"--against-config", ""}, rightSet: true},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
events = nil
|
||||
checked = nil
|
||||
checkedSelections = nil
|
||||
loadedInputs = nil
|
||||
loadedStates = nil
|
||||
loadedSelections = nil
|
||||
args := []string{"config", "diff", "--scope", "promptkit", "--config", "before.yml", "--campaign", "spring"}
|
||||
args = append(args, test.rightArgs...)
|
||||
var output bytes.Buffer
|
||||
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: io.Discard})
|
||||
if outcome.Kind != commandkit.OutcomeSuccess {
|
||||
t.Fatalf("comparison outcome = %#v, output %q", outcome, output.String())
|
||||
}
|
||||
var report struct {
|
||||
Additions any `yaml:"additions"`
|
||||
Changes map[string]any `yaml:"changes"`
|
||||
Deletions any `yaml:"deletions"`
|
||||
}
|
||||
if err := yaml.Unmarshal(output.Bytes(), &report); err != nil {
|
||||
t.Fatalf("decode YAML report %q: %v", output.String(), err)
|
||||
}
|
||||
if len(report.Changes) == 0 {
|
||||
t.Fatalf("PromptKit change missing from report: %q", output.String())
|
||||
}
|
||||
if len(checked) != 2 || checked[0] == checked[1] || len(loadedInputs) != 2 || loadedInputs[0] != checked[0] || loadedInputs[1] != checked[1] {
|
||||
t.Fatalf("separately prepared side inputs were not passed through: checked=%#v loaded=%#v", checked, loadedInputs)
|
||||
}
|
||||
if len(loadedStates) != 2 || loadedStates[0] == nil || loadedStates[0] != loadedStates[1] {
|
||||
t.Fatalf("comparison state was not shared: %#v", loadedStates)
|
||||
}
|
||||
if len(checkedSelections) != 2 || len(loadedSelections) != 2 {
|
||||
t.Fatalf("side selections: checked=%#v loaded=%#v", checkedSelections, loadedSelections)
|
||||
}
|
||||
for _, selections := range [][]ckconfig.Selection{checkedSelections, loadedSelections} {
|
||||
left, right := selections[0].Config, selections[1].Config
|
||||
if !left.Set || left.Value != "before.yml" || right.Set != test.rightSet || right.Value != test.rightValue {
|
||||
t.Errorf("left/right config selection = %#v / %#v", left, right)
|
||||
}
|
||||
}
|
||||
if len(events) < 4 || events[0] != "check" || events[1] != "check" || events[2] != "state" || events[3] != "load" {
|
||||
t.Fatalf("checks/state/load order = %#v", events)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
if loadCalls != 6 {
|
||||
t.Fatalf("PromptKit loader calls = %d, want six reached sides", loadCalls)
|
||||
}
|
||||
var appOutput bytes.Buffer
|
||||
appOutcome := app.Execute(context.Background(), []string{"config", "diff"}, commandkit.Streams{Out: &appOutput, Err: io.Discard})
|
||||
if appOutcome.Kind != commandkit.OutcomeSuccess || loadCalls != 6 {
|
||||
t.Fatalf("bare diff should use app scope without loading PromptKit: outcome=%#v loads=%d", appOutcome, loadCalls)
|
||||
}
|
||||
unsupported := app.Execute(context.Background(), []string{"config", "diff", "--scope", "all"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
|
||||
if unsupported.Kind != commandkit.OutcomeUsage || loadCalls != 6 {
|
||||
t.Fatalf("comparison scope all should be rejected before PromptKit loading: outcome=%#v loads=%d", unsupported, loadCalls)
|
||||
}
|
||||
|
||||
var diagnostics bytes.Buffer
|
||||
failure := app.Execute(context.Background(), []string{"config", "diff", "--scope", "promptkit", "--config", "bad-left.yml", "--against-config", "right.yml", "--campaign", "spring"}, commandkit.Streams{Out: io.Discard, Err: &diagnostics})
|
||||
if failure.Kind != commandkit.OutcomeOperational || !strings.Contains(diagnostics.String(), `Configuration scope "promptkit" (left):`) {
|
||||
t.Fatalf("installed load failure = %#v, diagnostics %q", failure, diagnostics.String())
|
||||
}
|
||||
var loadFailure *configkit.LoadError
|
||||
if !errors.As(failure.Cause, &loadFailure) || !errors.Is(failure.Cause, badConfigError) {
|
||||
t.Fatalf("installed failure lost ConfigKit error identity: %#v", failure)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublicConfigDiffDiscoveryAndNarrowedCapabilities(t *testing.T) {
|
||||
loads := 0
|
||||
engine := comparisonEngine(t, "Hello")
|
||||
loadEngine := func(_ context.Context, _ ckconfig.Selection, _ *compositionInput, _ *compositionState) (*promptkit.Engine, error) {
|
||||
loads++
|
||||
return engine, nil
|
||||
}
|
||||
scope := ckprompt.ConfigScope(loadEngine)
|
||||
scope.Operations = []ckconfig.Operation{ckconfig.Diff, ckconfig.Show}
|
||||
app, err := commandkit.New(commandkit.Identity{Name: "comparison"}, ckconfig.Command(ckconfig.Suite[*compositionInput, *compositionState]{Scopes: []ckconfig.Scope[*compositionInput, *compositionState]{scope}}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, args := range [][]string{
|
||||
{"config", "diff", "--help"},
|
||||
{"config", "scopes"},
|
||||
{"version"},
|
||||
} {
|
||||
var output bytes.Buffer
|
||||
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: io.Discard})
|
||||
if outcome.Kind != commandkit.OutcomeSuccess {
|
||||
t.Errorf("%v outcome = %#v", args, outcome)
|
||||
}
|
||||
if strings.Join(args, " ") == "config scopes" {
|
||||
for _, operation := range []string{"diff", "show"} {
|
||||
if !strings.Contains(output.String(), operation) {
|
||||
t.Errorf("scope discovery omitted %q: %s", operation, output.String())
|
||||
}
|
||||
}
|
||||
for _, operation := range []string{"validate", "sources", "explain"} {
|
||||
if strings.Contains(output.String(), operation) {
|
||||
t.Errorf("scope discovery included narrowed operation %q: %s", operation, output.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if loads != 0 {
|
||||
t.Fatalf("help, version, or scope discovery loaded %d engines", loads)
|
||||
}
|
||||
|
||||
showOnly := ckprompt.ConfigScope(loadEngine)
|
||||
showOnly.Operations = []ckconfig.Operation{ckconfig.Show}
|
||||
showApp, err := commandkit.New(commandkit.Identity{Name: "comparison"}, ckconfig.Command(ckconfig.Suite[*compositionInput, *compositionState]{Scopes: []ckconfig.Scope[*compositionInput, *compositionState]{showOnly}}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var output bytes.Buffer
|
||||
result := showApp.Execute(context.Background(), []string{"config", "show", "--scope", "promptkit"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
||||
if result.Kind != commandkit.OutcomeSuccess || loads != 1 || !strings.Contains(output.String(), "prompts:") {
|
||||
t.Fatalf("Show after clearing Diff = %#v, loads=%d, output=%q", result, loads, output.String())
|
||||
}
|
||||
output.Reset()
|
||||
result = showApp.Execute(context.Background(), []string{"config", "diff", "--scope", "promptkit"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
||||
if result.Kind != commandkit.OutcomeUsage || loads != 1 {
|
||||
t.Fatalf("cleared Diff remained callable: %#v, loads=%d", result, loads)
|
||||
}
|
||||
}
|
||||
|
||||
func comparisonEngine(t *testing.T, greeting string) *promptkit.Engine {
|
||||
t.Helper()
|
||||
configuration := `prompts:
|
||||
briefing:
|
||||
"1":
|
||||
messages:
|
||||
- role: user
|
||||
content: "Hello"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
backends:
|
||||
local:
|
||||
endpoint: "https://example.invalid/v1"
|
||||
profiles:
|
||||
default:
|
||||
backend: local
|
||||
model: example
|
||||
`
|
||||
configuration = strings.Replace(configuration, `content: "Hello"`, `content: "`+greeting+`"`, 1)
|
||||
inputs, err := promptkit.FilesystemInputs("comparison fixture", fstest.MapFS{
|
||||
"config/prompts.yml": &fstest.MapFile{Data: []byte(configuration)},
|
||||
}, "config")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: inputs}}}})
|
||||
if err != nil {
|
||||
t.Fatalf("construct comparison engine: %v", err)
|
||||
}
|
||||
return engine
|
||||
}
|
||||
Reference in New Issue
Block a user