256 lines
11 KiB
Go
256 lines
11 KiB
Go
package ckprompt_test
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"io"
|
|
"strings"
|
|
"testing"
|
|
"testing/fstest"
|
|
|
|
commandkit "gitea.maximumdirect.net/eric/CommandKit"
|
|
"gitea.maximumdirect.net/eric/CommandKit/integrations/ckconfig"
|
|
"gitea.maximumdirect.net/eric/CommandKit/integrations/ckprompt"
|
|
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
|
promptkit "gitea.maximumdirect.net/eric/promptkit"
|
|
"gopkg.in/yaml.v3"
|
|
)
|
|
|
|
func TestPublicConfigDiffUsesPromptKitPairAndPreparedInputs(t *testing.T) {
|
|
leftEngine := comparisonEngine(t, "Hello")
|
|
rightEngine := comparisonEngine(t, "Goodbye")
|
|
_, badConfigError := configkit.Load[map[string]any](configkit.Bytes("invalid PromptKit configuration", []byte("broken: [")))
|
|
if badConfigError == nil {
|
|
t.Fatal("invalid configuration unexpectedly loaded")
|
|
}
|
|
flag := commandkit.ValueFlag(commandkit.FlagMeta{Name: "campaign"}, (*string)(nil),
|
|
func(value string) (string, error) { return value, nil },
|
|
func(input **compositionInput, value commandkit.Supplied[string]) { (*input).Campaign = value })
|
|
var events []string
|
|
var checked []*compositionInput
|
|
var checkedSelections []ckconfig.Selection
|
|
var loadedInputs []*compositionInput
|
|
var loadedStates []*compositionState
|
|
var loadedSelections []ckconfig.Selection
|
|
loadCalls := 0
|
|
loadEngine := func(_ context.Context, selection ckconfig.Selection, input *compositionInput, state *compositionState) (*promptkit.Engine, error) {
|
|
loadCalls++
|
|
events = append(events, "load")
|
|
loadedInputs = append(loadedInputs, input)
|
|
loadedStates = append(loadedStates, state)
|
|
loadedSelections = append(loadedSelections, selection)
|
|
if !input.Campaign.Set || input.Campaign.Value != "spring" {
|
|
t.Errorf("loader campaign input = %#v", input.Campaign)
|
|
}
|
|
if strings.HasPrefix(selection.Config.Value, "bad") {
|
|
return nil, badConfigError
|
|
}
|
|
if loadCalls%2 == 1 {
|
|
return leftEngine, nil
|
|
}
|
|
return rightEngine, nil
|
|
}
|
|
appScope := ckconfig.Scope[*compositionInput, *compositionState]{
|
|
Name: "app", Operations: []ckconfig.Operation{ckconfig.Diff},
|
|
Compare: func(_ context.Context, _ ckconfig.ComparisonRequest[*compositionInput], _ *compositionState) (commandkit.Acquired[[]configkit.Difference], error) {
|
|
return commandkit.Acquired[[]configkit.Difference]{}, nil
|
|
},
|
|
}
|
|
promptScope := ckprompt.ConfigScope(loadEngine)
|
|
configCommand := ckconfig.Command(ckconfig.Suite[*compositionInput, *compositionState]{
|
|
NewInput: func() *compositionInput { return &compositionInput{} },
|
|
NewState: func() *compositionState {
|
|
events = append(events, "state")
|
|
return &compositionState{token: len(events)}
|
|
},
|
|
Selectors: []ckconfig.Selector[*compositionInput]{{Binding: flag, Scopes: []string{"app", "promptkit"}}},
|
|
CheckInput: func(scope string, selection ckconfig.Selection, input *compositionInput) error {
|
|
if scope == "promptkit" {
|
|
events = append(events, "check")
|
|
checked = append(checked, input)
|
|
checkedSelections = append(checkedSelections, selection)
|
|
if !input.Campaign.Set || input.Campaign.Value != "spring" {
|
|
t.Errorf("checked campaign input = %#v", input.Campaign)
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
Scopes: []ckconfig.Scope[*compositionInput, *compositionState]{appScope, promptScope},
|
|
})
|
|
app, err := commandkit.New(commandkit.Identity{Name: "comparison"}, configCommand)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
for _, test := range []struct {
|
|
name string
|
|
rightArgs []string
|
|
rightValue string
|
|
rightSet bool
|
|
}{
|
|
{name: "right selection is explicit", rightArgs: []string{"--against-config", "after.yml"}, rightValue: "after.yml", rightSet: true},
|
|
{name: "right selection inherits", rightValue: "before.yml", rightSet: true},
|
|
{name: "empty right selection is supplied", rightArgs: []string{"--against-config", ""}, rightSet: true},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
events = nil
|
|
checked = nil
|
|
checkedSelections = nil
|
|
loadedInputs = nil
|
|
loadedStates = nil
|
|
loadedSelections = nil
|
|
args := []string{"config", "diff", "--scope", "promptkit", "--config", "before.yml", "--campaign", "spring"}
|
|
args = append(args, test.rightArgs...)
|
|
var output bytes.Buffer
|
|
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: io.Discard})
|
|
if outcome.Kind != commandkit.OutcomeSuccess {
|
|
t.Fatalf("comparison outcome = %#v, output %q", outcome, output.String())
|
|
}
|
|
var report struct {
|
|
Additions any `yaml:"additions"`
|
|
Changes map[string]any `yaml:"changes"`
|
|
Deletions any `yaml:"deletions"`
|
|
}
|
|
if err := yaml.Unmarshal(output.Bytes(), &report); err != nil {
|
|
t.Fatalf("decode YAML report %q: %v", output.String(), err)
|
|
}
|
|
if len(report.Changes) == 0 {
|
|
t.Fatalf("PromptKit change missing from report: %q", output.String())
|
|
}
|
|
if len(checked) != 2 || checked[0] == checked[1] || len(loadedInputs) != 2 || loadedInputs[0] != checked[0] || loadedInputs[1] != checked[1] {
|
|
t.Fatalf("separately prepared side inputs were not passed through: checked=%#v loaded=%#v", checked, loadedInputs)
|
|
}
|
|
if len(loadedStates) != 2 || loadedStates[0] == nil || loadedStates[0] != loadedStates[1] {
|
|
t.Fatalf("comparison state was not shared: %#v", loadedStates)
|
|
}
|
|
if len(checkedSelections) != 2 || len(loadedSelections) != 2 {
|
|
t.Fatalf("side selections: checked=%#v loaded=%#v", checkedSelections, loadedSelections)
|
|
}
|
|
for _, selections := range [][]ckconfig.Selection{checkedSelections, loadedSelections} {
|
|
left, right := selections[0].Config, selections[1].Config
|
|
if !left.Set || left.Value != "before.yml" || right.Set != test.rightSet || right.Value != test.rightValue {
|
|
t.Errorf("left/right config selection = %#v / %#v", left, right)
|
|
}
|
|
}
|
|
if len(events) < 4 || events[0] != "check" || events[1] != "check" || events[2] != "state" || events[3] != "load" {
|
|
t.Fatalf("checks/state/load order = %#v", events)
|
|
}
|
|
})
|
|
}
|
|
|
|
if loadCalls != 6 {
|
|
t.Fatalf("PromptKit loader calls = %d, want six reached sides", loadCalls)
|
|
}
|
|
var appOutput bytes.Buffer
|
|
appOutcome := app.Execute(context.Background(), []string{"config", "diff"}, commandkit.Streams{Out: &appOutput, Err: io.Discard})
|
|
if appOutcome.Kind != commandkit.OutcomeSuccess || loadCalls != 6 {
|
|
t.Fatalf("bare diff should use app scope without loading PromptKit: outcome=%#v loads=%d", appOutcome, loadCalls)
|
|
}
|
|
unsupported := app.Execute(context.Background(), []string{"config", "diff", "--scope", "all"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
|
|
if unsupported.Kind != commandkit.OutcomeUsage || loadCalls != 6 {
|
|
t.Fatalf("comparison scope all should be rejected before PromptKit loading: outcome=%#v loads=%d", unsupported, loadCalls)
|
|
}
|
|
|
|
var diagnostics bytes.Buffer
|
|
failure := app.Execute(context.Background(), []string{"config", "diff", "--scope", "promptkit", "--config", "bad-left.yml", "--against-config", "right.yml", "--campaign", "spring"}, commandkit.Streams{Out: io.Discard, Err: &diagnostics})
|
|
if failure.Kind != commandkit.OutcomeOperational || !strings.Contains(diagnostics.String(), `Configuration scope "promptkit" (left):`) {
|
|
t.Fatalf("installed load failure = %#v, diagnostics %q", failure, diagnostics.String())
|
|
}
|
|
var loadFailure *configkit.LoadError
|
|
if !errors.As(failure.Cause, &loadFailure) || !errors.Is(failure.Cause, badConfigError) {
|
|
t.Fatalf("installed failure lost ConfigKit error identity: %#v", failure)
|
|
}
|
|
}
|
|
|
|
func TestPublicConfigDiffDiscoveryAndNarrowedCapabilities(t *testing.T) {
|
|
loads := 0
|
|
engine := comparisonEngine(t, "Hello")
|
|
loadEngine := func(_ context.Context, _ ckconfig.Selection, _ *compositionInput, _ *compositionState) (*promptkit.Engine, error) {
|
|
loads++
|
|
return engine, nil
|
|
}
|
|
scope := ckprompt.ConfigScope(loadEngine)
|
|
scope.Operations = []ckconfig.Operation{ckconfig.Diff, ckconfig.Show}
|
|
app, err := commandkit.New(commandkit.Identity{Name: "comparison"}, ckconfig.Command(ckconfig.Suite[*compositionInput, *compositionState]{Scopes: []ckconfig.Scope[*compositionInput, *compositionState]{scope}}))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, args := range [][]string{
|
|
{"config", "diff", "--help"},
|
|
{"config", "scopes"},
|
|
{"version"},
|
|
} {
|
|
var output bytes.Buffer
|
|
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: io.Discard})
|
|
if outcome.Kind != commandkit.OutcomeSuccess {
|
|
t.Errorf("%v outcome = %#v", args, outcome)
|
|
}
|
|
if strings.Join(args, " ") == "config scopes" {
|
|
for _, operation := range []string{"diff", "show"} {
|
|
if !strings.Contains(output.String(), operation) {
|
|
t.Errorf("scope discovery omitted %q: %s", operation, output.String())
|
|
}
|
|
}
|
|
for _, operation := range []string{"validate", "sources", "explain"} {
|
|
if strings.Contains(output.String(), operation) {
|
|
t.Errorf("scope discovery included narrowed operation %q: %s", operation, output.String())
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if loads != 0 {
|
|
t.Fatalf("help, version, or scope discovery loaded %d engines", loads)
|
|
}
|
|
|
|
showOnly := ckprompt.ConfigScope(loadEngine)
|
|
showOnly.Operations = []ckconfig.Operation{ckconfig.Show}
|
|
showApp, err := commandkit.New(commandkit.Identity{Name: "comparison"}, ckconfig.Command(ckconfig.Suite[*compositionInput, *compositionState]{Scopes: []ckconfig.Scope[*compositionInput, *compositionState]{showOnly}}))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var output bytes.Buffer
|
|
result := showApp.Execute(context.Background(), []string{"config", "show", "--scope", "promptkit"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
|
if result.Kind != commandkit.OutcomeSuccess || loads != 1 || !strings.Contains(output.String(), "prompts:") {
|
|
t.Fatalf("Show after clearing Diff = %#v, loads=%d, output=%q", result, loads, output.String())
|
|
}
|
|
output.Reset()
|
|
result = showApp.Execute(context.Background(), []string{"config", "diff", "--scope", "promptkit"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
|
if result.Kind != commandkit.OutcomeUsage || loads != 1 {
|
|
t.Fatalf("cleared Diff remained callable: %#v, loads=%d", result, loads)
|
|
}
|
|
}
|
|
|
|
func comparisonEngine(t *testing.T, greeting string) *promptkit.Engine {
|
|
t.Helper()
|
|
configuration := `prompts:
|
|
briefing:
|
|
"1":
|
|
messages:
|
|
- role: user
|
|
content: "Hello"
|
|
output:
|
|
format: text
|
|
validation_mode: none
|
|
backends:
|
|
local:
|
|
endpoint: "https://example.invalid/v1"
|
|
profiles:
|
|
default:
|
|
backend: local
|
|
model: example
|
|
`
|
|
configuration = strings.Replace(configuration, `content: "Hello"`, `content: "`+greeting+`"`, 1)
|
|
inputs, err := promptkit.FilesystemInputs("comparison fixture", fstest.MapFS{
|
|
"config/prompts.yml": &fstest.MapFile{Data: []byte(configuration)},
|
|
}, "config")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: inputs}}}})
|
|
if err != nil {
|
|
t.Fatalf("construct comparison engine: %v", err)
|
|
}
|
|
return engine
|
|
}
|