package ckprompt_test import ( "bytes" "context" "errors" "io" "strings" "testing" "testing/fstest" commandkit "gitea.maximumdirect.net/eric/CommandKit" "gitea.maximumdirect.net/eric/CommandKit/integrations/ckconfig" "gitea.maximumdirect.net/eric/CommandKit/integrations/ckprompt" configkit "gitea.maximumdirect.net/eric/ConfigKit" promptkit "gitea.maximumdirect.net/eric/promptkit" "go.yaml.in/yaml/v4" ) func TestPublicConfigDiffUsesPromptKitPairAndPreparedInputs(t *testing.T) { leftEngine := comparisonEngine(t, "Hello") rightEngine := comparisonEngine(t, "Goodbye") _, badConfigError := configkit.Load[map[string]any](configkit.Bytes("invalid PromptKit configuration", []byte("broken: ["))) if badConfigError == nil { t.Fatal("invalid configuration unexpectedly loaded") } flag := commandkit.ValueFlag(commandkit.FlagMeta{Name: "campaign"}, (*string)(nil), func(value string) (string, error) { return value, nil }, func(input **compositionInput, value commandkit.Supplied[string]) { (*input).Campaign = value }) var events []string var checked []*compositionInput var checkedSelections []ckconfig.Selection var loadedInputs []*compositionInput var loadedStates []*compositionState var loadedSelections []ckconfig.Selection loadCalls := 0 loadEngine := func(_ context.Context, selection ckconfig.Selection, input *compositionInput, state *compositionState) (*promptkit.Engine, error) { loadCalls++ events = append(events, "load") loadedInputs = append(loadedInputs, input) loadedStates = append(loadedStates, state) loadedSelections = append(loadedSelections, selection) if !input.Campaign.Set || input.Campaign.Value != "spring" { t.Errorf("loader campaign input = %#v", input.Campaign) } if strings.HasPrefix(selection.Config.Value, "bad") { return nil, badConfigError } if loadCalls%2 == 1 { return leftEngine, nil } return rightEngine, nil } appScope := ckconfig.Scope[*compositionInput, *compositionState]{ Name: "app", Operations: []ckconfig.Operation{ckconfig.Diff}, Compare: func(_ context.Context, _ ckconfig.ComparisonRequest[*compositionInput], _ *compositionState) (commandkit.Acquired[[]configkit.Difference], error) { return commandkit.Acquired[[]configkit.Difference]{}, nil }, } promptScope := ckprompt.ConfigScope(loadEngine) configCommand := ckconfig.Command(ckconfig.Suite[*compositionInput, *compositionState]{ NewInput: func() *compositionInput { return &compositionInput{} }, NewState: func() *compositionState { events = append(events, "state") return &compositionState{token: len(events)} }, Selectors: []ckconfig.Selector[*compositionInput]{{Binding: flag, Scopes: []string{"app", "promptkit"}}}, CheckInput: func(scope string, selection ckconfig.Selection, input *compositionInput) error { if scope == "promptkit" { events = append(events, "check") checked = append(checked, input) checkedSelections = append(checkedSelections, selection) if !input.Campaign.Set || input.Campaign.Value != "spring" { t.Errorf("checked campaign input = %#v", input.Campaign) } } return nil }, Scopes: []ckconfig.Scope[*compositionInput, *compositionState]{appScope, promptScope}, }) app, err := commandkit.New(commandkit.Identity{Name: "comparison"}, configCommand) if err != nil { t.Fatal(err) } for _, test := range []struct { name string rightArgs []string rightValue string rightSet bool }{ {name: "right selection is explicit", rightArgs: []string{"--against-config", "after.yml"}, rightValue: "after.yml", rightSet: true}, {name: "right selection inherits", rightValue: "before.yml", rightSet: true}, {name: "empty right selection is supplied", rightArgs: []string{"--against-config", ""}, rightSet: true}, } { t.Run(test.name, func(t *testing.T) { events = nil checked = nil checkedSelections = nil loadedInputs = nil loadedStates = nil loadedSelections = nil args := []string{"config", "diff", "--scope", "promptkit", "--config", "before.yml", "--campaign", "spring"} args = append(args, test.rightArgs...) var output bytes.Buffer outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess { t.Fatalf("comparison outcome = %#v, output %q", outcome, output.String()) } var report struct { Additions any `yaml:"additions"` Changes map[string]any `yaml:"changes"` Deletions any `yaml:"deletions"` } if err := yaml.Unmarshal(output.Bytes(), &report); err != nil { t.Fatalf("decode YAML report %q: %v", output.String(), err) } if len(report.Changes) == 0 { t.Fatalf("PromptKit change missing from report: %q", output.String()) } if len(checked) != 2 || checked[0] == checked[1] || len(loadedInputs) != 2 || loadedInputs[0] != checked[0] || loadedInputs[1] != checked[1] { t.Fatalf("separately prepared side inputs were not passed through: checked=%#v loaded=%#v", checked, loadedInputs) } if len(loadedStates) != 2 || loadedStates[0] == nil || loadedStates[0] != loadedStates[1] { t.Fatalf("comparison state was not shared: %#v", loadedStates) } if len(checkedSelections) != 2 || len(loadedSelections) != 2 { t.Fatalf("side selections: checked=%#v loaded=%#v", checkedSelections, loadedSelections) } for _, selections := range [][]ckconfig.Selection{checkedSelections, loadedSelections} { left, right := selections[0].Config, selections[1].Config if !left.Set || left.Value != "before.yml" || right.Set != test.rightSet || right.Value != test.rightValue { t.Errorf("left/right config selection = %#v / %#v", left, right) } } if len(events) < 4 || events[0] != "check" || events[1] != "check" || events[2] != "state" || events[3] != "load" { t.Fatalf("checks/state/load order = %#v", events) } }) } if loadCalls != 6 { t.Fatalf("PromptKit loader calls = %d, want six reached sides", loadCalls) } var appOutput bytes.Buffer appOutcome := app.Execute(context.Background(), []string{"config", "diff"}, commandkit.Streams{Out: &appOutput, Err: io.Discard}) if appOutcome.Kind != commandkit.OutcomeSuccess || loadCalls != 6 { t.Fatalf("bare diff should use app scope without loading PromptKit: outcome=%#v loads=%d", appOutcome, loadCalls) } unsupported := app.Execute(context.Background(), []string{"config", "diff", "--scope", "all"}, commandkit.Streams{Out: io.Discard, Err: io.Discard}) if unsupported.Kind != commandkit.OutcomeUsage || loadCalls != 6 { t.Fatalf("comparison scope all should be rejected before PromptKit loading: outcome=%#v loads=%d", unsupported, loadCalls) } var diagnostics bytes.Buffer failure := app.Execute(context.Background(), []string{"config", "diff", "--scope", "promptkit", "--config", "bad-left.yml", "--against-config", "right.yml", "--campaign", "spring"}, commandkit.Streams{Out: io.Discard, Err: &diagnostics}) if failure.Kind != commandkit.OutcomeOperational || !strings.Contains(diagnostics.String(), `Configuration scope "promptkit" (left):`) { t.Fatalf("installed load failure = %#v, diagnostics %q", failure, diagnostics.String()) } var loadFailure *configkit.LoadError if !errors.As(failure.Cause, &loadFailure) || !errors.Is(failure.Cause, badConfigError) { t.Fatalf("installed failure lost ConfigKit error identity: %#v", failure) } } func TestPublicConfigDiffDiscoveryAndNarrowedCapabilities(t *testing.T) { loads := 0 engine := comparisonEngine(t, "Hello") loadEngine := func(_ context.Context, _ ckconfig.Selection, _ *compositionInput, _ *compositionState) (*promptkit.Engine, error) { loads++ return engine, nil } scope := ckprompt.ConfigScope(loadEngine) scope.Operations = []ckconfig.Operation{ckconfig.Diff, ckconfig.Show} app, err := commandkit.New(commandkit.Identity{Name: "comparison"}, ckconfig.Command(ckconfig.Suite[*compositionInput, *compositionState]{Scopes: []ckconfig.Scope[*compositionInput, *compositionState]{scope}})) if err != nil { t.Fatal(err) } for _, args := range [][]string{ {"config", "diff", "--help"}, {"config", "scopes"}, {"version"}, } { var output bytes.Buffer outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess { t.Errorf("%v outcome = %#v", args, outcome) } if strings.Join(args, " ") == "config scopes" { for _, operation := range []string{"diff", "show"} { if !strings.Contains(output.String(), operation) { t.Errorf("scope discovery omitted %q: %s", operation, output.String()) } } for _, operation := range []string{"validate", "sources", "explain"} { if strings.Contains(output.String(), operation) { t.Errorf("scope discovery included narrowed operation %q: %s", operation, output.String()) } } } } if loads != 0 { t.Fatalf("help, version, or scope discovery loaded %d engines", loads) } showOnly := ckprompt.ConfigScope(loadEngine) showOnly.Operations = []ckconfig.Operation{ckconfig.Show} showApp, err := commandkit.New(commandkit.Identity{Name: "comparison"}, ckconfig.Command(ckconfig.Suite[*compositionInput, *compositionState]{Scopes: []ckconfig.Scope[*compositionInput, *compositionState]{showOnly}})) if err != nil { t.Fatal(err) } var output bytes.Buffer result := showApp.Execute(context.Background(), []string{"config", "show", "--scope", "promptkit"}, commandkit.Streams{Out: &output, Err: io.Discard}) if result.Kind != commandkit.OutcomeSuccess || loads != 1 || !strings.Contains(output.String(), "prompts:") { t.Fatalf("Show after clearing Diff = %#v, loads=%d, output=%q", result, loads, output.String()) } output.Reset() result = showApp.Execute(context.Background(), []string{"config", "diff", "--scope", "promptkit"}, commandkit.Streams{Out: &output, Err: io.Discard}) if result.Kind != commandkit.OutcomeUsage || loads != 1 { t.Fatalf("cleared Diff remained callable: %#v, loads=%d", result, loads) } } func comparisonEngine(t *testing.T, greeting string) *promptkit.Engine { t.Helper() configuration := `prompts: briefing: "1": messages: - role: user content: "Hello" output: format: text validation_mode: none backends: local: endpoint: "https://example.invalid/v1" profiles: default: backend: local model: example ` configuration = strings.Replace(configuration, `content: "Hello"`, `content: "`+greeting+`"`, 1) inputs, err := promptkit.FilesystemInputs("comparison fixture", fstest.MapFS{ "config/prompts.yml": &fstest.MapFile{Data: []byte(configuration)}, }, "config") if err != nil { t.Fatal(err) } engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: inputs}}}}) if err != nil { t.Fatalf("construct comparison engine: %v", err) } return engine }