commandkit/integrations/ckprompt/composition_test.go

package ckprompt_test

import (
	"bytes"
	"context"
	"io"
	"strings"
	"testing"
	"testing/fstest"

	commandkit "gitea.maximumdirect.net/eric/CommandKit"
	"gitea.maximumdirect.net/eric/CommandKit/integrations/ckconfig"
	"gitea.maximumdirect.net/eric/CommandKit/integrations/ckprompt"
	configkit "gitea.maximumdirect.net/eric/ConfigKit"
	promptkit "gitea.maximumdirect.net/eric/promptkit"
)

type compositionInput struct{ Campaign commandkit.Supplied[string] }
type compositionState struct{ token int }

func TestPublicDefinitionCommandsCaptureSelectorHandles(t *testing.T) {
	newFlag := func() commandkit.Flag[compositionInput] {
		return commandkit.ValueFlag(commandkit.FlagMeta{Name: "campaign", Required: true}, (*string)(nil),
			func(value string) (string, error) { return value, nil },
			func(input *compositionInput, value commandkit.Supplied[string]) { input.Campaign = value })
	}
	selectors := []commandkit.Flag[compositionInput]{newFlag()}
	calls := 0
	checkInput := func(input compositionInput) {
		t.Helper()
		calls++
		if !input.Campaign.Set || input.Campaign.Value != "spring" {
			t.Errorf("captured selector value = %#v", input.Campaign)
		}
	}
	declaration := ckprompt.Command(ckprompt.Suite[compositionInput, struct{}]{
		Selectors: selectors,
		InspectPrompt: func(_ context.Context, request ckprompt.PromptRequest, input compositionInput, _ struct{}) (commandkit.Acquired[*promptkit.PromptInspection], error) {
			checkInput(input)
			return commandkit.Acquired[*promptkit.PromptInspection]{Value: &promptkit.PromptInspection{PromptID: request.ID}}, nil
		},
		InspectProfile: func(_ context.Context, request ckprompt.ProfileRequest, input compositionInput, _ struct{}) (commandkit.Acquired[*promptkit.ProfileInspection], error) {
			checkInput(input)
			return commandkit.Acquired[*promptkit.ProfileInspection]{Value: &promptkit.ProfileInspection{ProfileID: request.ID}}, nil
		},
	})
	first, err := commandkit.New(commandkit.Identity{Name: "first"}, declaration)
	if err != nil {
		t.Fatal(err)
	}
	// A separately constructed lookalike must not replace the captured binding.
	selectors[0] = newFlag()
	second, err := commandkit.New(commandkit.Identity{Name: "second"}, declaration)
	if err != nil {
		t.Fatal(err)
	}
	for _, app := range []*commandkit.App{first, second} {
		for _, resource := range []string{"prompt", "profile"} {
			outcome := app.Execute(context.Background(), []string{"promptkit", "show", resource, "briefing", "--campaign", "spring"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
			if outcome.Kind != commandkit.OutcomeSuccess {
				t.Fatalf("%s after selector edit: %#v", resource, outcome)
			}
		}
	}
	if calls != 4 {
		t.Fatalf("inspection calls = %d, want 4", calls)
	}
}

func TestPromptKitAliasesComposeWithInstalledOperations(t *testing.T) {
	type input struct{ Campaign commandkit.Supplied[string] }
	selector := commandkit.ValueFlag(commandkit.FlagMeta{Name: "campaign"}, (*string)(nil),
		func(value string) (string, error) { return value, nil },
		func(target *input, value commandkit.Supplied[string]) { target.Campaign = value })
	var collectionInput input
	var inspectionInput input
	var inspectedID string
	var smokeInput input
	var smokeKind promptkit.SmokeTestKind
	callbacks := 0
	suite := ckprompt.Suite[input, struct{}]{
		Selectors: []commandkit.Flag[input]{selector},
		ListCollections: func(_ context.Context, _ ckconfig.Selection, value input, _ struct{}) (commandkit.Acquired[[]promptkit.CollectionInfo], error) {
			callbacks++
			collectionInput = value
			return commandkit.Acquired[[]promptkit.CollectionInfo]{Value: []promptkit.CollectionInfo{{Namespace: "spring"}}}, nil
		},
		InspectPrompt: func(_ context.Context, request ckprompt.PromptRequest, value input, _ struct{}) (commandkit.Acquired[*promptkit.PromptInspection], error) {
			callbacks++
			inspectionInput, inspectedID = value, request.ID
			return commandkit.Acquired[*promptkit.PromptInspection]{Value: &promptkit.PromptInspection{PromptID: request.ID}}, nil
		},
		SmokeTest: func(_ context.Context, request ckprompt.SmokeRequest, value input, _ struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
			callbacks++
			smokeInput, smokeKind = value, request.Request.Kind
			return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind, Supported: true, Completed: true}}, nil
		},
	}
	app, err := commandkit.New(commandkit.Identity{Name: "example"},
		ckprompt.Command(suite),
		commandkit.Alias("collect", "promptkit list collections"),
		commandkit.Alias("inspect", "promptkit show prompt"),
		commandkit.Alias("smoke", "promptkit test"),
		commandkit.FlagAliasFor("promptkit list collections", "--camp", "--campaign"),
		commandkit.FlagAliasFor("promptkit show prompt", "--camp", "--campaign"),
		commandkit.FlagAlias("--hlep", "--help"),
	)
	if err != nil {
		t.Fatal(err)
	}

	var output bytes.Buffer
	outcome := app.Execute(context.Background(), []string{"collect", "--camp=spring"}, commandkit.Streams{Out: &output, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeSuccess || !collectionInput.Campaign.Set || collectionInput.Campaign.Value != "spring" || !strings.Contains(output.String(), "spring") {
		t.Fatalf("collection shortcut outcome/input/report = (%#v, %#v, %q)", outcome, collectionInput, output.String())
	}
	output.Reset()
	outcome = app.Execute(context.Background(), []string{"inspect", "briefing", "--camp", "spring"}, commandkit.Streams{Out: &output, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeSuccess || !inspectionInput.Campaign.Set || inspectionInput.Campaign.Value != "spring" || inspectedID != "briefing" {
		t.Fatalf("inspection shortcut outcome/input/id = (%#v, %#v, %q)", outcome, inspectionInput, inspectedID)
	}

	output.Reset()
	outcome = app.Execute(context.Background(), []string{"smoke", "--hlep"}, commandkit.Streams{Out: &output, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(output.String(), "--help") || callbacks != 2 {
		t.Fatalf("global help alias outcome/report/callbacks = (%#v, %q, %d)", outcome, output.String(), callbacks)
	}

	if err := app.Register(commandkit.Alias("promptkit test text-inference", "promptkit test authentication")); err == nil {
		t.Fatal("accepted a command alias that maps one smoke operand to another")
	}
	output.Reset()
	outcome = app.Execute(context.Background(), []string{"promptkit", "test", "authentication", "--campaign=spring"}, commandkit.Streams{Out: &output, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeSuccess || smokeKind != promptkit.SmokeTestAuthentication || !smokeInput.Campaign.Set || smokeInput.Campaign.Value != "spring" || !strings.Contains(output.String(), "kind: authentication") || !strings.Contains(output.String(), "completed: true") {
		t.Fatalf("canonical smoke outcome/request/report = (%#v, %#v, %q)", outcome, smokeKind, output.String())
	}
	output.Reset()
	outcome = app.Execute(context.Background(), []string{"smoke", "text-inference", "--campaign=spring"}, commandkit.Streams{Out: &output, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeSuccess || smokeKind != promptkit.SmokeTestKind("text-inference") || callbacks != 4 || !strings.Contains(output.String(), "kind: text-inference") {
		t.Fatalf("smoke shortcut outcome/kind/report/callbacks = (%#v, %#v, %q, %d)", outcome, smokeKind, output.String(), callbacks)
	}
}

func TestPublicConfigurationAndDefinitionFamiliesRemainIndependent(t *testing.T) {
	inputs, err := promptkit.FilesystemInputs("composition fixture", fstest.MapFS{
		"config/prompts.yml": &fstest.MapFile{Data: []byte(`prompts:
  briefing:
    version: "1"
    default_profile: default
    messages:
      - role: user
        content: "Hello"
    output:
      format: text
      validation_mode: none
backends:
  local:
    endpoint: "https://example.invalid/v1"
profiles:
  default:
    backend: local
    model: example
`)}}, "config")
	if err != nil {
		t.Fatal(err)
	}
	engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: inputs}}}})
	if err != nil {
		t.Fatalf("construct engine: %v", err)
	}

	flag := commandkit.ValueFlag(commandkit.FlagMeta{Name: "campaign"}, (*string)(nil), func(value string) (string, error) { return value, nil },
		func(input **compositionInput, value commandkit.Supplied[string]) { (*input).Campaign = value })
	newInputCalls, newStateCalls := 0, 0
	newInput := func() *compositionInput { newInputCalls++; return &compositionInput{} }
	newState := func() *compositionState { newStateCalls++; return &compositionState{token: newStateCalls} }
	var appInput, engineInput *compositionInput
	var appState, configEngineState *compositionState
	appAcquisitions, engineScopeAcquisitions, loaderCalls := 0, 0, 0
	loadEngine := func(_ context.Context, _ ckconfig.Selection, input *compositionInput, state *compositionState) (*promptkit.Engine, error) {
		loaderCalls++
		if !input.Campaign.Set || input.Campaign.Value != "spring" {
			t.Errorf("engine loader input = %#v", input)
		}
		if loaderCalls == 1 {
			engineInput, configEngineState = input, state
		}
		return engine, nil
	}
	appScope := ckconfig.Scope[*compositionInput, *compositionState]{
		Name: "app", Operations: []ckconfig.Operation{ckconfig.Validate},
		Acquire: func(_ context.Context, _ ckconfig.Request, input *compositionInput, state *compositionState) (commandkit.Acquired[ckconfig.View], error) {
			appAcquisitions++
			appInput, appState = input, state
			return commandkit.Acquired[ckconfig.View]{}, nil
		},
	}
	engineScope := ckprompt.ConfigScope(loadEngine)
	acquireEngineScope := engineScope.Acquire
	engineScope.Acquire = func(ctx context.Context, request ckconfig.Request, input *compositionInput, state *compositionState) (commandkit.Acquired[ckconfig.View], error) {
		engineScopeAcquisitions++
		return acquireEngineScope(ctx, request, input, state)
	}
	configCommand := ckconfig.Command(ckconfig.Suite[*compositionInput, *compositionState]{
		NewInput: newInput, NewState: newState,
		Selectors: []ckconfig.Selector[*compositionInput]{{Binding: flag, Scopes: []string{"app", "promptkit"}}},
		Scopes:    []ckconfig.Scope[*compositionInput, *compositionState]{appScope, engineScope},
	})
	definitions := ckprompt.FromEngine(loadEngine)
	definitions.NewInput = newInput
	definitions.NewState = newState
	definitions.Selectors = []commandkit.Flag[*compositionInput]{flag}
	command, err := commandkit.New(commandkit.Identity{Name: "composition"}, configCommand, ckprompt.Command(definitions))
	if err != nil {
		t.Fatal(err)
	}

	var output bytes.Buffer
	outcome := command.Execute(context.Background(), []string{"config", "validate", "--scope", "all", "--campaign", "spring"}, commandkit.Streams{Out: &output, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeSuccess || appAcquisitions != 1 || engineScopeAcquisitions != 1 || loaderCalls != 1 {
		t.Fatalf("configuration run = (%#v, output=%q, counts=%d/%d/%d)", outcome, output.String(), appAcquisitions, engineScopeAcquisitions, loaderCalls)
	}
	if appInput == nil || engineInput == nil || appInput == engineInput || appState == nil || appState != configEngineState || appState.token != 1 {
		t.Fatalf("scope values were not isolated/shared as selected: inputs=%p/%p states=%p/%p", appInput, engineInput, appState, configEngineState)
	}

	output.Reset()
	outcome = command.Execute(context.Background(), []string{"promptkit", "show", "profile", "default", "--campaign", "spring", "--format", "yaml"}, commandkit.Streams{Out: &output, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeSuccess || appAcquisitions != 1 || engineScopeAcquisitions != 1 || loaderCalls != 2 || !bytes.Contains(output.Bytes(), []byte("profile_id: default")) {
		t.Fatalf("independent profile run = (%#v, output=%q, counts=%d/%d/%d)", outcome, output.String(), appAcquisitions, engineScopeAcquisitions, loaderCalls)
	}
	if newInputCalls != 3 || newStateCalls != 2 {
		t.Fatalf("invocation factory calls input/state=%d/%d", newInputCalls, newStateCalls)
	}
}