commandkit/integrations/ckprompt/smoke_lifecycle_test.go

package ckprompt

import (
	"bytes"
	"context"
	"errors"
	"io"
	"reflect"
	"strings"
	"testing"

	commandkit "gitea.maximumdirect.net/eric/CommandKit"
	"gitea.maximumdirect.net/eric/CommandKit/integrations/ckconfig"
	promptkit "gitea.maximumdirect.net/eric/promptkit"
)

func TestSmokeTestCommandRetainsSingletonValuesAndResetsInvocations(t *testing.T) {
	type input struct{ campaign string }
	type received struct {
		selection ckconfig.Selection
		request   promptkit.SmokeTestRequest
		input     input
	}
	campaignFlag := commandkit.ValueFlag(commandkit.FlagMeta{Name: "campaign"}, (*string)(nil), parseDefinitionString,
		func(target *input, value commandkit.Supplied[string]) { target.campaign = value.Value })
	var requests []received
	suite := Suite[input, struct{}]{
		NewInput:  func() input { return input{} },
		Selectors: []commandkit.Flag[input]{campaignFlag},
		SmokeTest: func(_ context.Context, request SmokeRequest, value input, _ struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
			requests = append(requests, received{selection: request.Selection, request: request.Request, input: value})
			return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind}}, nil
		},
	}
	app := newPromptTestApp(t, Command(suite))

	first := app.Execute(context.Background(), []string{
		"promptkit", "test", "authentication", "--config=old.yml", "--config=final.yml",
		"--overlay=old-overlay.yml", "--overlay=final-overlay.yml", "--campaign=old", "--campaign=final",
		"--backend=old-backend", "--backend=final-backend", "--prompt=old-prompt", "--prompt=final-prompt",
		"--profile=old-profile", "--profile=final-profile", "--model=old-model", "--model=final-model",
		"--temperature=0", "--transient-retries=0",
	}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
	if first.Kind != commandkit.OutcomeSuccess || len(requests) != 1 {
		t.Fatalf("first smoke invocation = %#v, requests=%#v", first, requests)
	}
	wantSelection := ckconfig.Selection{
		Config:  commandkit.Supplied[string]{Value: "final.yml", Set: true},
		Overlay: commandkit.Supplied[string]{Value: "final-overlay.yml", Set: true},
	}
	got := requests[0]
	if got.selection != wantSelection || got.input.campaign != "final" || got.request.BackendID != "final-backend" || got.request.PromptID != "final-prompt" || got.request.ProfileID != "final-profile" {
		t.Fatalf("last supplied values = %#v", got)
	}
	if got.request.Execution == nil || got.request.Execution.Model != "final-model" || got.request.Execution.Temperature == nil || *got.request.Execution.Temperature != 0 || got.request.TransientRetries == nil || *got.request.TransientRetries != 0 {
		t.Fatalf("last execution values = %#v", got.request)
	}

	second := app.Execute(context.Background(), []string{"promptkit", "test", "reachability"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
	if second.Kind != commandkit.OutcomeSuccess || len(requests) != 2 {
		t.Fatalf("second smoke invocation = %#v, requests=%#v", second, requests)
	}
	got = requests[1]
	if got.selection.Config.Set || got.selection.Overlay.Set || got.input.campaign != "" || got.request.BackendID != "" || got.request.PromptID != "" || got.request.ProfileID != "" || got.request.Execution != nil || got.request.TransientRetries != nil {
		t.Fatalf("values leaked between invocations: %#v", got)
	}
}

func TestSmokeTestCommandUsageAndInputFailuresStopProviderWork(t *testing.T) {
	for _, test := range []struct {
		name string
		args []string
	}{
		{name: "missing kind", args: []string{"promptkit", "test"}},
		{name: "extra operand", args: []string{"promptkit", "test", "authentication", "extra"}},
		{name: "malformed temperature", args: []string{"promptkit", "test", "authentication", "--temperature=warm"}},
		{name: "malformed token count", args: []string{"promptkit", "test", "authentication", "--max-tokens=many"}},
		{name: "malformed top-p", args: []string{"promptkit", "test", "authentication", "--top-p=high"}},
		{name: "malformed timeout", args: []string{"promptkit", "test", "authentication", "--timeout-seconds=soon"}},
		{name: "malformed retries", args: []string{"promptkit", "test", "authentication", "--transient-retries=several"}},
		{name: "unsupported format", args: []string{"promptkit", "test", "authentication", "--format=json"}},
		{name: "unknown flag", args: []string{"promptkit", "test", "authentication", "--unknown=value"}},
	} {
		t.Run(test.name, func(t *testing.T) {
			newInputCalls, checkCalls, newStateCalls, providerCalls := 0, 0, 0, 0
			suite := Suite[int, struct{}]{
				NewInput: func() int { newInputCalls++; return 1 },
				NewState: func() struct{} { newStateCalls++; return struct{}{} },
				CheckInput: func(ckconfig.Selection, int) error {
					checkCalls++
					return nil
				},
				SmokeTest: func(context.Context, SmokeRequest, int, struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
					providerCalls++
					return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{}}, nil
				},
			}
			outcome := newPromptTestApp(t, Command(suite)).Execute(context.Background(), test.args, commandkit.Streams{Out: io.Discard, Err: io.Discard})
			if outcome.Kind != commandkit.OutcomeUsage || outcome.Status != 2 || checkCalls != 0 || newStateCalls != 0 || providerCalls != 0 {
				t.Fatalf("preflight failure = %#v, input/check/state/provider=%d/%d/%d/%d", outcome, newInputCalls, checkCalls, newStateCalls, providerCalls)
			}
			if newInputCalls != 0 {
				t.Fatalf("preflight failure invoked suite input factory %d times", newInputCalls)
			}
		})
	}

	inputErr := errors.New("input rejected")
	newStateCalls, providerCalls := 0, 0
	suite := Suite[int, struct{}]{
		NewInput: func() int { return 1 },
		NewState: func() struct{} { newStateCalls++; return struct{}{} },
		CheckInput: func(ckconfig.Selection, int) error {
			return inputErr
		},
		SmokeTest: func(context.Context, SmokeRequest, int, struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
			providerCalls++
			return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{}}, nil
		},
	}
	outcome := newPromptTestApp(t, Command(suite)).Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeUsage || !errors.Is(outcome.Cause, inputErr) || newStateCalls != 0 || providerCalls != 0 {
		t.Fatalf("input-check failure reached state/provider: %#v state=%d provider=%d", outcome, newStateCalls, providerCalls)
	}
}

func TestSmokeTestCommandAvailabilityCaptureAndHelpLifecycle(t *testing.T) {
	type input struct{ campaign string }
	var originalCalls, replacementCalls, inputCalls, stateCalls int
	var capturedInput input
	campaignFlag := commandkit.ValueFlag(commandkit.FlagMeta{Name: "campaign"}, (*string)(nil), parseDefinitionString,
		func(value *input, supplied commandkit.Supplied[string]) { value.campaign = supplied.Value })
	replacementFlag := commandkit.ValueFlag(commandkit.FlagMeta{Name: "season"}, (*string)(nil), parseDefinitionString,
		func(value *input, supplied commandkit.Supplied[string]) {
			value.campaign = "replacement:" + supplied.Value
		})
	selectors := []commandkit.Flag[input]{campaignFlag}
	suite := Suite[input, int]{
		NewInput:  func() input { inputCalls++; return input{} },
		NewState:  func() int { stateCalls++; return 1 },
		Selectors: selectors,
		SmokeTest: func(_ context.Context, _ SmokeRequest, value input, _ int) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
			originalCalls++
			capturedInput = value
			return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: promptkit.SmokeTestAuthentication}}, nil
		},
	}
	declaration := Command(suite)
	suite.SmokeTest = func(context.Context, SmokeRequest, input, int) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
		replacementCalls++
		return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{}}, nil
	}
	selectors[0] = replacementFlag
	app := newPromptTestApp(t, declaration)
	var help bytes.Buffer
	outcome := app.Execute(context.Background(), []string{"help", "promptkit"}, commandkit.Streams{Out: &help, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(help.String(), "test") || inputCalls != 0 || stateCalls != 0 || originalCalls != 0 || replacementCalls != 0 {
		t.Fatalf("smoke help lifecycle = (%#v, %q), input/state/providers=%d/%d/%d/%d", outcome, help.String(), inputCalls, stateCalls, originalCalls, replacementCalls)
	}
	outcome = app.Execute(context.Background(), []string{"version"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeSuccess || inputCalls != 0 || stateCalls != 0 || originalCalls != 0 || replacementCalls != 0 {
		t.Fatalf("version lifecycle = %#v, input/state/providers=%d/%d/%d/%d", outcome, inputCalls, stateCalls, originalCalls, replacementCalls)
	}

	outcome = app.Execute(context.Background(), []string{"promptkit", "test", "authentication", "--campaign=spring"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeSuccess || originalCalls != 1 || replacementCalls != 0 || inputCalls != 1 || stateCalls != 1 || capturedInput.campaign != "spring" {
		t.Fatalf("captured callback/selector = (%#v, %#v), calls=%d/%d input/state=%d/%d", outcome, capturedInput, originalCalls, replacementCalls, inputCalls, stateCalls)
	}

	otherApp := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{
		ListPrompts: func(context.Context, CatalogRequest, struct{}, struct{}) (commandkit.Acquired[[]promptkit.PromptIdentity], error) {
			return commandkit.Acquired[[]promptkit.PromptIdentity]{}, nil
		},
	}))
	help.Reset()
	outcome = otherApp.Execute(context.Background(), []string{"help", "promptkit"}, commandkit.Streams{Out: &help, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeSuccess || strings.Contains(help.String(), "\n  test ") {
		t.Fatalf("smoke command installed without callback: (%#v, %q)", outcome, help.String())
	}

	issueSuite := WithSmokeTesting(Suite[struct{}, struct{}]{
		InspectRequirements: func(context.Context, RequirementsRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.RequirementInspection], error) {
			return commandkit.Acquired[*promptkit.RequirementInspection]{}, nil
		},
	}, nil, nil)
	issueSuite.SmokeTest = nil
	if _, err := commandkit.New(commandkit.Identity{Name: "example"}, Command(issueSuite)); err == nil || !strings.Contains(err.Error(), "PromptKit engine loader is nil") {
		t.Fatalf("narrowed suite lost smoke wiring issue: %v", err)
	}
}

func TestSmokeTestBuilderReceivesConcreteAggregateKinds(t *testing.T) {
	wantKinds := []promptkit.SmokeTestKind{
		promptkit.SmokeTestReachability,
		promptkit.SmokeTestAuthentication,
		promptkit.SmokeTestTextInference,
		promptkit.SmokeTestStructuredOutput,
	}
	var builtKinds []promptkit.SmokeTestKind
	loadErr := errors.New("engine unavailable")
	suite := WithSmokeTesting(Suite[struct{}, struct{}]{}, func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) {
		return nil, loadErr
	}, func(_ context.Context, request SmokeRequest, _ struct{}, _ struct{}) (promptkit.SmokeTestRequest, error) {
		builtKinds = append(builtKinds, request.Request.Kind)
		return request.Request, nil
	})
	app := newPromptTestApp(t, Command(suite))
	outcome := app.Execute(context.Background(), []string{"promptkit", "test", "all"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, loadErr) || !reflect.DeepEqual(builtKinds, wantKinds) {
		t.Fatalf("aggregate builder lifecycle = %#v, built=%#v", outcome, builtKinds)
	}
}

func TestSmokeTestHelpAndVersionLeaveWrappedProviderIdle(t *testing.T) {
	newInputCalls, checkCalls, newStateCalls, buildCalls, loadCalls := 0, 0, 0, 0, 0
	suite := WithSmokeTesting(Suite[int, int]{
		NewInput: func() int { newInputCalls++; return 1 },
		NewState: func() int { newStateCalls++; return 2 },
		CheckInput: func(ckconfig.Selection, int) error {
			checkCalls++
			return nil
		},
	}, func(context.Context, ckconfig.Selection, int, int) (*promptkit.Engine, error) {
		loadCalls++
		return nil, errors.New("loader should be idle")
	}, func(_ context.Context, request SmokeRequest, _ int, _ int) (promptkit.SmokeTestRequest, error) {
		buildCalls++
		return request.Request, nil
	})
	app := newPromptTestApp(t, Command(suite))
	for _, args := range [][]string{{"help", "promptkit"}, {"version"}} {
		outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: io.Discard, Err: io.Discard})
		if outcome.Kind != commandkit.OutcomeSuccess {
			t.Fatalf("%v outcome = %#v", args, outcome)
		}
	}
	if newInputCalls != 0 || checkCalls != 0 || newStateCalls != 0 || buildCalls != 0 || loadCalls != 0 {
		t.Fatalf("help/version invoked smoke work: input/check/state/build/load=%d/%d/%d/%d/%d", newInputCalls, checkCalls, newStateCalls, buildCalls, loadCalls)
	}
}

func TestSmokeTestNilFactoriesSupplyZeroValuesForAggregate(t *testing.T) {
	checkCalls, providerCalls := 0, 0
	suite := Suite[struct{ value string }, struct{ count int }]{
		CheckInput: func(_ ckconfig.Selection, value struct{ value string }) error {
			checkCalls++
			if value.value != "" {
				t.Errorf("nil input factory value = %#v", value)
			}
			return nil
		},
		SmokeTest: func(_ context.Context, _ SmokeRequest, value struct{ value string }, status struct{ count int }) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
			providerCalls++
			if value.value != "" || status.count != 0 {
				t.Errorf("nil factories yielded input/state (%#v, %#v)", value, status)
			}
			return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{}}, nil
		},
	}
	outcome := newPromptTestApp(t, Command(suite)).Execute(context.Background(), []string{"promptkit", "test", "all"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeSuccess || checkCalls != 1 || providerCalls != 4 {
		t.Fatalf("nil-factory aggregate = %#v, checks/providers=%d/%d", outcome, checkCalls, providerCalls)
	}
}