commandkit/integrations/ckprompt/smoke_command_test.go

package ckprompt

import (
	"bytes"
	"context"
	"errors"
	"io"
	"strings"
	"testing"

	commandkit "gitea.maximumdirect.net/eric/CommandKit"
	"gitea.maximumdirect.net/eric/CommandKit/integrations/ckconfig"
	promptkit "gitea.maximumdirect.net/eric/promptkit"
	"go.yaml.in/yaml/v4"
)

func TestSmokeTestCommandBindsRequestAndRendersFindings(t *testing.T) {
	type input struct{ campaign string }
	type state struct{ initialized bool }
	campaignFlag := commandkit.ValueFlag(commandkit.FlagMeta{Name: "campaign", Usage: "Campaign selector."}, (*string)(nil), parseDefinitionString,
		func(value *input, campaign commandkit.Supplied[string]) { value.campaign = campaign.Value })

	var gotSelection ckconfig.Selection
	var gotRequest promptkit.SmokeTestRequest
	var gotInput input
	var gotState state
	providerCalls, checkedCalls := 0, 0
	suite := Suite[input, state]{
		NewInput:  func() input { return input{} },
		NewState:  func() state { return state{initialized: true} },
		Selectors: []commandkit.Flag[input]{campaignFlag},
		CheckInput: func(selection ckconfig.Selection, value input) error {
			checkedCalls++
			if checkedCalls == 1 && (!selection.Config.Set || selection.Config.Value != "config.yml" || selection.Overlay.Value != "overlay.yml" || value.campaign != "spring") {
				t.Errorf("checked selection and input = (%#v, %#v)", selection, value)
			}
			return nil
		},
		SmokeTest: func(_ context.Context, request SmokeRequest, value input, status state) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
			providerCalls++
			gotSelection, gotRequest, gotInput, gotState = request.Selection, request.Request, value, status
			return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind, Supported: true, Completed: true}}, nil
		},
	}
	app := newPromptTestApp(t, Command(suite))

	var help bytes.Buffer
	outcome := app.Execute(context.Background(), []string{"help", "promptkit"}, commandkit.Streams{Out: &help, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(help.String(), "test") {
		t.Fatalf("promptkit help = (%#v, %q)", outcome, help.String())
	}

	var output bytes.Buffer
	outcome = app.Execute(context.Background(), []string{
		"promptkit", "test", "authentication", "--config=config.yml", "--overlay=overlay.yml", "--campaign=spring",
		"--backend=local", "--prompt=briefing", "--profile=editorial", "--endpoint=https://example.invalid",
		"--model=fixture", "--temperature=0", "--max-tokens=0", "--top-p=0", "--timeout-seconds=0",
		"--service-tier=", "--reasoning-effort=", "--api-key-env=TOKEN", "--transient-retries=0",
	}, commandkit.Streams{Out: &output, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeSuccess || providerCalls != 1 {
		t.Fatalf("smoke outcome = (%#v, %q), calls=%d", outcome, output.String(), providerCalls)
	}
	if gotSelection.Config.Value != "config.yml" || gotSelection.Overlay.Value != "overlay.yml" || gotInput.campaign != "spring" || !gotState.initialized {
		t.Errorf("provider selection/input/state = (%#v, %#v, %#v)", gotSelection, gotInput, gotState)
	}
	if gotRequest.Kind != promptkit.SmokeTestAuthentication || gotRequest.BackendID != "local" || gotRequest.PromptID != "briefing" || gotRequest.ProfileID != "editorial" {
		t.Errorf("smoke request identity = %#v", gotRequest)
	}
	if gotRequest.Execution == nil || gotRequest.Execution.Endpoint != "https://example.invalid" || gotRequest.Execution.Model != "fixture" || gotRequest.Execution.Temperature == nil || *gotRequest.Execution.Temperature != 0 || gotRequest.Execution.MaxTokens == nil || *gotRequest.Execution.MaxTokens != 0 || gotRequest.Execution.TopP == nil || *gotRequest.Execution.TopP != 0 || gotRequest.Execution.TimeoutSeconds == nil || *gotRequest.Execution.TimeoutSeconds != 0 || gotRequest.Execution.ServiceTier != "" || gotRequest.Execution.ReasoningEffort == nil || *gotRequest.Execution.ReasoningEffort != "" || gotRequest.Execution.APIKeyEnv != "TOKEN" || gotRequest.TransientRetries == nil || *gotRequest.TransientRetries != 0 {
		t.Errorf("smoke execution overrides did not preserve supplied values: %#v", gotRequest)
	}
	var report map[string]any
	if err := yaml.Unmarshal(output.Bytes(), &report); err != nil {
		t.Fatalf("decode smoke report %q: %v", output.String(), err)
	}
	if report["kind"] != "authentication" || report["completed"] != true {
		t.Errorf("smoke report = %#v", report)
	}

	var omittedRequest promptkit.SmokeTestRequest
	omittedApp := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{
		SmokeTest: func(_ context.Context, request SmokeRequest, _ struct{}, _ struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
			omittedRequest = request.Request
			return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind}}, nil
		},
	}))
	output.Reset()
	outcome = omittedApp.Execute(context.Background(), []string{"promptkit", "test", "text-inference"}, commandkit.Streams{Out: &output, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeSuccess || omittedRequest.Execution != nil || omittedRequest.TransientRetries != nil {
		t.Fatalf("omitted execution overrides = (%#v, %#v), execution nil=%t retries nil=%t", outcome, omittedRequest, omittedRequest.Execution == nil, omittedRequest.TransientRetries == nil)
	}

	providerCalls = 0
	output.Reset()
	outcome = app.Execute(context.Background(), []string{"promptkit", "test", "all"}, commandkit.Streams{Out: &output, Err: io.Discard})
	var aggregate struct {
		Tests []struct {
			Kind string `yaml:"kind"`
		} `yaml:"tests"`
	}
	if outcome.Kind != commandkit.OutcomeSuccess || providerCalls != 4 || yaml.Unmarshal(output.Bytes(), &aggregate) != nil || len(aggregate.Tests) != 4 {
		t.Fatalf("aggregate smoke result = (%#v, %q), calls=%d", outcome, output.String(), providerCalls)
	}
}

func TestSmokeTestCommandDeliversReportWithOperationError(t *testing.T) {
	sentinel := errors.New("native smoke operation failed")
	warning := commandkit.MessageDiagnostic(commandkit.SeverityWarning, "independent smoke warning")
	app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{
		SmokeTest: func(context.Context, SmokeRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
			return commandkit.Acquired[*promptkit.SmokeTestReport]{
				Value:       &promptkit.SmokeTestReport{Kind: promptkit.SmokeTestAuthentication, Completed: true},
				Diagnostics: []commandkit.Diagnostic{warning},
				Explanation: commandkit.Explanation{Details: []commandkit.Detail{func(writer io.Writer) error {
					_, err := io.WriteString(writer, "Native authentication context\n")
					return err
				}}},
			}, sentinel
		},
	}))
	var output, diagnosticOutput bytes.Buffer
	outcome := app.Execute(context.Background(), []string{"promptkit", "test", "authentication", "--verbose"}, commandkit.Streams{Out: &output, Err: &diagnosticOutput})
	if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, sentinel) || output.Len() == 0 {
		t.Fatalf("report-plus-error outcome = (%#v, %q)", outcome, output.String())
	}
	warningAt := strings.Index(diagnosticOutput.String(), "independent smoke warning")
	explanationAt := strings.Index(diagnosticOutput.String(), "Native authentication context")
	if warningAt < 0 || explanationAt < 0 || warningAt >= explanationAt || strings.Contains(diagnosticOutput.String(), sentinel.Error()) {
		t.Fatalf("smoke warning and native terminal context were not separate and ordered: %q", diagnosticOutput.String())
	}

	output.Reset()
	diagnosticOutput.Reset()
	outcome = app.Execute(context.Background(), []string{"promptkit", "test", "authentication", "--quiet"}, commandkit.Streams{Out: &output, Err: &diagnosticOutput})
	if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, sentinel) || output.Len() != 0 || !strings.Contains(diagnosticOutput.String(), "Native authentication context") || strings.Contains(diagnosticOutput.String(), "independent smoke warning") {
		t.Fatalf("quiet report-plus-error = (%#v, out=%q, err=%q)", outcome, output.String(), diagnosticOutput.String())
	}

	app = newPromptTestApp(t, Command(Suite[struct{}, struct{}]{
		SmokeTest: func(context.Context, SmokeRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
			return commandkit.Acquired[*promptkit.SmokeTestReport]{}, nil
		},
	}))
	output.Reset()
	outcome = app.Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: &output, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeOperational || outcome.Cause == nil || output.Len() != 0 {
		t.Fatalf("empty acquisition outcome = (%#v, %q)", outcome, output.String())
	}

	app = newPromptTestApp(t, Command(Suite[struct{}, struct{}]{
		SmokeTest: func(context.Context, SmokeRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
			return commandkit.Acquired[*promptkit.SmokeTestReport]{}, promptkit.ErrInvalidRequest
		},
	}))
	outcome = app.Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, promptkit.ErrInvalidRequest) {
		t.Fatalf("native semantic error classification = %#v", outcome)
	}
}

type smokeShortWriter struct{}

func (smokeShortWriter) Write(value []byte) (int, error) {
	if len(value) == 0 {
		return 0, nil
	}
	return len(value) - 1, nil
}

func TestSmokeTestCommandRetainsShortReportWrite(t *testing.T) {
	app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{
		SmokeTest: func(context.Context, SmokeRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
			return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: promptkit.SmokeTestAuthentication}}, nil
		},
	}))
	outcome := app.Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: smokeShortWriter{}, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Presentation.Output.WriteError, io.ErrShortWrite) || outcome.Presentation.Output.State != commandkit.DeliveryIncomplete {
		t.Fatalf("short report delivery = %#v", outcome)
	}
}