commandkit/integrations/ckprompt/smoke_aggregate_test.go

package ckprompt

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"io"
	"reflect"
	"strings"
	"testing"

	"go.maximumdirect.net/ckconfig"
	commandkit "go.maximumdirect.net/commandkit"
	promptkit "go.maximumdirect.net/promptkit"
	"go.yaml.in/yaml/v4"
)

func TestSmokeAggregateCoordinatesConcreteRequestsAndRetainsErrors(t *testing.T) {
	type input struct{ identity *int }
	type state struct{ identity *int }

	inputIdentity, stateIdentity := 1, 2
	wantKinds := []promptkit.SmokeTestKind{
		promptkit.SmokeTestReachability,
		promptkit.SmokeTestAuthentication,
		promptkit.SmokeTestTextInference,
		promptkit.SmokeTestStructuredOutput,
	}
	nativeErr := errors.New("authentication smoke failed")
	newInputCalls, checkCalls, newStateCalls := 0, 0, 0
	var gotKinds []promptkit.SmokeTestKind
	var firstInput *input
	var sharedState *state
	suite := Suite[*input, *state]{
		NewInput: func() *input {
			newInputCalls++
			return &input{identity: &inputIdentity}
		},
		NewState: func() *state {
			newStateCalls++
			return &state{identity: &stateIdentity}
		},
		CheckInput: func(_ ckconfig.Selection, value *input) error {
			checkCalls++
			firstInput = value
			return nil
		},
		SmokeTest: func(_ context.Context, request SmokeRequest, value *input, shared *state) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
			gotKinds = append(gotKinds, request.Request.Kind)
			if firstInput != value || value.identity != &inputIdentity || shared.identity != &stateIdentity {
				t.Errorf("input/state were not shared: input=%p state=%#v", value, shared)
			}
			if sharedState == nil {
				sharedState = shared
			} else if sharedState != shared {
				t.Errorf("state pointer changed between smoke tests: first=%p current=%p", sharedState, shared)
			}
			if request.Request.Execution == nil || request.Request.Execution.Model != "base-model" || request.Request.Execution.Temperature == nil || *request.Request.Execution.Temperature != 0.25 || request.Request.TransientRetries == nil || *request.Request.TransientRetries != 3 {
				t.Errorf("seed for %s = %#v", request.Request.Kind, request.Request)
			}
			if len(gotKinds) == 1 {
				request.Request.Execution.Model = "builder mutation"
				*request.Request.Execution.Temperature = 0.9
				*request.Request.TransientRetries = 11
			}
			result := &promptkit.SmokeTestReport{Kind: promptkit.SmokeTestAuthentication, Supported: true, Completed: true}
			if request.Request.Kind == promptkit.SmokeTestAuthentication {
				return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: result}, nativeErr
			}
			return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: result}, nil
		},
	}
	app := newPromptTestApp(t, Command(suite))
	var output bytes.Buffer
	outcome := app.Execute(context.Background(), []string{
		"promptkit", "test", "all", "--llm-model=base-model", "--llm-temperature=0.25", "--llm-transient-retries=3",
	}, commandkit.Streams{Out: &output, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, nativeErr) {
		t.Fatalf("aggregate outcome = %#v", outcome)
	}
	if newInputCalls != 1 || checkCalls != 1 || newStateCalls != 1 || !reflect.DeepEqual(gotKinds, wantKinds) {
		t.Fatalf("invocation calls = input %d, check %d, state %d, kinds %#v", newInputCalls, checkCalls, newStateCalls, gotKinds)
	}

	var document struct {
		Tests []struct {
			Kind   string `yaml:"kind"`
			Report *struct {
				Kind string `yaml:"kind"`
			} `yaml:"report"`
			Error string `yaml:"error"`
		} `yaml:"tests"`
	}
	if err := yaml.Unmarshal(output.Bytes(), &document); err != nil {
		t.Fatalf("decode aggregate report %q: %v", output.String(), err)
	}
	if len(document.Tests) != len(wantKinds) {
		t.Fatalf("aggregate entries = %#v", document.Tests)
	}
	for index, test := range document.Tests {
		if test.Kind != string(wantKinds[index]) || test.Report == nil || test.Report.Kind != string(promptkit.SmokeTestAuthentication) {
			t.Errorf("aggregate entry %d = %#v", index, test)
		}
		if index == 1 && test.Error != nativeErr.Error() {
			t.Errorf("authentication entry error = %q", test.Error)
		}
		if index != 1 && test.Error != "" {
			t.Errorf("unexpected entry error at %d: %q", index, test.Error)
		}
	}
}

type cancelProgressWriter struct {
	bytes.Buffer
	cancel context.CancelFunc
}

func (writer *cancelProgressWriter) Write(p []byte) (int, error) {
	writer.cancel()
	return writer.Buffer.Write(p)
}

func TestSmokeAggregateCancellationDuringProgress(t *testing.T) {
	providerErr := errors.New("provider failed")
	for _, test := range []struct {
		name     string
		err      error
		cancelAt int
	}{
		{name: "partial success", cancelAt: 1},
		{name: "partial provider failure", err: providerErr, cancelAt: 1},
		{name: "last completed test", cancelAt: 4},
	} {
		t.Run(test.name, func(t *testing.T) {
			ctx, cancel := context.WithCancel(context.Background())
			defer cancel()
			var kinds []promptkit.SmokeTestKind
			app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{
				SmokeTest: func(_ context.Context, request SmokeRequest, _, _ struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
					kinds = append(kinds, request.Request.Kind)
					acquired := commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind, Completed: true}}
					if len(kinds) == test.cancelAt {
						acquired.Diagnostics = []commandkit.Diagnostic{{Severity: commandkit.SeverityProgress, Write: func(w io.Writer) error { _, err := io.WriteString(w, "progress evidence\n"); return err }}}
						acquired.Explanation = commandkit.Explanation{Message: "provider explanation"}
						return acquired, test.err
					}
					return acquired, nil
				},
			}))
			var output bytes.Buffer
			diagnostics := &cancelProgressWriter{cancel: cancel}
			outcome := app.Execute(ctx, []string{"promptkit", "test", "all", "--progress", "--format=json"}, commandkit.Streams{Out: &output, Err: diagnostics})
			if len(kinds) != test.cancelAt || !strings.Contains(diagnostics.String(), "progress evidence") {
				t.Fatalf("kinds=%v diagnostics=%q", kinds, diagnostics.String())
			}
			if test.cancelAt < 4 {
				if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, context.Canceled) || (test.err != nil && (!errors.Is(outcome.Cause, test.err) || !strings.Contains(diagnostics.String(), "provider explanation"))) {
					t.Fatalf("partial outcome=%#v diagnostics=%q", outcome, diagnostics.String())
				}
			} else if outcome.Kind != commandkit.OutcomeSuccess || outcome.Cause != nil {
				t.Fatalf("completed outcome=%#v", outcome)
			}
			var report struct {
				Tests []struct {
					Kind   string `json:"kind"`
					Report *struct {
						Completed bool `json:"completed"`
					} `json:"report"`
				} `json:"tests"`
			}
			if err := json.Unmarshal(output.Bytes(), &report); err != nil {
				t.Fatal(err)
			}
			if len(report.Tests) != len(kinds) {
				t.Fatalf("report=%s", output.String())
			}
			for i, entry := range report.Tests {
				if entry.Kind != string(kinds[i]) || entry.Report == nil || !entry.Report.Completed {
					t.Fatalf("report entry=%#v", entry)
				}
			}
		})
	}
}

func TestSmokeAggregateCancellationRetainsPartialResults(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	t.Cleanup(cancel)
	operationErr := errors.New("second smoke operation failed")
	var gotKinds []promptkit.SmokeTestKind
	app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{
		SmokeTest: func(_ context.Context, request SmokeRequest, _ struct{}, _ struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
			gotKinds = append(gotKinds, request.Request.Kind)
			if len(gotKinds) == 2 {
				cancel()
				return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind}}, operationErr
			}
			return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind}}, nil
		},
	}))
	var output bytes.Buffer
	outcome := app.Execute(ctx, []string{"promptkit", "test", "all"}, commandkit.Streams{Out: &output, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, operationErr) || !errors.Is(outcome.Cause, context.Canceled) {
		t.Fatalf("partial aggregate outcome = %#v", outcome)
	}
	if !reflect.DeepEqual(gotKinds, []promptkit.SmokeTestKind{promptkit.SmokeTestReachability, promptkit.SmokeTestAuthentication}) {
		t.Fatalf("callbacks after cancellation = %#v", gotKinds)
	}
	var document struct {
		Tests []struct {
			Kind string `yaml:"kind"`
		} `yaml:"tests"`
	}
	if err := yaml.Unmarshal(output.Bytes(), &document); err != nil {
		t.Fatalf("decode partial aggregate report %q: %v", output.String(), err)
	}
	if len(document.Tests) != 2 || document.Tests[0].Kind != string(promptkit.SmokeTestReachability) || document.Tests[1].Kind != string(promptkit.SmokeTestAuthentication) {
		t.Fatalf("partial aggregate entries = %#v", document.Tests)
	}
}

func TestSmokeAggregateIsolatesProviderParameters(t *testing.T) {
	calls := 0
	app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{
		SmokeTest: func(_ context.Context, request SmokeRequest, _, _ struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
			calls++
			params := request.Request.Execution.ExtraParams
			value := params["nested"].(map[string]any)["array"].([]any)[0].(map[string]any)
			if value["number"] != int64(1) {
				t.Errorf("call %d: %#v", calls, value)
			}
			value["number"] = int64(9)
			return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind}}, nil
		},
	}))
	outcome := app.Execute(context.Background(), []string{"promptkit", "test", "all", `--llm-extra-params={"nested":{"array":[{"number":1}]}}`}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeSuccess || calls != 4 {
		t.Fatalf("outcome=%#v calls=%d", outcome, calls)
	}
}

func TestCopyProviderParamsPreservesEmptyAndNilValues(t *testing.T) {
	if copyProviderParams(nil) != nil {
		t.Fatal("nil object became non-nil")
	}
	copied := copyProviderParams(map[string]any{"emptyObject": map[string]any{}, "nilObject": map[string]any(nil), "emptyArray": []any{}, "nilArray": []any(nil), "null": nil, "decimal": float64(1.5)})
	if copied["emptyObject"].(map[string]any) == nil || copied["nilObject"].(map[string]any) != nil || copied["emptyArray"].([]any) == nil || copied["nilArray"].([]any) != nil || copied["null"] != nil || copied["decimal"] != float64(1.5) {
		t.Fatalf("copied = %#v", copied)
	}
}