commandkit/integrations/ckprompt/smoke_aggregate_test.go

package ckprompt

import (
	"bytes"
	"context"
	"errors"
	"io"
	"reflect"
	"testing"

	commandkit "gitea.maximumdirect.net/eric/CommandKit"
	"gitea.maximumdirect.net/eric/CommandKit/integrations/ckconfig"
	promptkit "gitea.maximumdirect.net/eric/promptkit"
	"go.yaml.in/yaml/v4"
)

func TestSmokeAggregateCoordinatesConcreteRequestsAndRetainsErrors(t *testing.T) {
	type input struct{ identity *int }
	type state struct{ identity *int }

	inputIdentity, stateIdentity := 1, 2
	wantKinds := []promptkit.SmokeTestKind{
		promptkit.SmokeTestReachability,
		promptkit.SmokeTestAuthentication,
		promptkit.SmokeTestTextInference,
		promptkit.SmokeTestStructuredOutput,
	}
	nativeErr := errors.New("authentication smoke failed")
	newInputCalls, checkCalls, newStateCalls := 0, 0, 0
	var gotKinds []promptkit.SmokeTestKind
	var firstInput *input
	var sharedState *state
	suite := Suite[*input, *state]{
		NewInput: func() *input {
			newInputCalls++
			return &input{identity: &inputIdentity}
		},
		NewState: func() *state {
			newStateCalls++
			return &state{identity: &stateIdentity}
		},
		CheckInput: func(_ ckconfig.Selection, value *input) error {
			checkCalls++
			firstInput = value
			return nil
		},
		SmokeTest: func(_ context.Context, request SmokeRequest, value *input, shared *state) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
			gotKinds = append(gotKinds, request.Request.Kind)
			if firstInput != value || value.identity != &inputIdentity || shared.identity != &stateIdentity {
				t.Errorf("input/state were not shared: input=%p state=%#v", value, shared)
			}
			if sharedState == nil {
				sharedState = shared
			} else if sharedState != shared {
				t.Errorf("state pointer changed between smoke tests: first=%p current=%p", sharedState, shared)
			}
			if request.Request.Execution == nil || request.Request.Execution.Model != "base-model" || request.Request.Execution.Temperature == nil || *request.Request.Execution.Temperature != 0.25 || request.Request.TransientRetries == nil || *request.Request.TransientRetries != 3 {
				t.Errorf("seed for %s = %#v", request.Request.Kind, request.Request)
			}
			if len(gotKinds) == 1 {
				request.Request.Execution.Model = "builder mutation"
				*request.Request.Execution.Temperature = 0.9
				*request.Request.TransientRetries = 11
			}
			result := &promptkit.SmokeTestReport{Kind: promptkit.SmokeTestAuthentication, Supported: true, Completed: true}
			if request.Request.Kind == promptkit.SmokeTestAuthentication {
				return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: result}, nativeErr
			}
			return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: result}, nil
		},
	}
	app := newPromptTestApp(t, Command(suite))
	var output bytes.Buffer
	outcome := app.Execute(context.Background(), []string{
		"promptkit", "test", "all", "--model=base-model", "--temperature=0.25", "--transient-retries=3",
	}, commandkit.Streams{Out: &output, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, nativeErr) {
		t.Fatalf("aggregate outcome = %#v", outcome)
	}
	if newInputCalls != 1 || checkCalls != 1 || newStateCalls != 1 || !reflect.DeepEqual(gotKinds, wantKinds) {
		t.Fatalf("invocation calls = input %d, check %d, state %d, kinds %#v", newInputCalls, checkCalls, newStateCalls, gotKinds)
	}

	var document struct {
		Tests []struct {
			Kind   string `yaml:"kind"`
			Report *struct {
				Kind string `yaml:"kind"`
			} `yaml:"report"`
			Error string `yaml:"error"`
		} `yaml:"tests"`
	}
	if err := yaml.Unmarshal(output.Bytes(), &document); err != nil {
		t.Fatalf("decode aggregate report %q: %v", output.String(), err)
	}
	if len(document.Tests) != len(wantKinds) {
		t.Fatalf("aggregate entries = %#v", document.Tests)
	}
	for index, test := range document.Tests {
		if test.Kind != string(wantKinds[index]) || test.Report == nil || test.Report.Kind != string(promptkit.SmokeTestAuthentication) {
			t.Errorf("aggregate entry %d = %#v", index, test)
		}
		if index == 1 && test.Error != nativeErr.Error() {
			t.Errorf("authentication entry error = %q", test.Error)
		}
		if index != 1 && test.Error != "" {
			t.Errorf("unexpected entry error at %d: %q", index, test.Error)
		}
	}
}

func TestSmokeAggregateCancellationRetainsPartialResults(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	t.Cleanup(cancel)
	operationErr := errors.New("second smoke operation failed")
	var gotKinds []promptkit.SmokeTestKind
	app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{
		SmokeTest: func(_ context.Context, request SmokeRequest, _ struct{}, _ struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
			gotKinds = append(gotKinds, request.Request.Kind)
			if len(gotKinds) == 2 {
				cancel()
				return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind}}, operationErr
			}
			return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind}}, nil
		},
	}))
	var output bytes.Buffer
	outcome := app.Execute(ctx, []string{"promptkit", "test", "all"}, commandkit.Streams{Out: &output, Err: io.Discard})
	if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, operationErr) || !errors.Is(outcome.Cause, context.Canceled) {
		t.Fatalf("partial aggregate outcome = %#v", outcome)
	}
	if !reflect.DeepEqual(gotKinds, []promptkit.SmokeTestKind{promptkit.SmokeTestReachability, promptkit.SmokeTestAuthentication}) {
		t.Fatalf("callbacks after cancellation = %#v", gotKinds)
	}
	var document struct {
		Tests []struct {
			Kind string `yaml:"kind"`
		} `yaml:"tests"`
	}
	if err := yaml.Unmarshal(output.Bytes(), &document); err != nil {
		t.Fatalf("decode partial aggregate report %q: %v", output.String(), err)
	}
	if len(document.Tests) != 2 || document.Tests[0].Kind != string(promptkit.SmokeTestReachability) || document.Tests[1].Kind != string(promptkit.SmokeTestAuthentication) {
		t.Fatalf("partial aggregate entries = %#v", document.Tests)
	}
}