package ckprompt import ( "bytes" "context" "encoding/json" "errors" "io" "reflect" "strings" "testing" "go.maximumdirect.net/ckconfig" commandkit "go.maximumdirect.net/commandkit" promptkit "go.maximumdirect.net/promptkit" "go.yaml.in/yaml/v4" ) func TestSmokeAggregateCoordinatesConcreteRequestsAndRetainsErrors(t *testing.T) { type input struct{ identity *int } type state struct{ identity *int } inputIdentity, stateIdentity := 1, 2 wantKinds := []promptkit.SmokeTestKind{ promptkit.SmokeTestReachability, promptkit.SmokeTestAuthentication, promptkit.SmokeTestTextInference, promptkit.SmokeTestStructuredOutput, } nativeErr := errors.New("authentication smoke failed") newInputCalls, checkCalls, newStateCalls := 0, 0, 0 var gotKinds []promptkit.SmokeTestKind var firstInput *input var sharedState *state suite := Suite[*input, *state]{ NewInput: func() *input { newInputCalls++ return &input{identity: &inputIdentity} }, NewState: func() *state { newStateCalls++ return &state{identity: &stateIdentity} }, CheckInput: func(_ ckconfig.Selection, value *input) error { checkCalls++ firstInput = value return nil }, SmokeTest: func(_ context.Context, request SmokeRequest, value *input, shared *state) (commandkit.Acquired[*promptkit.SmokeTestReport], error) { gotKinds = append(gotKinds, request.Request.Kind) if firstInput != value || value.identity != &inputIdentity || shared.identity != &stateIdentity { t.Errorf("input/state were not shared: input=%p state=%#v", value, shared) } if sharedState == nil { sharedState = shared } else if sharedState != shared { t.Errorf("state pointer changed between smoke tests: first=%p current=%p", sharedState, shared) } if request.Request.Execution == nil || request.Request.Execution.Model != "base-model" || request.Request.Execution.Temperature == nil || *request.Request.Execution.Temperature != 0.25 || request.Request.TransientRetries == nil || *request.Request.TransientRetries != 3 { t.Errorf("seed for %s = %#v", request.Request.Kind, request.Request) } if len(gotKinds) == 1 { request.Request.Execution.Model = "builder mutation" *request.Request.Execution.Temperature = 0.9 *request.Request.TransientRetries = 11 } result := &promptkit.SmokeTestReport{Kind: promptkit.SmokeTestAuthentication, Supported: true, Completed: true} if request.Request.Kind == promptkit.SmokeTestAuthentication { return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: result}, nativeErr } return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: result}, nil }, } app := newPromptTestApp(t, Command(suite)) var output bytes.Buffer outcome := app.Execute(context.Background(), []string{ "promptkit", "test", "all", "--llm-model=base-model", "--llm-temperature=0.25", "--llm-transient-retries=3", }, commandkit.Streams{Out: &output, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, nativeErr) { t.Fatalf("aggregate outcome = %#v", outcome) } if newInputCalls != 1 || checkCalls != 1 || newStateCalls != 1 || !reflect.DeepEqual(gotKinds, wantKinds) { t.Fatalf("invocation calls = input %d, check %d, state %d, kinds %#v", newInputCalls, checkCalls, newStateCalls, gotKinds) } var document struct { Tests []struct { Kind string `yaml:"kind"` Report *struct { Kind string `yaml:"kind"` } `yaml:"report"` Error string `yaml:"error"` } `yaml:"tests"` } if err := yaml.Unmarshal(output.Bytes(), &document); err != nil { t.Fatalf("decode aggregate report %q: %v", output.String(), err) } if len(document.Tests) != len(wantKinds) { t.Fatalf("aggregate entries = %#v", document.Tests) } for index, test := range document.Tests { if test.Kind != string(wantKinds[index]) || test.Report == nil || test.Report.Kind != string(promptkit.SmokeTestAuthentication) { t.Errorf("aggregate entry %d = %#v", index, test) } if index == 1 && test.Error != nativeErr.Error() { t.Errorf("authentication entry error = %q", test.Error) } if index != 1 && test.Error != "" { t.Errorf("unexpected entry error at %d: %q", index, test.Error) } } } type cancelProgressWriter struct { bytes.Buffer cancel context.CancelFunc } func (writer *cancelProgressWriter) Write(p []byte) (int, error) { writer.cancel() return writer.Buffer.Write(p) } func TestSmokeAggregateCancellationDuringProgress(t *testing.T) { providerErr := errors.New("provider failed") for _, test := range []struct { name string err error cancelAt int }{ {name: "partial success", cancelAt: 1}, {name: "partial provider failure", err: providerErr, cancelAt: 1}, {name: "last completed test", cancelAt: 4}, } { t.Run(test.name, func(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() var kinds []promptkit.SmokeTestKind app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ SmokeTest: func(_ context.Context, request SmokeRequest, _, _ struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) { kinds = append(kinds, request.Request.Kind) acquired := commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind, Completed: true}} if len(kinds) == test.cancelAt { acquired.Diagnostics = []commandkit.Diagnostic{{Severity: commandkit.SeverityProgress, Write: func(w io.Writer) error { _, err := io.WriteString(w, "progress evidence\n"); return err }}} acquired.Explanation = commandkit.Explanation{Message: "provider explanation"} return acquired, test.err } return acquired, nil }, })) var output bytes.Buffer diagnostics := &cancelProgressWriter{cancel: cancel} outcome := app.Execute(ctx, []string{"promptkit", "test", "all", "--progress", "--format=json"}, commandkit.Streams{Out: &output, Err: diagnostics}) if len(kinds) != test.cancelAt || !strings.Contains(diagnostics.String(), "progress evidence") { t.Fatalf("kinds=%v diagnostics=%q", kinds, diagnostics.String()) } if test.cancelAt < 4 { if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, context.Canceled) || (test.err != nil && (!errors.Is(outcome.Cause, test.err) || !strings.Contains(diagnostics.String(), "provider explanation"))) { t.Fatalf("partial outcome=%#v diagnostics=%q", outcome, diagnostics.String()) } } else if outcome.Kind != commandkit.OutcomeSuccess || outcome.Cause != nil { t.Fatalf("completed outcome=%#v", outcome) } var report struct { Tests []struct { Kind string `json:"kind"` Report *struct { Completed bool `json:"completed"` } `json:"report"` } `json:"tests"` } if err := json.Unmarshal(output.Bytes(), &report); err != nil { t.Fatal(err) } if len(report.Tests) != len(kinds) { t.Fatalf("report=%s", output.String()) } for i, entry := range report.Tests { if entry.Kind != string(kinds[i]) || entry.Report == nil || !entry.Report.Completed { t.Fatalf("report entry=%#v", entry) } } }) } } func TestSmokeAggregateCancellationRetainsPartialResults(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) operationErr := errors.New("second smoke operation failed") var gotKinds []promptkit.SmokeTestKind app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ SmokeTest: func(_ context.Context, request SmokeRequest, _ struct{}, _ struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) { gotKinds = append(gotKinds, request.Request.Kind) if len(gotKinds) == 2 { cancel() return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind}}, operationErr } return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind}}, nil }, })) var output bytes.Buffer outcome := app.Execute(ctx, []string{"promptkit", "test", "all"}, commandkit.Streams{Out: &output, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, operationErr) || !errors.Is(outcome.Cause, context.Canceled) { t.Fatalf("partial aggregate outcome = %#v", outcome) } if !reflect.DeepEqual(gotKinds, []promptkit.SmokeTestKind{promptkit.SmokeTestReachability, promptkit.SmokeTestAuthentication}) { t.Fatalf("callbacks after cancellation = %#v", gotKinds) } var document struct { Tests []struct { Kind string `yaml:"kind"` } `yaml:"tests"` } if err := yaml.Unmarshal(output.Bytes(), &document); err != nil { t.Fatalf("decode partial aggregate report %q: %v", output.String(), err) } if len(document.Tests) != 2 || document.Tests[0].Kind != string(promptkit.SmokeTestReachability) || document.Tests[1].Kind != string(promptkit.SmokeTestAuthentication) { t.Fatalf("partial aggregate entries = %#v", document.Tests) } } func TestSmokeAggregateIsolatesProviderParameters(t *testing.T) { calls := 0 app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ SmokeTest: func(_ context.Context, request SmokeRequest, _, _ struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) { calls++ params := request.Request.Execution.ExtraParams value := params["nested"].(map[string]any)["array"].([]any)[0].(map[string]any) if value["number"] != int64(1) { t.Errorf("call %d: %#v", calls, value) } value["number"] = int64(9) return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind}}, nil }, })) outcome := app.Execute(context.Background(), []string{"promptkit", "test", "all", `--llm-extra-params={"nested":{"array":[{"number":1}]}}`}, commandkit.Streams{Out: io.Discard, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess || calls != 4 { t.Fatalf("outcome=%#v calls=%d", outcome, calls) } } func TestCopyProviderParamsPreservesEmptyAndNilValues(t *testing.T) { if copyProviderParams(nil) != nil { t.Fatal("nil object became non-nil") } copied := copyProviderParams(map[string]any{"emptyObject": map[string]any{}, "nilObject": map[string]any(nil), "emptyArray": []any{}, "nilArray": []any(nil), "null": nil, "decimal": float64(1.5)}) if copied["emptyObject"].(map[string]any) == nil || copied["nilObject"].(map[string]any) != nil || copied["emptyArray"].([]any) == nil || copied["nilArray"].([]any) != nil || copied["null"] != nil || copied["decimal"] != float64(1.5) { t.Fatalf("copied = %#v", copied) } }