commandkit/integrations/ckprompt/smoke_report_test.go

package ckprompt

import (
	"errors"
	"testing"
	"time"

	promptkit "gitea.maximumdirect.net/eric/promptkit"
	"go.yaml.in/yaml/v4"
)

func TestSmokeReportYAMLProjectsNativeFieldsAndPresence(t *testing.T) {
	applicable := false
	credentialsSupplied := false
	endpointReachable := true
	textReturned := false
	zero := time.Duration(0)
	retryDelay := 1500 * time.Millisecond
	version := ""
	httpStatus := 201
	started := time.Date(2026, time.October, 17, 12, 30, 0, 0, time.UTC)
	report := &promptkit.SmokeTestReport{
		Kind:       promptkit.SmokeTestReachability,
		Supported:  true,
		Applicable: &applicable,
		Completed:  false,
		Reason:     promptkit.SmokeTestReasonHTTPDiagnosticsUnavailable,
		Target: promptkit.SmokeTestTarget{
			BackendID:              "local",
			ProfileID:              "default",
			PromptID:               "team/briefing",
			PromptVersion:          &version,
			DefaultSourceBackendID: "fallback",
			SelectionSource:        "prompt",
			Endpoint:               "https://example.invalid/v1",
			EffectiveSettings: promptkit.ExecutionTarget{
				BackendID:       "local",
				Endpoint:        "https://example.invalid/v1",
				Model:           "example-model",
				Temperature:     0,
				MaxTokens:       0,
				TopP:            0,
				TimeoutSeconds:  0,
				ServiceTier:     "",
				ReasoningEffort: "",
				APIKeyEnv:       "PROMPTKIT_KEY",
				ExtraParams:     map[string]any{"token": "caller-value", "nested": map[string]any{"count": 0}},
			},
			SettingsPresence: promptkit.ExecutionTargetPresence{Temperature: true, MaxTokens: false, TopP: true, TimeoutSeconds: false},
		},
		Facts: promptkit.SmokeTestFacts{
			CredentialsSupplied:       &credentialsSupplied,
			EndpointReachable:         &endpointReachable,
			CredentialsAccepted:       nil,
			ResponseReceived:          &textReturned,
			GenerationCompleted:       &endpointReachable,
			TextReturned:              &textReturned,
			ExpectedTextMatched:       &textReturned,
			StructuredOutputRequested: &endpointReachable,
			ValidJSON:                 &textReturned,
			SchemaMatched:             nil,
		},
		Timing: promptkit.SmokeTestTiming{
			StartedAt:          started,
			Duration:           zero,
			CapacityWait:       &zero,
			RequestDuration:    nil,
			RetryDelay:         &retryDelay,
			EvaluationDuration: nil,
		},
		Attempts: []promptkit.SmokeTestAttempt{{
			Dispatched:      true,
			InitialEndpoint: "https://initial.example/v1",
			Endpoint:        "https://final.example/v1",
			HTTPStatus:      &httpStatus,
			Facts:           promptkit.SmokeTestFacts{EndpointReachable: &endpointReachable},
			Timing:          promptkit.SmokeTestTiming{Duration: retryDelay},
			Failure:         promptkit.ExecutionFailureNone,
			Usage:           promptkit.ObservedTokenUsage{PromptTokens: promptkit.ObservedTokenCount{Value: 2, Known: true, Complete: true}},
		}},
		Usage: promptkit.ObservedTokenUsage{
			PromptTokens:     promptkit.ObservedTokenCount{Value: 0, Known: true, Complete: true},
			CompletionTokens: promptkit.ObservedTokenCount{Value: 0, Known: false, Complete: false},
			TotalTokens:      promptkit.ObservedTokenCount{Value: 7, Known: true, Complete: false},
		},
		Failure: promptkit.ExecutionFailureTransport,
	}

	encoded, err := encodeSmokeReport(report)
	if err != nil {
		t.Fatal(err)
	}
	var document map[string]any
	if err := yaml.Unmarshal(encoded, &document); err != nil {
		t.Fatalf("decode smoke report: %v\n%s", err, encoded)
	}
	if document["kind"] != "reachability" || document["supported"] != true || document["applicable"] != false || document["completed"] != false || document["reason"] != "http_diagnostics_unavailable" || document["failure"] != "transport" {
		t.Fatalf("report scalar values = %#v", document)
	}
	target := document["target"].(map[string]any)
	if target["prompt_id"] != "team/briefing" || target["version"] != "" || target["selection_source"] != "prompt" {
		t.Fatalf("target values = %#v", target)
	}
	settings := target["effective_settings"].(map[string]any)
	if settings["model"] != "example-model" || settings["temperature"] != 0 || settings["api_key_env"] != "PROMPTKIT_KEY" {
		t.Fatalf("effective settings = %#v", settings)
	}
	extra := settings["extra_params"].(map[string]any)
	if extra["token"] != "caller-value" {
		t.Fatalf("extra parameters were not retained: %#v", extra)
	}
	presence := target["settings_presence"].(map[string]any)
	if presence["temperature"] != true || presence["max_tokens"] != false || presence["top_p"] != true || presence["timeout_seconds"] != false {
		t.Fatalf("setting presence = %#v", presence)
	}
	facts := document["facts"].(map[string]any)
	if facts["credentials_supplied"] != false || facts["endpoint_reachable"] != true || facts["credentials_accepted"] != nil || facts["response_received"] != false || facts["generation_completed"] != true || facts["text_returned"] != false || facts["expected_text_matched"] != false || facts["structured_output_requested"] != true || facts["valid_json"] != false || facts["schema_matched"] != nil {
		t.Fatalf("nullable facts = %#v", facts)
	}
	timing := document["timing"].(map[string]any)
	if timing["started_at"] == nil || timing["duration"] != "0s" || timing["capacity_wait"] != "0s" || timing["request_duration"] != nil || timing["retry_delay"] != "1.5s" || timing["evaluation_duration"] != nil {
		t.Fatalf("timing values = %#v", timing)
	}
	attempts, ok := document["attempts"].([]any)
	if !ok || len(attempts) != 1 {
		t.Fatalf("attempts = %#v", document["attempts"])
	}
	attempt := attempts[0].(map[string]any)
	if attempt["dispatched"] != true || attempt["initial_endpoint"] != "https://initial.example/v1" || attempt["endpoint"] != "https://final.example/v1" || attempt["http_status"] != 201 {
		t.Fatalf("attempt fields = %#v", attempt)
	}
	if attempt["timing"].(map[string]any)["duration"] != "1.5s" {
		t.Fatalf("attempt timing = %#v", attempt["timing"])
	}
	usage := document["usage"].(map[string]any)
	promptTokens := usage["prompt_tokens"].(map[string]any)
	unknownTokens := usage["completion_tokens"].(map[string]any)
	if promptTokens["value"] != 0 || promptTokens["known"] != true || promptTokens["complete"] != true || unknownTokens["known"] != false || unknownTokens["complete"] != false {
		t.Fatalf("presence-aware usage = %#v", usage)
	}
	if containsSmokeKey(document, "request") || containsSmokeKey(document, "api_key") {
		t.Fatalf("serialized request or direct API key: %#v", document)
	}
}

func TestSmokeReportYAMLEmitsNilAttemptsAsEmptySequence(t *testing.T) {
	encoded, err := encodeSmokeReport(&promptkit.SmokeTestReport{Attempts: nil})
	if err != nil {
		t.Fatal(err)
	}
	var document map[string]any
	if err := yaml.Unmarshal(encoded, &document); err != nil {
		t.Fatal(err)
	}
	if attempts, ok := document["attempts"].([]any); !ok || len(attempts) != 0 {
		t.Fatalf("nil native attempts = %#v", document["attempts"])
	}
}

func TestSmokeAggregateYAMLRetainsOrderErrorAndNativeReportKind(t *testing.T) {
	operationErr := errors.New("operation failed")
	report := &promptkit.SmokeTestReport{Kind: promptkit.SmokeTestReachability, Attempts: []promptkit.SmokeTestAttempt{}}
	encoded, err := encodeSmokeResults([]smokeResult{
		{Kind: promptkit.SmokeTestTextInference, Report: report, Err: operationErr},
		{Kind: promptkit.SmokeTestAuthentication},
		{Kind: promptkit.SmokeTestStructuredOutput, Report: report, Err: errors.New("")},
	})
	if err != nil {
		t.Fatal(err)
	}
	var document map[string]any
	if err := yaml.Unmarshal(encoded, &document); err != nil {
		t.Fatalf("decode aggregate report: %v\n%s", err, encoded)
	}
	tests, ok := document["tests"].([]any)
	if !ok || len(tests) != 3 {
		t.Fatalf("aggregate entries = %#v", document)
	}
	first := tests[0].(map[string]any)
	firstReport := first["report"].(map[string]any)
	if first["kind"] != "text_inference" || firstReport["kind"] != "reachability" || first["error"] != "operation failed" {
		t.Fatalf("first aggregate entry = %#v", first)
	}
	if _, exists := firstReport["target"].(map[string]any)["version"]; exists {
		t.Fatalf("absent target version was emitted: %#v", firstReport["target"])
	}
	second := tests[1].(map[string]any)
	if second["kind"] != "authentication" || second["report"] != nil {
		t.Fatalf("second aggregate entry = %#v", second)
	}
	if _, exists := second["error"]; exists {
		t.Fatalf("nil error should be omitted: %#v", second)
	}
	third := tests[2].(map[string]any)
	if message, exists := third["error"]; !exists || message != "" {
		t.Fatalf("empty error message should remain present: %#v", third)
	}
}

func containsSmokeKey(value any, want string) bool {
	switch value := value.(type) {
	case map[string]any:
		for key, nested := range value {
			if key == want || containsSmokeKey(nested, want) {
				return true
			}
		}
	case []any:
		for _, nested := range value {
			if containsSmokeKey(nested, want) {
				return true
			}
		}
	}
	return false
}