217 lines
8.7 KiB
Go
217 lines
8.7 KiB
Go
package ckprompt
|
|
|
|
import (
|
|
"errors"
|
|
"testing"
|
|
"time"
|
|
|
|
promptkit "gitea.maximumdirect.net/eric/promptkit"
|
|
"go.yaml.in/yaml/v4"
|
|
)
|
|
|
|
func TestSmokeReportYAMLProjectsNativeFieldsAndPresence(t *testing.T) {
|
|
applicable := false
|
|
credentialsSupplied := false
|
|
endpointReachable := true
|
|
textReturned := false
|
|
zero := time.Duration(0)
|
|
retryDelay := 1500 * time.Millisecond
|
|
version := ""
|
|
httpStatus := 201
|
|
started := time.Date(2026, time.October, 17, 12, 30, 0, 0, time.UTC)
|
|
report := &promptkit.SmokeTestReport{
|
|
Kind: promptkit.SmokeTestReachability,
|
|
Supported: true,
|
|
Applicable: &applicable,
|
|
Completed: false,
|
|
Reason: promptkit.SmokeTestReasonHTTPDiagnosticsUnavailable,
|
|
Target: promptkit.SmokeTestTarget{
|
|
BackendID: "local",
|
|
ProfileID: "default",
|
|
PromptID: "team/briefing",
|
|
PromptVersion: &version,
|
|
DefaultSourceBackendID: "fallback",
|
|
SelectionSource: "prompt",
|
|
Endpoint: "https://example.invalid/v1",
|
|
EffectiveSettings: promptkit.ExecutionTarget{
|
|
BackendID: "local",
|
|
Endpoint: "https://example.invalid/v1",
|
|
Model: "example-model",
|
|
Temperature: 0,
|
|
MaxTokens: 0,
|
|
TopP: 0,
|
|
TimeoutSeconds: 0,
|
|
ServiceTier: "",
|
|
ReasoningEffort: "",
|
|
APIKeyEnv: "PROMPTKIT_KEY",
|
|
ExtraParams: map[string]any{"token": "caller-value", "nested": map[string]any{"count": 0}},
|
|
},
|
|
SettingsPresence: promptkit.ExecutionTargetPresence{Temperature: true, MaxTokens: false, TopP: true, TimeoutSeconds: false},
|
|
},
|
|
Facts: promptkit.SmokeTestFacts{
|
|
CredentialsSupplied: &credentialsSupplied,
|
|
EndpointReachable: &endpointReachable,
|
|
CredentialsAccepted: nil,
|
|
ResponseReceived: &textReturned,
|
|
GenerationCompleted: &endpointReachable,
|
|
TextReturned: &textReturned,
|
|
ExpectedTextMatched: &textReturned,
|
|
StructuredOutputRequested: &endpointReachable,
|
|
ValidJSON: &textReturned,
|
|
SchemaMatched: nil,
|
|
},
|
|
Timing: promptkit.SmokeTestTiming{
|
|
StartedAt: started,
|
|
Duration: zero,
|
|
CapacityWait: &zero,
|
|
RequestDuration: nil,
|
|
RetryDelay: &retryDelay,
|
|
EvaluationDuration: nil,
|
|
},
|
|
Attempts: []promptkit.SmokeTestAttempt{{
|
|
Dispatched: true,
|
|
InitialEndpoint: "https://initial.example/v1",
|
|
Endpoint: "https://final.example/v1",
|
|
HTTPStatus: &httpStatus,
|
|
Facts: promptkit.SmokeTestFacts{EndpointReachable: &endpointReachable},
|
|
Timing: promptkit.SmokeTestTiming{Duration: retryDelay},
|
|
Failure: promptkit.ExecutionFailureNone,
|
|
Usage: promptkit.ObservedTokenUsage{PromptTokens: promptkit.ObservedTokenCount{Value: 2, Known: true, Complete: true}},
|
|
}},
|
|
Usage: promptkit.ObservedTokenUsage{
|
|
PromptTokens: promptkit.ObservedTokenCount{Value: 0, Known: true, Complete: true},
|
|
CompletionTokens: promptkit.ObservedTokenCount{Value: 0, Known: false, Complete: false},
|
|
TotalTokens: promptkit.ObservedTokenCount{Value: 7, Known: true, Complete: false},
|
|
},
|
|
Failure: promptkit.ExecutionFailureTransport,
|
|
}
|
|
|
|
encoded, err := encodeSmokeReport(report)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var document map[string]any
|
|
if err := yaml.Unmarshal(encoded, &document); err != nil {
|
|
t.Fatalf("decode smoke report: %v\n%s", err, encoded)
|
|
}
|
|
if document["kind"] != "reachability" || document["supported"] != true || document["applicable"] != false || document["completed"] != false || document["reason"] != "http_diagnostics_unavailable" || document["failure"] != "transport" {
|
|
t.Fatalf("report scalar values = %#v", document)
|
|
}
|
|
target := document["target"].(map[string]any)
|
|
if target["prompt_id"] != "team/briefing" || target["version"] != "" || target["selection_source"] != "prompt" {
|
|
t.Fatalf("target values = %#v", target)
|
|
}
|
|
settings := target["effective_settings"].(map[string]any)
|
|
if settings["model"] != "example-model" || settings["temperature"] != 0 || settings["api_key_env"] != "PROMPTKIT_KEY" {
|
|
t.Fatalf("effective settings = %#v", settings)
|
|
}
|
|
extra := settings["extra_params"].(map[string]any)
|
|
if extra["token"] != "caller-value" {
|
|
t.Fatalf("extra parameters were not retained: %#v", extra)
|
|
}
|
|
presence := target["settings_presence"].(map[string]any)
|
|
if presence["temperature"] != true || presence["max_tokens"] != false || presence["top_p"] != true || presence["timeout_seconds"] != false {
|
|
t.Fatalf("setting presence = %#v", presence)
|
|
}
|
|
facts := document["facts"].(map[string]any)
|
|
if facts["credentials_supplied"] != false || facts["endpoint_reachable"] != true || facts["credentials_accepted"] != nil || facts["response_received"] != false || facts["generation_completed"] != true || facts["text_returned"] != false || facts["expected_text_matched"] != false || facts["structured_output_requested"] != true || facts["valid_json"] != false || facts["schema_matched"] != nil {
|
|
t.Fatalf("nullable facts = %#v", facts)
|
|
}
|
|
timing := document["timing"].(map[string]any)
|
|
if timing["started_at"] == nil || timing["duration"] != "0s" || timing["capacity_wait"] != "0s" || timing["request_duration"] != nil || timing["retry_delay"] != "1.5s" || timing["evaluation_duration"] != nil {
|
|
t.Fatalf("timing values = %#v", timing)
|
|
}
|
|
attempts, ok := document["attempts"].([]any)
|
|
if !ok || len(attempts) != 1 {
|
|
t.Fatalf("attempts = %#v", document["attempts"])
|
|
}
|
|
attempt := attempts[0].(map[string]any)
|
|
if attempt["dispatched"] != true || attempt["initial_endpoint"] != "https://initial.example/v1" || attempt["endpoint"] != "https://final.example/v1" || attempt["http_status"] != 201 {
|
|
t.Fatalf("attempt fields = %#v", attempt)
|
|
}
|
|
if attempt["timing"].(map[string]any)["duration"] != "1.5s" {
|
|
t.Fatalf("attempt timing = %#v", attempt["timing"])
|
|
}
|
|
usage := document["usage"].(map[string]any)
|
|
promptTokens := usage["prompt_tokens"].(map[string]any)
|
|
unknownTokens := usage["completion_tokens"].(map[string]any)
|
|
if promptTokens["value"] != 0 || promptTokens["known"] != true || promptTokens["complete"] != true || unknownTokens["known"] != false || unknownTokens["complete"] != false {
|
|
t.Fatalf("presence-aware usage = %#v", usage)
|
|
}
|
|
if containsSmokeKey(document, "request") || containsSmokeKey(document, "api_key") {
|
|
t.Fatalf("serialized request or direct API key: %#v", document)
|
|
}
|
|
}
|
|
|
|
func TestSmokeReportYAMLEmitsNilAttemptsAsEmptySequence(t *testing.T) {
|
|
encoded, err := encodeSmokeReport(&promptkit.SmokeTestReport{Attempts: nil})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var document map[string]any
|
|
if err := yaml.Unmarshal(encoded, &document); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if attempts, ok := document["attempts"].([]any); !ok || len(attempts) != 0 {
|
|
t.Fatalf("nil native attempts = %#v", document["attempts"])
|
|
}
|
|
}
|
|
|
|
func TestSmokeAggregateYAMLRetainsOrderErrorAndNativeReportKind(t *testing.T) {
|
|
operationErr := errors.New("operation failed")
|
|
report := &promptkit.SmokeTestReport{Kind: promptkit.SmokeTestReachability, Attempts: []promptkit.SmokeTestAttempt{}}
|
|
encoded, err := encodeSmokeResults([]smokeResult{
|
|
{Kind: promptkit.SmokeTestTextInference, Report: report, Err: operationErr},
|
|
{Kind: promptkit.SmokeTestAuthentication},
|
|
{Kind: promptkit.SmokeTestStructuredOutput, Report: report, Err: errors.New("")},
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var document map[string]any
|
|
if err := yaml.Unmarshal(encoded, &document); err != nil {
|
|
t.Fatalf("decode aggregate report: %v\n%s", err, encoded)
|
|
}
|
|
tests, ok := document["tests"].([]any)
|
|
if !ok || len(tests) != 3 {
|
|
t.Fatalf("aggregate entries = %#v", document)
|
|
}
|
|
first := tests[0].(map[string]any)
|
|
firstReport := first["report"].(map[string]any)
|
|
if first["kind"] != "text_inference" || firstReport["kind"] != "reachability" || first["error"] != "operation failed" {
|
|
t.Fatalf("first aggregate entry = %#v", first)
|
|
}
|
|
if _, exists := firstReport["target"].(map[string]any)["version"]; exists {
|
|
t.Fatalf("absent target version was emitted: %#v", firstReport["target"])
|
|
}
|
|
second := tests[1].(map[string]any)
|
|
if second["kind"] != "authentication" || second["report"] != nil {
|
|
t.Fatalf("second aggregate entry = %#v", second)
|
|
}
|
|
if _, exists := second["error"]; exists {
|
|
t.Fatalf("nil error should be omitted: %#v", second)
|
|
}
|
|
third := tests[2].(map[string]any)
|
|
if message, exists := third["error"]; !exists || message != "" {
|
|
t.Fatalf("empty error message should remain present: %#v", third)
|
|
}
|
|
}
|
|
|
|
func containsSmokeKey(value any, want string) bool {
|
|
switch value := value.(type) {
|
|
case map[string]any:
|
|
for key, nested := range value {
|
|
if key == want || containsSmokeKey(nested, want) {
|
|
return true
|
|
}
|
|
}
|
|
case []any:
|
|
for _, nested := range value {
|
|
if containsSmokeKey(nested, want) {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|