Project native smoke reports to YAML
This commit is contained in:
234
integrations/ckprompt/smoke_report.go
Normal file
234
integrations/ckprompt/smoke_report.go
Normal file
@@ -0,0 +1,234 @@
|
||||
package ckprompt
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
promptkit "gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
type smokeReportDTO struct {
|
||||
Kind promptkit.SmokeTestKind `yaml:"kind"`
|
||||
Supported bool `yaml:"supported"`
|
||||
Applicable *bool `yaml:"applicable"`
|
||||
Completed bool `yaml:"completed"`
|
||||
Reason promptkit.SmokeTestReason `yaml:"reason"`
|
||||
Target smokeTargetDTO `yaml:"target"`
|
||||
Facts smokeFactsDTO `yaml:"facts"`
|
||||
Timing smokeTimingDTO `yaml:"timing"`
|
||||
Attempts []smokeAttemptDTO `yaml:"attempts"`
|
||||
Usage smokeUsageDTO `yaml:"usage"`
|
||||
Failure promptkit.ExecutionFailureKind `yaml:"failure"`
|
||||
}
|
||||
|
||||
type smokeTargetDTO struct {
|
||||
BackendID string `yaml:"backend_id"`
|
||||
ProfileID string `yaml:"profile_id"`
|
||||
PromptID string `yaml:"prompt_id"`
|
||||
PromptVersion *string `yaml:"version,omitempty"`
|
||||
DefaultSourceBackendID string `yaml:"default_source_backend_id"`
|
||||
SelectionSource string `yaml:"selection_source"`
|
||||
Endpoint string `yaml:"endpoint"`
|
||||
EffectiveSettings smokeExecutionTargetDTO `yaml:"effective_settings"`
|
||||
SettingsPresence smokeSettingsPresenceDTO `yaml:"settings_presence"`
|
||||
}
|
||||
|
||||
type smokeSettingsPresenceDTO struct {
|
||||
Temperature bool `yaml:"temperature"`
|
||||
MaxTokens bool `yaml:"max_tokens"`
|
||||
TopP bool `yaml:"top_p"`
|
||||
TimeoutSeconds bool `yaml:"timeout_seconds"`
|
||||
}
|
||||
|
||||
type smokeExecutionTargetDTO struct {
|
||||
BackendID string `yaml:"backend_id,omitempty"`
|
||||
Endpoint string `yaml:"endpoint"`
|
||||
Model string `yaml:"model"`
|
||||
Temperature float64 `yaml:"temperature"`
|
||||
MaxTokens int `yaml:"max_tokens"`
|
||||
TopP float64 `yaml:"top_p"`
|
||||
TimeoutSeconds int `yaml:"timeout_seconds"`
|
||||
ServiceTier string `yaml:"service_tier"`
|
||||
ReasoningEffort string `yaml:"reasoning_effort"`
|
||||
APIKeyEnv string `yaml:"api_key_env"`
|
||||
ExtraParams map[string]any `yaml:"extra_params"`
|
||||
}
|
||||
|
||||
type smokeFactsDTO struct {
|
||||
CredentialsSupplied *bool `yaml:"credentials_supplied"`
|
||||
EndpointReachable *bool `yaml:"endpoint_reachable"`
|
||||
CredentialsAccepted *bool `yaml:"credentials_accepted"`
|
||||
ResponseReceived *bool `yaml:"response_received"`
|
||||
GenerationCompleted *bool `yaml:"generation_completed"`
|
||||
TextReturned *bool `yaml:"text_returned"`
|
||||
ExpectedTextMatched *bool `yaml:"expected_text_matched"`
|
||||
StructuredOutputRequested *bool `yaml:"structured_output_requested"`
|
||||
ValidJSON *bool `yaml:"valid_json"`
|
||||
SchemaMatched *bool `yaml:"schema_matched"`
|
||||
}
|
||||
|
||||
type smokeTimingDTO struct {
|
||||
StartedAt time.Time `yaml:"started_at"`
|
||||
Duration string `yaml:"duration"`
|
||||
CapacityWait *string `yaml:"capacity_wait"`
|
||||
RequestDuration *string `yaml:"request_duration"`
|
||||
RetryDelay *string `yaml:"retry_delay"`
|
||||
EvaluationDuration *string `yaml:"evaluation_duration"`
|
||||
}
|
||||
|
||||
type smokeAttemptDTO struct {
|
||||
Dispatched bool `yaml:"dispatched"`
|
||||
InitialEndpoint string `yaml:"initial_endpoint"`
|
||||
Endpoint string `yaml:"endpoint"`
|
||||
HTTPStatus *int `yaml:"http_status"`
|
||||
Facts smokeFactsDTO `yaml:"facts"`
|
||||
Timing smokeTimingDTO `yaml:"timing"`
|
||||
Failure promptkit.ExecutionFailureKind `yaml:"failure"`
|
||||
Usage smokeUsageDTO `yaml:"usage"`
|
||||
}
|
||||
|
||||
type smokeUsageDTO struct {
|
||||
PromptTokens promptkit.ObservedTokenCount `yaml:"prompt_tokens"`
|
||||
CompletionTokens promptkit.ObservedTokenCount `yaml:"completion_tokens"`
|
||||
TotalTokens promptkit.ObservedTokenCount `yaml:"total_tokens"`
|
||||
CachedTokens promptkit.ObservedTokenCount `yaml:"cached_tokens"`
|
||||
CacheWriteTokens promptkit.ObservedTokenCount `yaml:"cache_write_tokens"`
|
||||
}
|
||||
|
||||
type smokeAggregateDTO struct {
|
||||
Tests []smokeResultDTO `yaml:"tests"`
|
||||
}
|
||||
|
||||
type smokeResultDTO struct {
|
||||
Kind promptkit.SmokeTestKind `yaml:"kind"`
|
||||
Report *smokeReportDTO `yaml:"report"`
|
||||
Error string `yaml:"error,omitempty"`
|
||||
}
|
||||
|
||||
type smokeResult struct {
|
||||
Kind promptkit.SmokeTestKind
|
||||
Report *promptkit.SmokeTestReport
|
||||
Err error
|
||||
}
|
||||
|
||||
func encodeSmokeReport(report *promptkit.SmokeTestReport) ([]byte, error) {
|
||||
return encodePromptYAMLBytes(smokeReportFromNative(report))
|
||||
}
|
||||
|
||||
func encodeSmokeResults(results []smokeResult) ([]byte, error) {
|
||||
entries := make([]smokeResultDTO, len(results))
|
||||
for index, result := range results {
|
||||
entries[index] = smokeResultDTO{
|
||||
Kind: result.Kind,
|
||||
Report: smokeReportFromNative(result.Report),
|
||||
}
|
||||
if result.Err != nil {
|
||||
entries[index].Error = result.Err.Error()
|
||||
}
|
||||
}
|
||||
return encodePromptYAMLBytes(smokeAggregateDTO{Tests: entries})
|
||||
}
|
||||
|
||||
func smokeReportFromNative(report *promptkit.SmokeTestReport) *smokeReportDTO {
|
||||
if report == nil {
|
||||
return nil
|
||||
}
|
||||
attempts := make([]smokeAttemptDTO, len(report.Attempts))
|
||||
for index, attempt := range report.Attempts {
|
||||
attempts[index] = smokeAttemptDTO{
|
||||
Dispatched: attempt.Dispatched,
|
||||
InitialEndpoint: attempt.InitialEndpoint,
|
||||
Endpoint: attempt.Endpoint,
|
||||
HTTPStatus: attempt.HTTPStatus,
|
||||
Facts: smokeFactsFromNative(attempt.Facts),
|
||||
Timing: smokeTimingFromNative(attempt.Timing),
|
||||
Failure: attempt.Failure,
|
||||
Usage: smokeUsageFromNative(attempt.Usage),
|
||||
}
|
||||
}
|
||||
target := report.Target
|
||||
settings := target.EffectiveSettings
|
||||
return &smokeReportDTO{
|
||||
Kind: report.Kind,
|
||||
Supported: report.Supported,
|
||||
Applicable: report.Applicable,
|
||||
Completed: report.Completed,
|
||||
Reason: report.Reason,
|
||||
Target: smokeTargetDTO{
|
||||
BackendID: target.BackendID,
|
||||
ProfileID: target.ProfileID,
|
||||
PromptID: target.PromptID,
|
||||
PromptVersion: target.PromptVersion,
|
||||
DefaultSourceBackendID: target.DefaultSourceBackendID,
|
||||
SelectionSource: target.SelectionSource,
|
||||
Endpoint: target.Endpoint,
|
||||
EffectiveSettings: smokeExecutionTargetDTO{
|
||||
BackendID: settings.BackendID,
|
||||
Endpoint: settings.Endpoint,
|
||||
Model: settings.Model,
|
||||
Temperature: settings.Temperature,
|
||||
MaxTokens: settings.MaxTokens,
|
||||
TopP: settings.TopP,
|
||||
TimeoutSeconds: settings.TimeoutSeconds,
|
||||
ServiceTier: settings.ServiceTier,
|
||||
ReasoningEffort: settings.ReasoningEffort,
|
||||
APIKeyEnv: settings.APIKeyEnv,
|
||||
ExtraParams: settings.ExtraParams,
|
||||
},
|
||||
SettingsPresence: smokeSettingsPresenceDTO{
|
||||
Temperature: target.SettingsPresence.Temperature,
|
||||
MaxTokens: target.SettingsPresence.MaxTokens,
|
||||
TopP: target.SettingsPresence.TopP,
|
||||
TimeoutSeconds: target.SettingsPresence.TimeoutSeconds,
|
||||
},
|
||||
},
|
||||
Facts: smokeFactsFromNative(report.Facts),
|
||||
Timing: smokeTimingFromNative(report.Timing),
|
||||
Attempts: attempts,
|
||||
Usage: smokeUsageFromNative(report.Usage),
|
||||
Failure: report.Failure,
|
||||
}
|
||||
}
|
||||
|
||||
func smokeFactsFromNative(facts promptkit.SmokeTestFacts) smokeFactsDTO {
|
||||
return smokeFactsDTO{
|
||||
CredentialsSupplied: facts.CredentialsSupplied,
|
||||
EndpointReachable: facts.EndpointReachable,
|
||||
CredentialsAccepted: facts.CredentialsAccepted,
|
||||
ResponseReceived: facts.ResponseReceived,
|
||||
GenerationCompleted: facts.GenerationCompleted,
|
||||
TextReturned: facts.TextReturned,
|
||||
ExpectedTextMatched: facts.ExpectedTextMatched,
|
||||
StructuredOutputRequested: facts.StructuredOutputRequested,
|
||||
ValidJSON: facts.ValidJSON,
|
||||
SchemaMatched: facts.SchemaMatched,
|
||||
}
|
||||
}
|
||||
|
||||
func smokeTimingFromNative(timing promptkit.SmokeTestTiming) smokeTimingDTO {
|
||||
return smokeTimingDTO{
|
||||
StartedAt: timing.StartedAt,
|
||||
Duration: timing.Duration.String(),
|
||||
CapacityWait: durationString(timing.CapacityWait),
|
||||
RequestDuration: durationString(timing.RequestDuration),
|
||||
RetryDelay: durationString(timing.RetryDelay),
|
||||
EvaluationDuration: durationString(timing.EvaluationDuration),
|
||||
}
|
||||
}
|
||||
|
||||
func durationString(duration *time.Duration) *string {
|
||||
if duration == nil {
|
||||
return nil
|
||||
}
|
||||
value := duration.String()
|
||||
return &value
|
||||
}
|
||||
|
||||
func smokeUsageFromNative(usage promptkit.ObservedTokenUsage) smokeUsageDTO {
|
||||
return smokeUsageDTO{
|
||||
PromptTokens: usage.PromptTokens,
|
||||
CompletionTokens: usage.CompletionTokens,
|
||||
TotalTokens: usage.TotalTokens,
|
||||
CachedTokens: usage.CachedTokens,
|
||||
CacheWriteTokens: usage.CacheWriteTokens,
|
||||
}
|
||||
}
|
||||
211
integrations/ckprompt/smoke_report_test.go
Normal file
211
integrations/ckprompt/smoke_report_test.go
Normal file
@@ -0,0 +1,211 @@
|
||||
package ckprompt
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
promptkit "gitea.maximumdirect.net/eric/promptkit"
|
||||
"go.yaml.in/yaml/v4"
|
||||
)
|
||||
|
||||
func TestSmokeReportYAMLProjectsNativeFieldsAndPresence(t *testing.T) {
|
||||
applicable := false
|
||||
credentialsSupplied := false
|
||||
endpointReachable := true
|
||||
textReturned := false
|
||||
zero := time.Duration(0)
|
||||
retryDelay := 1500 * time.Millisecond
|
||||
version := ""
|
||||
httpStatus := 201
|
||||
started := time.Date(2026, time.October, 17, 12, 30, 0, 0, time.UTC)
|
||||
report := &promptkit.SmokeTestReport{
|
||||
Kind: promptkit.SmokeTestReachability,
|
||||
Supported: true,
|
||||
Applicable: &applicable,
|
||||
Completed: false,
|
||||
Reason: promptkit.SmokeTestReasonHTTPDiagnosticsUnavailable,
|
||||
Target: promptkit.SmokeTestTarget{
|
||||
BackendID: "local",
|
||||
ProfileID: "default",
|
||||
PromptID: "team/briefing",
|
||||
PromptVersion: &version,
|
||||
DefaultSourceBackendID: "fallback",
|
||||
SelectionSource: "prompt",
|
||||
Endpoint: "https://example.invalid/v1",
|
||||
EffectiveSettings: promptkit.ExecutionTarget{
|
||||
BackendID: "local",
|
||||
Endpoint: "https://example.invalid/v1",
|
||||
Model: "example-model",
|
||||
Temperature: 0,
|
||||
MaxTokens: 0,
|
||||
TopP: 0,
|
||||
TimeoutSeconds: 0,
|
||||
ServiceTier: "",
|
||||
ReasoningEffort: "",
|
||||
APIKeyEnv: "PROMPTKIT_KEY",
|
||||
ExtraParams: map[string]any{"token": "caller-value", "nested": map[string]any{"count": 0}},
|
||||
},
|
||||
SettingsPresence: promptkit.ExecutionTargetPresence{Temperature: true, MaxTokens: false, TopP: true, TimeoutSeconds: false},
|
||||
},
|
||||
Facts: promptkit.SmokeTestFacts{
|
||||
CredentialsSupplied: &credentialsSupplied,
|
||||
EndpointReachable: &endpointReachable,
|
||||
CredentialsAccepted: nil,
|
||||
ResponseReceived: &textReturned,
|
||||
GenerationCompleted: &endpointReachable,
|
||||
TextReturned: &textReturned,
|
||||
ExpectedTextMatched: &textReturned,
|
||||
StructuredOutputRequested: &endpointReachable,
|
||||
ValidJSON: &textReturned,
|
||||
SchemaMatched: nil,
|
||||
},
|
||||
Timing: promptkit.SmokeTestTiming{
|
||||
StartedAt: started,
|
||||
Duration: zero,
|
||||
CapacityWait: &zero,
|
||||
RequestDuration: nil,
|
||||
RetryDelay: &retryDelay,
|
||||
EvaluationDuration: nil,
|
||||
},
|
||||
Attempts: []promptkit.SmokeTestAttempt{{
|
||||
Dispatched: true,
|
||||
InitialEndpoint: "https://initial.example/v1",
|
||||
Endpoint: "https://final.example/v1",
|
||||
HTTPStatus: &httpStatus,
|
||||
Facts: promptkit.SmokeTestFacts{EndpointReachable: &endpointReachable},
|
||||
Timing: promptkit.SmokeTestTiming{Duration: retryDelay},
|
||||
Failure: promptkit.ExecutionFailureNone,
|
||||
Usage: promptkit.ObservedTokenUsage{PromptTokens: promptkit.ObservedTokenCount{Value: 2, Known: true, Complete: true}},
|
||||
}},
|
||||
Usage: promptkit.ObservedTokenUsage{
|
||||
PromptTokens: promptkit.ObservedTokenCount{Value: 0, Known: true, Complete: true},
|
||||
CompletionTokens: promptkit.ObservedTokenCount{Value: 0, Known: false, Complete: false},
|
||||
TotalTokens: promptkit.ObservedTokenCount{Value: 7, Known: true, Complete: false},
|
||||
},
|
||||
Failure: promptkit.ExecutionFailureTransport,
|
||||
}
|
||||
|
||||
encoded, err := encodeSmokeReport(report)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var document map[string]any
|
||||
if err := yaml.Unmarshal(encoded, &document); err != nil {
|
||||
t.Fatalf("decode smoke report: %v\n%s", err, encoded)
|
||||
}
|
||||
if document["kind"] != "reachability" || document["supported"] != true || document["applicable"] != false || document["completed"] != false || document["reason"] != "http_diagnostics_unavailable" || document["failure"] != "transport" {
|
||||
t.Fatalf("report scalar values = %#v", document)
|
||||
}
|
||||
target := document["target"].(map[string]any)
|
||||
if target["prompt_id"] != "team/briefing" || target["version"] != "" || target["selection_source"] != "prompt" {
|
||||
t.Fatalf("target values = %#v", target)
|
||||
}
|
||||
settings := target["effective_settings"].(map[string]any)
|
||||
if settings["model"] != "example-model" || settings["temperature"] != 0 || settings["api_key_env"] != "PROMPTKIT_KEY" {
|
||||
t.Fatalf("effective settings = %#v", settings)
|
||||
}
|
||||
extra := settings["extra_params"].(map[string]any)
|
||||
if extra["token"] != "caller-value" {
|
||||
t.Fatalf("extra parameters were not retained: %#v", extra)
|
||||
}
|
||||
presence := target["settings_presence"].(map[string]any)
|
||||
if presence["temperature"] != true || presence["max_tokens"] != false || presence["top_p"] != true || presence["timeout_seconds"] != false {
|
||||
t.Fatalf("setting presence = %#v", presence)
|
||||
}
|
||||
facts := document["facts"].(map[string]any)
|
||||
if facts["credentials_supplied"] != false || facts["endpoint_reachable"] != true || facts["credentials_accepted"] != nil || facts["response_received"] != false || facts["generation_completed"] != true || facts["text_returned"] != false || facts["expected_text_matched"] != false || facts["structured_output_requested"] != true || facts["valid_json"] != false || facts["schema_matched"] != nil {
|
||||
t.Fatalf("nullable facts = %#v", facts)
|
||||
}
|
||||
timing := document["timing"].(map[string]any)
|
||||
if timing["started_at"] == nil || timing["duration"] != "0s" || timing["capacity_wait"] != "0s" || timing["request_duration"] != nil || timing["retry_delay"] != "1.5s" || timing["evaluation_duration"] != nil {
|
||||
t.Fatalf("timing values = %#v", timing)
|
||||
}
|
||||
attempts, ok := document["attempts"].([]any)
|
||||
if !ok || len(attempts) != 1 {
|
||||
t.Fatalf("attempts = %#v", document["attempts"])
|
||||
}
|
||||
attempt := attempts[0].(map[string]any)
|
||||
if attempt["dispatched"] != true || attempt["initial_endpoint"] != "https://initial.example/v1" || attempt["endpoint"] != "https://final.example/v1" || attempt["http_status"] != 201 {
|
||||
t.Fatalf("attempt fields = %#v", attempt)
|
||||
}
|
||||
if attempt["timing"].(map[string]any)["duration"] != "1.5s" {
|
||||
t.Fatalf("attempt timing = %#v", attempt["timing"])
|
||||
}
|
||||
usage := document["usage"].(map[string]any)
|
||||
promptTokens := usage["prompt_tokens"].(map[string]any)
|
||||
unknownTokens := usage["completion_tokens"].(map[string]any)
|
||||
if promptTokens["value"] != 0 || promptTokens["known"] != true || promptTokens["complete"] != true || unknownTokens["known"] != false || unknownTokens["complete"] != false {
|
||||
t.Fatalf("presence-aware usage = %#v", usage)
|
||||
}
|
||||
if containsSmokeKey(document, "request") || containsSmokeKey(document, "api_key") {
|
||||
t.Fatalf("serialized request or direct API key: %#v", document)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSmokeReportYAMLEmitsNilAttemptsAsEmptySequence(t *testing.T) {
|
||||
encoded, err := encodeSmokeReport(&promptkit.SmokeTestReport{Attempts: nil})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var document map[string]any
|
||||
if err := yaml.Unmarshal(encoded, &document); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if attempts, ok := document["attempts"].([]any); !ok || len(attempts) != 0 {
|
||||
t.Fatalf("nil native attempts = %#v", document["attempts"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestSmokeAggregateYAMLRetainsOrderErrorAndNativeReportKind(t *testing.T) {
|
||||
operationErr := errors.New("operation failed")
|
||||
report := &promptkit.SmokeTestReport{Kind: promptkit.SmokeTestReachability, Attempts: []promptkit.SmokeTestAttempt{}}
|
||||
encoded, err := encodeSmokeResults([]smokeResult{
|
||||
{Kind: promptkit.SmokeTestTextInference, Report: report, Err: operationErr},
|
||||
{Kind: promptkit.SmokeTestAuthentication},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var document map[string]any
|
||||
if err := yaml.Unmarshal(encoded, &document); err != nil {
|
||||
t.Fatalf("decode aggregate report: %v\n%s", err, encoded)
|
||||
}
|
||||
tests, ok := document["tests"].([]any)
|
||||
if !ok || len(tests) != 2 {
|
||||
t.Fatalf("aggregate entries = %#v", document)
|
||||
}
|
||||
first := tests[0].(map[string]any)
|
||||
firstReport := first["report"].(map[string]any)
|
||||
if first["kind"] != "text_inference" || firstReport["kind"] != "reachability" || first["error"] != "operation failed" {
|
||||
t.Fatalf("first aggregate entry = %#v", first)
|
||||
}
|
||||
if _, exists := firstReport["target"].(map[string]any)["version"]; exists {
|
||||
t.Fatalf("absent target version was emitted: %#v", firstReport["target"])
|
||||
}
|
||||
second := tests[1].(map[string]any)
|
||||
if second["kind"] != "authentication" || second["report"] != nil {
|
||||
t.Fatalf("second aggregate entry = %#v", second)
|
||||
}
|
||||
if _, exists := second["error"]; exists {
|
||||
t.Fatalf("nil error should be omitted: %#v", second)
|
||||
}
|
||||
}
|
||||
|
||||
func containsSmokeKey(value any, want string) bool {
|
||||
switch value := value.(type) {
|
||||
case map[string]any:
|
||||
for key, nested := range value {
|
||||
if key == want || containsSmokeKey(nested, want) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
case []any:
|
||||
for _, nested := range value {
|
||||
if containsSmokeKey(nested, want) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
Reference in New Issue
Block a user