Project native smoke reports to YAML

This commit is contained in:
2026-10-17 18:51:17 +00:00
parent 3e9c871cdc
commit 1650680723
3 changed files with 456 additions and 1 deletions

View File

@@ -1,6 +1,6 @@
# Dependency upgrades, discovery, and smoke testing implementation plan
Status: Stages 1–9 complete; Stage 10 is next.
Status: Stages 1–10 complete; Stage 11 is next.
## Authority and baseline
@@ -533,6 +533,16 @@ serialized; do not introduce filtering of returned `ExtraParams`.
**Complete when:** native smoke evidence and aggregate entries have a complete,
neutral YAML representation.
**Completed:** 2026-10-17. Added private report and aggregate DTOs that explicitly
project every exported PromptKit smoke report field, including target settings,
presence, nullable facts, timing, attempts, usage, and failure details. Durations
render as Go duration strings, empty attempts as `[]`, and optional prompt
versions preserve empty values while omitting absent ones. Aggregate entries
retain order, nullable reports, native report kinds, and optional error text.
Semantic YAML tests cover all field categories, nested values, omission,
presence, direct-request/key exclusion, and report-plus-error entries. Focused
projection tests and `GOTOOLCHAIN=local go build ./...` passed.
## Stage 11: Bind and execute single smoke tests
**Primary files:** private `smoke_command.go`, `Command` assembly, basic command

View File

@@ -0,0 +1,234 @@
package ckprompt
import (
"time"
promptkit "gitea.maximumdirect.net/eric/promptkit"
)
type smokeReportDTO struct {
Kind promptkit.SmokeTestKind `yaml:"kind"`
Supported bool `yaml:"supported"`
Applicable *bool `yaml:"applicable"`
Completed bool `yaml:"completed"`
Reason promptkit.SmokeTestReason `yaml:"reason"`
Target smokeTargetDTO `yaml:"target"`
Facts smokeFactsDTO `yaml:"facts"`
Timing smokeTimingDTO `yaml:"timing"`
Attempts []smokeAttemptDTO `yaml:"attempts"`
Usage smokeUsageDTO `yaml:"usage"`
Failure promptkit.ExecutionFailureKind `yaml:"failure"`
}
type smokeTargetDTO struct {
BackendID string `yaml:"backend_id"`
ProfileID string `yaml:"profile_id"`
PromptID string `yaml:"prompt_id"`
PromptVersion *string `yaml:"version,omitempty"`
DefaultSourceBackendID string `yaml:"default_source_backend_id"`
SelectionSource string `yaml:"selection_source"`
Endpoint string `yaml:"endpoint"`
EffectiveSettings smokeExecutionTargetDTO `yaml:"effective_settings"`
SettingsPresence smokeSettingsPresenceDTO `yaml:"settings_presence"`
}
type smokeSettingsPresenceDTO struct {
Temperature bool `yaml:"temperature"`
MaxTokens bool `yaml:"max_tokens"`
TopP bool `yaml:"top_p"`
TimeoutSeconds bool `yaml:"timeout_seconds"`
}
type smokeExecutionTargetDTO struct {
BackendID string `yaml:"backend_id,omitempty"`
Endpoint string `yaml:"endpoint"`
Model string `yaml:"model"`
Temperature float64 `yaml:"temperature"`
MaxTokens int `yaml:"max_tokens"`
TopP float64 `yaml:"top_p"`
TimeoutSeconds int `yaml:"timeout_seconds"`
ServiceTier string `yaml:"service_tier"`
ReasoningEffort string `yaml:"reasoning_effort"`
APIKeyEnv string `yaml:"api_key_env"`
ExtraParams map[string]any `yaml:"extra_params"`
}
type smokeFactsDTO struct {
CredentialsSupplied *bool `yaml:"credentials_supplied"`
EndpointReachable *bool `yaml:"endpoint_reachable"`
CredentialsAccepted *bool `yaml:"credentials_accepted"`
ResponseReceived *bool `yaml:"response_received"`
GenerationCompleted *bool `yaml:"generation_completed"`
TextReturned *bool `yaml:"text_returned"`
ExpectedTextMatched *bool `yaml:"expected_text_matched"`
StructuredOutputRequested *bool `yaml:"structured_output_requested"`
ValidJSON *bool `yaml:"valid_json"`
SchemaMatched *bool `yaml:"schema_matched"`
}
type smokeTimingDTO struct {
StartedAt time.Time `yaml:"started_at"`
Duration string `yaml:"duration"`
CapacityWait *string `yaml:"capacity_wait"`
RequestDuration *string `yaml:"request_duration"`
RetryDelay *string `yaml:"retry_delay"`
EvaluationDuration *string `yaml:"evaluation_duration"`
}
type smokeAttemptDTO struct {
Dispatched bool `yaml:"dispatched"`
InitialEndpoint string `yaml:"initial_endpoint"`
Endpoint string `yaml:"endpoint"`
HTTPStatus *int `yaml:"http_status"`
Facts smokeFactsDTO `yaml:"facts"`
Timing smokeTimingDTO `yaml:"timing"`
Failure promptkit.ExecutionFailureKind `yaml:"failure"`
Usage smokeUsageDTO `yaml:"usage"`
}
type smokeUsageDTO struct {
PromptTokens promptkit.ObservedTokenCount `yaml:"prompt_tokens"`
CompletionTokens promptkit.ObservedTokenCount `yaml:"completion_tokens"`
TotalTokens promptkit.ObservedTokenCount `yaml:"total_tokens"`
CachedTokens promptkit.ObservedTokenCount `yaml:"cached_tokens"`
CacheWriteTokens promptkit.ObservedTokenCount `yaml:"cache_write_tokens"`
}
type smokeAggregateDTO struct {
Tests []smokeResultDTO `yaml:"tests"`
}
type smokeResultDTO struct {
Kind promptkit.SmokeTestKind `yaml:"kind"`
Report *smokeReportDTO `yaml:"report"`
Error string `yaml:"error,omitempty"`
}
type smokeResult struct {
Kind promptkit.SmokeTestKind
Report *promptkit.SmokeTestReport
Err error
}
func encodeSmokeReport(report *promptkit.SmokeTestReport) ([]byte, error) {
return encodePromptYAMLBytes(smokeReportFromNative(report))
}
func encodeSmokeResults(results []smokeResult) ([]byte, error) {
entries := make([]smokeResultDTO, len(results))
for index, result := range results {
entries[index] = smokeResultDTO{
Kind: result.Kind,
Report: smokeReportFromNative(result.Report),
}
if result.Err != nil {
entries[index].Error = result.Err.Error()
}
}
return encodePromptYAMLBytes(smokeAggregateDTO{Tests: entries})
}
func smokeReportFromNative(report *promptkit.SmokeTestReport) *smokeReportDTO {
if report == nil {
return nil
}
attempts := make([]smokeAttemptDTO, len(report.Attempts))
for index, attempt := range report.Attempts {
attempts[index] = smokeAttemptDTO{
Dispatched: attempt.Dispatched,
InitialEndpoint: attempt.InitialEndpoint,
Endpoint: attempt.Endpoint,
HTTPStatus: attempt.HTTPStatus,
Facts: smokeFactsFromNative(attempt.Facts),
Timing: smokeTimingFromNative(attempt.Timing),
Failure: attempt.Failure,
Usage: smokeUsageFromNative(attempt.Usage),
}
}
target := report.Target
settings := target.EffectiveSettings
return &smokeReportDTO{
Kind: report.Kind,
Supported: report.Supported,
Applicable: report.Applicable,
Completed: report.Completed,
Reason: report.Reason,
Target: smokeTargetDTO{
BackendID: target.BackendID,
ProfileID: target.ProfileID,
PromptID: target.PromptID,
PromptVersion: target.PromptVersion,
DefaultSourceBackendID: target.DefaultSourceBackendID,
SelectionSource: target.SelectionSource,
Endpoint: target.Endpoint,
EffectiveSettings: smokeExecutionTargetDTO{
BackendID: settings.BackendID,
Endpoint: settings.Endpoint,
Model: settings.Model,
Temperature: settings.Temperature,
MaxTokens: settings.MaxTokens,
TopP: settings.TopP,
TimeoutSeconds: settings.TimeoutSeconds,
ServiceTier: settings.ServiceTier,
ReasoningEffort: settings.ReasoningEffort,
APIKeyEnv: settings.APIKeyEnv,
ExtraParams: settings.ExtraParams,
},
SettingsPresence: smokeSettingsPresenceDTO{
Temperature: target.SettingsPresence.Temperature,
MaxTokens: target.SettingsPresence.MaxTokens,
TopP: target.SettingsPresence.TopP,
TimeoutSeconds: target.SettingsPresence.TimeoutSeconds,
},
},
Facts: smokeFactsFromNative(report.Facts),
Timing: smokeTimingFromNative(report.Timing),
Attempts: attempts,
Usage: smokeUsageFromNative(report.Usage),
Failure: report.Failure,
}
}
func smokeFactsFromNative(facts promptkit.SmokeTestFacts) smokeFactsDTO {
return smokeFactsDTO{
CredentialsSupplied: facts.CredentialsSupplied,
EndpointReachable: facts.EndpointReachable,
CredentialsAccepted: facts.CredentialsAccepted,
ResponseReceived: facts.ResponseReceived,
GenerationCompleted: facts.GenerationCompleted,
TextReturned: facts.TextReturned,
ExpectedTextMatched: facts.ExpectedTextMatched,
StructuredOutputRequested: facts.StructuredOutputRequested,
ValidJSON: facts.ValidJSON,
SchemaMatched: facts.SchemaMatched,
}
}
func smokeTimingFromNative(timing promptkit.SmokeTestTiming) smokeTimingDTO {
return smokeTimingDTO{
StartedAt: timing.StartedAt,
Duration: timing.Duration.String(),
CapacityWait: durationString(timing.CapacityWait),
RequestDuration: durationString(timing.RequestDuration),
RetryDelay: durationString(timing.RetryDelay),
EvaluationDuration: durationString(timing.EvaluationDuration),
}
}
func durationString(duration *time.Duration) *string {
if duration == nil {
return nil
}
value := duration.String()
return &value
}
func smokeUsageFromNative(usage promptkit.ObservedTokenUsage) smokeUsageDTO {
return smokeUsageDTO{
PromptTokens: usage.PromptTokens,
CompletionTokens: usage.CompletionTokens,
TotalTokens: usage.TotalTokens,
CachedTokens: usage.CachedTokens,
CacheWriteTokens: usage.CacheWriteTokens,
}
}

View File

@@ -0,0 +1,211 @@
package ckprompt
import (
"errors"
"testing"
"time"
promptkit "gitea.maximumdirect.net/eric/promptkit"
"go.yaml.in/yaml/v4"
)
func TestSmokeReportYAMLProjectsNativeFieldsAndPresence(t *testing.T) {
applicable := false
credentialsSupplied := false
endpointReachable := true
textReturned := false
zero := time.Duration(0)
retryDelay := 1500 * time.Millisecond
version := ""
httpStatus := 201
started := time.Date(2026, time.October, 17, 12, 30, 0, 0, time.UTC)
report := &promptkit.SmokeTestReport{
Kind: promptkit.SmokeTestReachability,
Supported: true,
Applicable: &applicable,
Completed: false,
Reason: promptkit.SmokeTestReasonHTTPDiagnosticsUnavailable,
Target: promptkit.SmokeTestTarget{
BackendID: "local",
ProfileID: "default",
PromptID: "team/briefing",
PromptVersion: &version,
DefaultSourceBackendID: "fallback",
SelectionSource: "prompt",
Endpoint: "https://example.invalid/v1",
EffectiveSettings: promptkit.ExecutionTarget{
BackendID: "local",
Endpoint: "https://example.invalid/v1",
Model: "example-model",
Temperature: 0,
MaxTokens: 0,
TopP: 0,
TimeoutSeconds: 0,
ServiceTier: "",
ReasoningEffort: "",
APIKeyEnv: "PROMPTKIT_KEY",
ExtraParams: map[string]any{"token": "caller-value", "nested": map[string]any{"count": 0}},
},
SettingsPresence: promptkit.ExecutionTargetPresence{Temperature: true, MaxTokens: false, TopP: true, TimeoutSeconds: false},
},
Facts: promptkit.SmokeTestFacts{
CredentialsSupplied: &credentialsSupplied,
EndpointReachable: &endpointReachable,
CredentialsAccepted: nil,
ResponseReceived: &textReturned,
GenerationCompleted: &endpointReachable,
TextReturned: &textReturned,
ExpectedTextMatched: &textReturned,
StructuredOutputRequested: &endpointReachable,
ValidJSON: &textReturned,
SchemaMatched: nil,
},
Timing: promptkit.SmokeTestTiming{
StartedAt: started,
Duration: zero,
CapacityWait: &zero,
RequestDuration: nil,
RetryDelay: &retryDelay,
EvaluationDuration: nil,
},
Attempts: []promptkit.SmokeTestAttempt{{
Dispatched: true,
InitialEndpoint: "https://initial.example/v1",
Endpoint: "https://final.example/v1",
HTTPStatus: &httpStatus,
Facts: promptkit.SmokeTestFacts{EndpointReachable: &endpointReachable},
Timing: promptkit.SmokeTestTiming{Duration: retryDelay},
Failure: promptkit.ExecutionFailureNone,
Usage: promptkit.ObservedTokenUsage{PromptTokens: promptkit.ObservedTokenCount{Value: 2, Known: true, Complete: true}},
}},
Usage: promptkit.ObservedTokenUsage{
PromptTokens: promptkit.ObservedTokenCount{Value: 0, Known: true, Complete: true},
CompletionTokens: promptkit.ObservedTokenCount{Value: 0, Known: false, Complete: false},
TotalTokens: promptkit.ObservedTokenCount{Value: 7, Known: true, Complete: false},
},
Failure: promptkit.ExecutionFailureTransport,
}
encoded, err := encodeSmokeReport(report)
if err != nil {
t.Fatal(err)
}
var document map[string]any
if err := yaml.Unmarshal(encoded, &document); err != nil {
t.Fatalf("decode smoke report: %v\n%s", err, encoded)
}
if document["kind"] != "reachability" || document["supported"] != true || document["applicable"] != false || document["completed"] != false || document["reason"] != "http_diagnostics_unavailable" || document["failure"] != "transport" {
t.Fatalf("report scalar values = %#v", document)
}
target := document["target"].(map[string]any)
if target["prompt_id"] != "team/briefing" || target["version"] != "" || target["selection_source"] != "prompt" {
t.Fatalf("target values = %#v", target)
}
settings := target["effective_settings"].(map[string]any)
if settings["model"] != "example-model" || settings["temperature"] != 0 || settings["api_key_env"] != "PROMPTKIT_KEY" {
t.Fatalf("effective settings = %#v", settings)
}
extra := settings["extra_params"].(map[string]any)
if extra["token"] != "caller-value" {
t.Fatalf("extra parameters were not retained: %#v", extra)
}
presence := target["settings_presence"].(map[string]any)
if presence["temperature"] != true || presence["max_tokens"] != false || presence["top_p"] != true || presence["timeout_seconds"] != false {
t.Fatalf("setting presence = %#v", presence)
}
facts := document["facts"].(map[string]any)
if facts["credentials_supplied"] != false || facts["endpoint_reachable"] != true || facts["credentials_accepted"] != nil || facts["response_received"] != false || facts["generation_completed"] != true || facts["text_returned"] != false || facts["expected_text_matched"] != false || facts["structured_output_requested"] != true || facts["valid_json"] != false || facts["schema_matched"] != nil {
t.Fatalf("nullable facts = %#v", facts)
}
timing := document["timing"].(map[string]any)
if timing["started_at"] == nil || timing["duration"] != "0s" || timing["capacity_wait"] != "0s" || timing["request_duration"] != nil || timing["retry_delay"] != "1.5s" || timing["evaluation_duration"] != nil {
t.Fatalf("timing values = %#v", timing)
}
attempts, ok := document["attempts"].([]any)
if !ok || len(attempts) != 1 {
t.Fatalf("attempts = %#v", document["attempts"])
}
attempt := attempts[0].(map[string]any)
if attempt["dispatched"] != true || attempt["initial_endpoint"] != "https://initial.example/v1" || attempt["endpoint"] != "https://final.example/v1" || attempt["http_status"] != 201 {
t.Fatalf("attempt fields = %#v", attempt)
}
if attempt["timing"].(map[string]any)["duration"] != "1.5s" {
t.Fatalf("attempt timing = %#v", attempt["timing"])
}
usage := document["usage"].(map[string]any)
promptTokens := usage["prompt_tokens"].(map[string]any)
unknownTokens := usage["completion_tokens"].(map[string]any)
if promptTokens["value"] != 0 || promptTokens["known"] != true || promptTokens["complete"] != true || unknownTokens["known"] != false || unknownTokens["complete"] != false {
t.Fatalf("presence-aware usage = %#v", usage)
}
if containsSmokeKey(document, "request") || containsSmokeKey(document, "api_key") {
t.Fatalf("serialized request or direct API key: %#v", document)
}
}
func TestSmokeReportYAMLEmitsNilAttemptsAsEmptySequence(t *testing.T) {
encoded, err := encodeSmokeReport(&promptkit.SmokeTestReport{Attempts: nil})
if err != nil {
t.Fatal(err)
}
var document map[string]any
if err := yaml.Unmarshal(encoded, &document); err != nil {
t.Fatal(err)
}
if attempts, ok := document["attempts"].([]any); !ok || len(attempts) != 0 {
t.Fatalf("nil native attempts = %#v", document["attempts"])
}
}
func TestSmokeAggregateYAMLRetainsOrderErrorAndNativeReportKind(t *testing.T) {
operationErr := errors.New("operation failed")
report := &promptkit.SmokeTestReport{Kind: promptkit.SmokeTestReachability, Attempts: []promptkit.SmokeTestAttempt{}}
encoded, err := encodeSmokeResults([]smokeResult{
{Kind: promptkit.SmokeTestTextInference, Report: report, Err: operationErr},
{Kind: promptkit.SmokeTestAuthentication},
})
if err != nil {
t.Fatal(err)
}
var document map[string]any
if err := yaml.Unmarshal(encoded, &document); err != nil {
t.Fatalf("decode aggregate report: %v\n%s", err, encoded)
}
tests, ok := document["tests"].([]any)
if !ok || len(tests) != 2 {
t.Fatalf("aggregate entries = %#v", document)
}
first := tests[0].(map[string]any)
firstReport := first["report"].(map[string]any)
if first["kind"] != "text_inference" || firstReport["kind"] != "reachability" || first["error"] != "operation failed" {
t.Fatalf("first aggregate entry = %#v", first)
}
if _, exists := firstReport["target"].(map[string]any)["version"]; exists {
t.Fatalf("absent target version was emitted: %#v", firstReport["target"])
}
second := tests[1].(map[string]any)
if second["kind"] != "authentication" || second["report"] != nil {
t.Fatalf("second aggregate entry = %#v", second)
}
if _, exists := second["error"]; exists {
t.Fatalf("nil error should be omitted: %#v", second)
}
}
func containsSmokeKey(value any, want string) bool {
switch value := value.(type) {
case map[string]any:
for key, nested := range value {
if key == want || containsSmokeKey(nested, want) {
return true
}
}
case []any:
for _, nested := range value {
if containsSmokeKey(nested, want) {
return true
}
}
}
return false
}