base-layer/commandkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Migrate PromptKit reports to shared rendering

Browse Source
This commit is contained in:
Eric Rakestraw
2026-11-07 04:10:47 +00:00
parent 5ff0dd4ea2
commit 6378ceb3c5
15 changed files with 249 additions and 265 deletions
Download Patch File Download Diff File Expand all files Collapse all files

163
integrations/ckprompt/smoke_report.go
View File

@@ -7,100 +7,106 @@ import (
)
type smokeReportDTO struct {
Kind promptkit.SmokeTestKind `yaml:"kind"`
Supported bool `yaml:"supported"`
Applicable *bool `yaml:"applicable"`
Completed bool `yaml:"completed"`
Reason promptkit.SmokeTestReason `yaml:"reason"`
Target smokeTargetDTO `yaml:"target"`
Facts smokeFactsDTO `yaml:"facts"`
Timing smokeTimingDTO `yaml:"timing"`
Attempts []smokeAttemptDTO `yaml:"attempts"`
Usage smokeUsageDTO `yaml:"usage"`
Failure promptkit.ExecutionFailureKind `yaml:"failure"`
Kind promptkit.SmokeTestKind `json:"kind"`
Supported bool `json:"supported"`
Applicable *bool `json:"applicable"`
Completed bool `json:"completed"`
Reason promptkit.SmokeTestReason `json:"reason"`
Target smokeTargetDTO `json:"target"`
Facts smokeFactsDTO `json:"facts"`
Timing smokeTimingDTO `json:"timing"`
Attempts []smokeAttemptDTO `json:"attempts"`
Usage smokeUsageDTO `json:"usage"`
Failure promptkit.ExecutionFailureKind `json:"failure"`
}
type smokeTargetDTO struct {
BackendID string `yaml:"backend_id"`
ProfileID string `yaml:"profile_id"`
PromptID string `yaml:"prompt_id"`
DefaultSourceBackendID string `yaml:"default_source_backend_id"`
SelectionSource string `yaml:"selection_source"`
Endpoint string `yaml:"endpoint"`
EffectiveSettings smokeExecutionTargetDTO `yaml:"effective_settings"`
SettingsPresence smokeSettingsPresenceDTO `yaml:"settings_presence"`
BackendID string `json:"backend_id"`
ProfileID string `json:"profile_id"`
PromptID string `json:"prompt_id"`
DefaultSourceBackendID string `json:"default_source_backend_id"`
SelectionSource string `json:"selection_source"`
Endpoint string `json:"endpoint"`
EffectiveSettings smokeExecutionTargetDTO `json:"effective_settings"`
SettingsPresence smokeSettingsPresenceDTO `json:"settings_presence"`
}
type smokeSettingsPresenceDTO struct {
Temperature bool `yaml:"temperature"`
MaxTokens bool `yaml:"max_tokens"`
TopP bool `yaml:"top_p"`
TimeoutSeconds bool `yaml:"timeout_seconds"`
Temperature bool `json:"temperature"`
MaxTokens bool `json:"max_tokens"`
TopP bool `json:"top_p"`
TimeoutSeconds bool `json:"timeout_seconds"`
}
type smokeExecutionTargetDTO struct {
BackendID string `yaml:"backend_id,omitempty"`
Endpoint string `yaml:"endpoint"`
Model string `yaml:"model"`
Temperature float64 `yaml:"temperature"`
MaxTokens int `yaml:"max_tokens"`
TopP float64 `yaml:"top_p"`
TimeoutSeconds int `yaml:"timeout_seconds"`
ServiceTier string `yaml:"service_tier"`
ReasoningEffort string `yaml:"reasoning_effort"`
APIKeyEnv string `yaml:"api_key_env"`
ExtraParams map[string]any `yaml:"extra_params"`
BackendID string `json:"backend_id,omitempty"`
Endpoint string `json:"endpoint"`
Model string `json:"model"`
Temperature float64 `json:"temperature"`
MaxTokens int `json:"max_tokens"`
TopP float64 `json:"top_p"`
TimeoutSeconds int `json:"timeout_seconds"`
ServiceTier string `json:"service_tier"`
ReasoningEffort string `json:"reasoning_effort"`
APIKeyEnv string `json:"api_key_env"`
ExtraParams map[string]any `json:"extra_params"`
}
type smokeFactsDTO struct {
CredentialsSupplied *bool `yaml:"credentials_supplied"`
EndpointReachable *bool `yaml:"endpoint_reachable"`
CredentialsAccepted *bool `yaml:"credentials_accepted"`
ResponseReceived *bool `yaml:"response_received"`
GenerationCompleted *bool `yaml:"generation_completed"`
TextReturned *bool `yaml:"text_returned"`
ExpectedTextMatched *bool `yaml:"expected_text_matched"`
StructuredOutputRequested *bool `yaml:"structured_output_requested"`
ValidJSON *bool `yaml:"valid_json"`
SchemaMatched *bool `yaml:"schema_matched"`
CredentialsSupplied *bool `json:"credentials_supplied"`
EndpointReachable *bool `json:"endpoint_reachable"`
CredentialsAccepted *bool `json:"credentials_accepted"`
ResponseReceived *bool `json:"response_received"`
GenerationCompleted *bool `json:"generation_completed"`
TextReturned *bool `json:"text_returned"`
ExpectedTextMatched *bool `json:"expected_text_matched"`
StructuredOutputRequested *bool `json:"structured_output_requested"`
ValidJSON *bool `json:"valid_json"`
SchemaMatched *bool `json:"schema_matched"`
}
type smokeTimingDTO struct {
StartedAt time.Time `yaml:"started_at"`
Duration string `yaml:"duration"`
CapacityWait *string `yaml:"capacity_wait"`
RequestDuration *string `yaml:"request_duration"`
RetryDelay *string `yaml:"retry_delay"`
EvaluationDuration *string `yaml:"evaluation_duration"`
StartedAt time.Time `json:"started_at"`
Duration string `json:"duration"`
CapacityWait *string `json:"capacity_wait"`
RequestDuration *string `json:"request_duration"`
RetryDelay *string `json:"retry_delay"`
EvaluationDuration *string `json:"evaluation_duration"`
}
type smokeAttemptDTO struct {
Dispatched bool `yaml:"dispatched"`
InitialEndpoint string `yaml:"initial_endpoint"`
Endpoint string `yaml:"endpoint"`
HTTPStatus *int `yaml:"http_status"`
Facts smokeFactsDTO `yaml:"facts"`
Timing smokeTimingDTO `yaml:"timing"`
Failure promptkit.ExecutionFailureKind `yaml:"failure"`
Usage smokeUsageDTO `yaml:"usage"`
Dispatched bool `json:"dispatched"`
InitialEndpoint string `json:"initial_endpoint"`
Endpoint string `json:"endpoint"`
HTTPStatus *int `json:"http_status"`
Facts smokeFactsDTO `json:"facts"`
Timing smokeTimingDTO `json:"timing"`
Failure promptkit.ExecutionFailureKind `json:"failure"`
Usage smokeUsageDTO `json:"usage"`
}
type smokeUsageDTO struct {
PromptTokens promptkit.ObservedTokenCount `yaml:"prompt_tokens"`
CompletionTokens promptkit.ObservedTokenCount `yaml:"completion_tokens"`
TotalTokens promptkit.ObservedTokenCount `yaml:"total_tokens"`
CachedTokens promptkit.ObservedTokenCount `yaml:"cached_tokens"`
CacheWriteTokens promptkit.ObservedTokenCount `yaml:"cache_write_tokens"`
PromptTokens smokeTokenCount `json:"prompt_tokens"`
CompletionTokens smokeTokenCount `json:"completion_tokens"`
TotalTokens smokeTokenCount `json:"total_tokens"`
CachedTokens smokeTokenCount `json:"cached_tokens"`
CacheWriteTokens smokeTokenCount `json:"cache_write_tokens"`
}
type smokeTokenCount struct {
Value int `json:"value"`
Known bool `json:"known"`
Complete bool `json:"complete"`
}
type smokeAggregateDTO struct {
Tests []smokeResultDTO `yaml:"tests"`
Tests []smokeResultDTO `json:"tests"`
}
type smokeResultDTO struct {
Kind promptkit.SmokeTestKind `yaml:"kind"`
Report *smokeReportDTO `yaml:"report"`
Error *string `yaml:"error,omitempty"`
Kind promptkit.SmokeTestKind `json:"kind"`
Report *smokeReportDTO `json:"report"`
Error *string `json:"error,omitempty"`
}
type smokeResult struct {
@@ -109,23 +115,16 @@ type smokeResult struct {
Err error
}
func encodeSmokeReport(report *promptkit.SmokeTestReport) ([]byte, error) {
return encodePromptYAMLBytes(smokeReportFromNative(report))
}
func encodeSmokeResults(results []smokeResult) ([]byte, error) {
func smokeResultsReport(results []smokeResult) smokeAggregateDTO {
entries := make([]smokeResultDTO, len(results))
for index, result := range results {
entries[index] = smokeResultDTO{
Kind: result.Kind,
Report: smokeReportFromNative(result.Report),
}
entries[index] = smokeResultDTO{Kind: result.Kind, Report: smokeReportFromNative(result.Report)}
if result.Err != nil {
message := result.Err.Error()
entries[index].Error = &message
}
}
return encodePromptYAMLBytes(smokeAggregateDTO{Tests: entries})
return smokeAggregateDTO{Tests: entries}
}
func smokeReportFromNative(report *promptkit.SmokeTestReport) *smokeReportDTO {
@@ -224,10 +223,10 @@ func durationString(duration *time.Duration) *string {
func smokeUsageFromNative(usage promptkit.ObservedTokenUsage) smokeUsageDTO {
return smokeUsageDTO{
PromptTokens: usage.PromptTokens,
CompletionTokens: usage.CompletionTokens,
TotalTokens: usage.TotalTokens,
CachedTokens: usage.CachedTokens,
CacheWriteTokens: usage.CacheWriteTokens,
PromptTokens: smokeTokenCount(usage.PromptTokens),
CompletionTokens: smokeTokenCount(usage.CompletionTokens),
TotalTokens: smokeTokenCount(usage.TotalTokens),
CachedTokens: smokeTokenCount(usage.CachedTokens),
CacheWriteTokens: smokeTokenCount(usage.CacheWriteTokens),
}
}