233 lines
8.6 KiB
Go
233 lines
8.6 KiB
Go
package ckprompt
|
|
|
|
import (
|
|
"time"
|
|
|
|
promptkit "go.maximumdirect.net/promptkit"
|
|
)
|
|
|
|
type smokeReportDTO struct {
|
|
Kind promptkit.SmokeTestKind `json:"kind"`
|
|
Supported bool `json:"supported"`
|
|
Applicable *bool `json:"applicable"`
|
|
Completed bool `json:"completed"`
|
|
Reason promptkit.SmokeTestReason `json:"reason"`
|
|
Target smokeTargetDTO `json:"target"`
|
|
Facts smokeFactsDTO `json:"facts"`
|
|
Timing smokeTimingDTO `json:"timing"`
|
|
Attempts []smokeAttemptDTO `json:"attempts"`
|
|
Usage smokeUsageDTO `json:"usage"`
|
|
Failure promptkit.ExecutionFailureKind `json:"failure"`
|
|
}
|
|
|
|
type smokeTargetDTO struct {
|
|
BackendID string `json:"backend_id"`
|
|
ProfileID string `json:"profile_id"`
|
|
PromptID string `json:"prompt_id"`
|
|
DefaultSourceBackendID string `json:"default_source_backend_id"`
|
|
SelectionSource string `json:"selection_source"`
|
|
Endpoint string `json:"endpoint"`
|
|
EffectiveSettings smokeExecutionTargetDTO `json:"effective_settings"`
|
|
SettingsPresence smokeSettingsPresenceDTO `json:"settings_presence"`
|
|
}
|
|
|
|
type smokeSettingsPresenceDTO struct {
|
|
Temperature bool `json:"temperature"`
|
|
MaxTokens bool `json:"max_tokens"`
|
|
TopP bool `json:"top_p"`
|
|
TimeoutSeconds bool `json:"timeout_seconds"`
|
|
}
|
|
|
|
type smokeExecutionTargetDTO struct {
|
|
BackendID string `json:"backend_id,omitempty"`
|
|
Endpoint string `json:"endpoint"`
|
|
Model string `json:"model"`
|
|
Temperature float64 `json:"temperature"`
|
|
MaxTokens int `json:"max_tokens"`
|
|
TopP float64 `json:"top_p"`
|
|
TimeoutSeconds int `json:"timeout_seconds"`
|
|
ServiceTier string `json:"service_tier"`
|
|
ReasoningEffort string `json:"reasoning_effort"`
|
|
APIKeyEnv string `json:"api_key_env"`
|
|
ExtraParams map[string]any `json:"extra_params"`
|
|
}
|
|
|
|
type smokeFactsDTO struct {
|
|
CredentialsSupplied *bool `json:"credentials_supplied"`
|
|
EndpointReachable *bool `json:"endpoint_reachable"`
|
|
CredentialsAccepted *bool `json:"credentials_accepted"`
|
|
ResponseReceived *bool `json:"response_received"`
|
|
GenerationCompleted *bool `json:"generation_completed"`
|
|
TextReturned *bool `json:"text_returned"`
|
|
ExpectedTextMatched *bool `json:"expected_text_matched"`
|
|
StructuredOutputRequested *bool `json:"structured_output_requested"`
|
|
ValidJSON *bool `json:"valid_json"`
|
|
SchemaMatched *bool `json:"schema_matched"`
|
|
}
|
|
|
|
type smokeTimingDTO struct {
|
|
StartedAt time.Time `json:"started_at"`
|
|
Duration string `json:"duration"`
|
|
CapacityWait *string `json:"capacity_wait"`
|
|
RequestDuration *string `json:"request_duration"`
|
|
RetryDelay *string `json:"retry_delay"`
|
|
EvaluationDuration *string `json:"evaluation_duration"`
|
|
}
|
|
|
|
type smokeAttemptDTO struct {
|
|
Dispatched bool `json:"dispatched"`
|
|
InitialEndpoint string `json:"initial_endpoint"`
|
|
Endpoint string `json:"endpoint"`
|
|
HTTPStatus *int `json:"http_status"`
|
|
Facts smokeFactsDTO `json:"facts"`
|
|
Timing smokeTimingDTO `json:"timing"`
|
|
Failure promptkit.ExecutionFailureKind `json:"failure"`
|
|
Usage smokeUsageDTO `json:"usage"`
|
|
}
|
|
|
|
type smokeUsageDTO struct {
|
|
PromptTokens smokeTokenCount `json:"prompt_tokens"`
|
|
CompletionTokens smokeTokenCount `json:"completion_tokens"`
|
|
TotalTokens smokeTokenCount `json:"total_tokens"`
|
|
CachedTokens smokeTokenCount `json:"cached_tokens"`
|
|
CacheWriteTokens smokeTokenCount `json:"cache_write_tokens"`
|
|
}
|
|
|
|
type smokeTokenCount struct {
|
|
Value int `json:"value"`
|
|
Known bool `json:"known"`
|
|
Complete bool `json:"complete"`
|
|
}
|
|
|
|
type smokeAggregateDTO struct {
|
|
Tests []smokeResultDTO `json:"tests"`
|
|
}
|
|
|
|
type smokeResultDTO struct {
|
|
Kind promptkit.SmokeTestKind `json:"kind"`
|
|
Report *smokeReportDTO `json:"report"`
|
|
Error *string `json:"error,omitempty"`
|
|
}
|
|
|
|
type smokeResult struct {
|
|
Kind promptkit.SmokeTestKind
|
|
Report *promptkit.SmokeTestReport
|
|
Err error
|
|
}
|
|
|
|
func smokeResultsReport(results []smokeResult) smokeAggregateDTO {
|
|
entries := make([]smokeResultDTO, len(results))
|
|
for index, result := range results {
|
|
entries[index] = smokeResultDTO{Kind: result.Kind, Report: smokeReportFromNative(result.Report)}
|
|
if result.Err != nil {
|
|
message := result.Err.Error()
|
|
entries[index].Error = &message
|
|
}
|
|
}
|
|
return smokeAggregateDTO{Tests: entries}
|
|
}
|
|
|
|
func smokeReportFromNative(report *promptkit.SmokeTestReport) *smokeReportDTO {
|
|
if report == nil {
|
|
return nil
|
|
}
|
|
attempts := make([]smokeAttemptDTO, len(report.Attempts))
|
|
for index, attempt := range report.Attempts {
|
|
attempts[index] = smokeAttemptDTO{
|
|
Dispatched: attempt.Dispatched,
|
|
InitialEndpoint: attempt.InitialEndpoint,
|
|
Endpoint: attempt.Endpoint,
|
|
HTTPStatus: attempt.HTTPStatus,
|
|
Facts: smokeFactsFromNative(attempt.Facts),
|
|
Timing: smokeTimingFromNative(attempt.Timing),
|
|
Failure: attempt.Failure,
|
|
Usage: smokeUsageFromNative(attempt.Usage),
|
|
}
|
|
}
|
|
target := report.Target
|
|
settings := target.EffectiveSettings
|
|
return &smokeReportDTO{
|
|
Kind: report.Kind,
|
|
Supported: report.Supported,
|
|
Applicable: report.Applicable,
|
|
Completed: report.Completed,
|
|
Reason: report.Reason,
|
|
Target: smokeTargetDTO{
|
|
BackendID: target.BackendID,
|
|
ProfileID: target.ProfileID,
|
|
PromptID: target.PromptID,
|
|
DefaultSourceBackendID: target.DefaultSourceBackendID,
|
|
SelectionSource: target.SelectionSource,
|
|
Endpoint: target.Endpoint,
|
|
EffectiveSettings: smokeExecutionTargetDTO{
|
|
BackendID: settings.BackendID,
|
|
Endpoint: settings.Endpoint,
|
|
Model: settings.Model,
|
|
Temperature: settings.Temperature,
|
|
MaxTokens: settings.MaxTokens,
|
|
TopP: settings.TopP,
|
|
TimeoutSeconds: settings.TimeoutSeconds,
|
|
ServiceTier: settings.ServiceTier,
|
|
ReasoningEffort: settings.ReasoningEffort,
|
|
APIKeyEnv: settings.APIKeyEnv,
|
|
ExtraParams: settings.ExtraParams,
|
|
},
|
|
SettingsPresence: smokeSettingsPresenceDTO{
|
|
Temperature: target.SettingsPresence.Temperature,
|
|
MaxTokens: target.SettingsPresence.MaxTokens,
|
|
TopP: target.SettingsPresence.TopP,
|
|
TimeoutSeconds: target.SettingsPresence.TimeoutSeconds,
|
|
},
|
|
},
|
|
Facts: smokeFactsFromNative(report.Facts),
|
|
Timing: smokeTimingFromNative(report.Timing),
|
|
Attempts: attempts,
|
|
Usage: smokeUsageFromNative(report.Usage),
|
|
Failure: report.Failure,
|
|
}
|
|
}
|
|
|
|
func smokeFactsFromNative(facts promptkit.SmokeTestFacts) smokeFactsDTO {
|
|
return smokeFactsDTO{
|
|
CredentialsSupplied: facts.CredentialsSupplied,
|
|
EndpointReachable: facts.EndpointReachable,
|
|
CredentialsAccepted: facts.CredentialsAccepted,
|
|
ResponseReceived: facts.ResponseReceived,
|
|
GenerationCompleted: facts.GenerationCompleted,
|
|
TextReturned: facts.TextReturned,
|
|
ExpectedTextMatched: facts.ExpectedTextMatched,
|
|
StructuredOutputRequested: facts.StructuredOutputRequested,
|
|
ValidJSON: facts.ValidJSON,
|
|
SchemaMatched: facts.SchemaMatched,
|
|
}
|
|
}
|
|
|
|
func smokeTimingFromNative(timing promptkit.SmokeTestTiming) smokeTimingDTO {
|
|
return smokeTimingDTO{
|
|
StartedAt: timing.StartedAt,
|
|
Duration: timing.Duration.String(),
|
|
CapacityWait: durationString(timing.CapacityWait),
|
|
RequestDuration: durationString(timing.RequestDuration),
|
|
RetryDelay: durationString(timing.RetryDelay),
|
|
EvaluationDuration: durationString(timing.EvaluationDuration),
|
|
}
|
|
}
|
|
|
|
func durationString(duration *time.Duration) *string {
|
|
if duration == nil {
|
|
return nil
|
|
}
|
|
value := duration.String()
|
|
return &value
|
|
}
|
|
|
|
func smokeUsageFromNative(usage promptkit.ObservedTokenUsage) smokeUsageDTO {
|
|
return smokeUsageDTO{
|
|
PromptTokens: smokeTokenCount(usage.PromptTokens),
|
|
CompletionTokens: smokeTokenCount(usage.CompletionTokens),
|
|
TotalTokens: smokeTokenCount(usage.TotalTokens),
|
|
CachedTokens: smokeTokenCount(usage.CachedTokens),
|
|
CacheWriteTokens: smokeTokenCount(usage.CacheWriteTokens),
|
|
}
|
|
}
|