commandkit/integrations/ckprompt/smoke_report.go

package ckprompt

import (
	"time"

	promptkit "gitea.maximumdirect.net/eric/promptkit"
)

type smokeReportDTO struct {
	Kind       promptkit.SmokeTestKind        `yaml:"kind"`
	Supported  bool                           `yaml:"supported"`
	Applicable *bool                          `yaml:"applicable"`
	Completed  bool                           `yaml:"completed"`
	Reason     promptkit.SmokeTestReason      `yaml:"reason"`
	Target     smokeTargetDTO                 `yaml:"target"`
	Facts      smokeFactsDTO                  `yaml:"facts"`
	Timing     smokeTimingDTO                 `yaml:"timing"`
	Attempts   []smokeAttemptDTO              `yaml:"attempts"`
	Usage      smokeUsageDTO                  `yaml:"usage"`
	Failure    promptkit.ExecutionFailureKind `yaml:"failure"`
}

type smokeTargetDTO struct {
	BackendID              string                   `yaml:"backend_id"`
	ProfileID              string                   `yaml:"profile_id"`
	PromptID               string                   `yaml:"prompt_id"`
	PromptVersion          *string                  `yaml:"version,omitempty"`
	DefaultSourceBackendID string                   `yaml:"default_source_backend_id"`
	SelectionSource        string                   `yaml:"selection_source"`
	Endpoint               string                   `yaml:"endpoint"`
	EffectiveSettings      smokeExecutionTargetDTO  `yaml:"effective_settings"`
	SettingsPresence       smokeSettingsPresenceDTO `yaml:"settings_presence"`
}

type smokeSettingsPresenceDTO struct {
	Temperature    bool `yaml:"temperature"`
	MaxTokens      bool `yaml:"max_tokens"`
	TopP           bool `yaml:"top_p"`
	TimeoutSeconds bool `yaml:"timeout_seconds"`
}

type smokeExecutionTargetDTO struct {
	BackendID       string         `yaml:"backend_id,omitempty"`
	Endpoint        string         `yaml:"endpoint"`
	Model           string         `yaml:"model"`
	Temperature     float64        `yaml:"temperature"`
	MaxTokens       int            `yaml:"max_tokens"`
	TopP            float64        `yaml:"top_p"`
	TimeoutSeconds  int            `yaml:"timeout_seconds"`
	ServiceTier     string         `yaml:"service_tier"`
	ReasoningEffort string         `yaml:"reasoning_effort"`
	APIKeyEnv       string         `yaml:"api_key_env"`
	ExtraParams     map[string]any `yaml:"extra_params"`
}

type smokeFactsDTO struct {
	CredentialsSupplied       *bool `yaml:"credentials_supplied"`
	EndpointReachable         *bool `yaml:"endpoint_reachable"`
	CredentialsAccepted       *bool `yaml:"credentials_accepted"`
	ResponseReceived          *bool `yaml:"response_received"`
	GenerationCompleted       *bool `yaml:"generation_completed"`
	TextReturned              *bool `yaml:"text_returned"`
	ExpectedTextMatched       *bool `yaml:"expected_text_matched"`
	StructuredOutputRequested *bool `yaml:"structured_output_requested"`
	ValidJSON                 *bool `yaml:"valid_json"`
	SchemaMatched             *bool `yaml:"schema_matched"`
}

type smokeTimingDTO struct {
	StartedAt          time.Time `yaml:"started_at"`
	Duration           string    `yaml:"duration"`
	CapacityWait       *string   `yaml:"capacity_wait"`
	RequestDuration    *string   `yaml:"request_duration"`
	RetryDelay         *string   `yaml:"retry_delay"`
	EvaluationDuration *string   `yaml:"evaluation_duration"`
}

type smokeAttemptDTO struct {
	Dispatched      bool                           `yaml:"dispatched"`
	InitialEndpoint string                         `yaml:"initial_endpoint"`
	Endpoint        string                         `yaml:"endpoint"`
	HTTPStatus      *int                           `yaml:"http_status"`
	Facts           smokeFactsDTO                  `yaml:"facts"`
	Timing          smokeTimingDTO                 `yaml:"timing"`
	Failure         promptkit.ExecutionFailureKind `yaml:"failure"`
	Usage           smokeUsageDTO                  `yaml:"usage"`
}

type smokeUsageDTO struct {
	PromptTokens     promptkit.ObservedTokenCount `yaml:"prompt_tokens"`
	CompletionTokens promptkit.ObservedTokenCount `yaml:"completion_tokens"`
	TotalTokens      promptkit.ObservedTokenCount `yaml:"total_tokens"`
	CachedTokens     promptkit.ObservedTokenCount `yaml:"cached_tokens"`
	CacheWriteTokens promptkit.ObservedTokenCount `yaml:"cache_write_tokens"`
}

type smokeAggregateDTO struct {
	Tests []smokeResultDTO `yaml:"tests"`
}

type smokeResultDTO struct {
	Kind   promptkit.SmokeTestKind `yaml:"kind"`
	Report *smokeReportDTO         `yaml:"report"`
	Error  *string                 `yaml:"error,omitempty"`
}

type smokeResult struct {
	Kind   promptkit.SmokeTestKind
	Report *promptkit.SmokeTestReport
	Err    error
}

func encodeSmokeReport(report *promptkit.SmokeTestReport) ([]byte, error) {
	return encodePromptYAMLBytes(smokeReportFromNative(report))
}

func encodeSmokeResults(results []smokeResult) ([]byte, error) {
	entries := make([]smokeResultDTO, len(results))
	for index, result := range results {
		entries[index] = smokeResultDTO{
			Kind:   result.Kind,
			Report: smokeReportFromNative(result.Report),
		}
		if result.Err != nil {
			message := result.Err.Error()
			entries[index].Error = &message
		}
	}
	return encodePromptYAMLBytes(smokeAggregateDTO{Tests: entries})
}

func smokeReportFromNative(report *promptkit.SmokeTestReport) *smokeReportDTO {
	if report == nil {
		return nil
	}
	attempts := make([]smokeAttemptDTO, len(report.Attempts))
	for index, attempt := range report.Attempts {
		attempts[index] = smokeAttemptDTO{
			Dispatched:      attempt.Dispatched,
			InitialEndpoint: attempt.InitialEndpoint,
			Endpoint:        attempt.Endpoint,
			HTTPStatus:      attempt.HTTPStatus,
			Facts:           smokeFactsFromNative(attempt.Facts),
			Timing:          smokeTimingFromNative(attempt.Timing),
			Failure:         attempt.Failure,
			Usage:           smokeUsageFromNative(attempt.Usage),
		}
	}
	target := report.Target
	settings := target.EffectiveSettings
	return &smokeReportDTO{
		Kind:       report.Kind,
		Supported:  report.Supported,
		Applicable: report.Applicable,
		Completed:  report.Completed,
		Reason:     report.Reason,
		Target: smokeTargetDTO{
			BackendID:              target.BackendID,
			ProfileID:              target.ProfileID,
			PromptID:               target.PromptID,
			PromptVersion:          target.PromptVersion,
			DefaultSourceBackendID: target.DefaultSourceBackendID,
			SelectionSource:        target.SelectionSource,
			Endpoint:               target.Endpoint,
			EffectiveSettings: smokeExecutionTargetDTO{
				BackendID:       settings.BackendID,
				Endpoint:        settings.Endpoint,
				Model:           settings.Model,
				Temperature:     settings.Temperature,
				MaxTokens:       settings.MaxTokens,
				TopP:            settings.TopP,
				TimeoutSeconds:  settings.TimeoutSeconds,
				ServiceTier:     settings.ServiceTier,
				ReasoningEffort: settings.ReasoningEffort,
				APIKeyEnv:       settings.APIKeyEnv,
				ExtraParams:     settings.ExtraParams,
			},
			SettingsPresence: smokeSettingsPresenceDTO{
				Temperature:    target.SettingsPresence.Temperature,
				MaxTokens:      target.SettingsPresence.MaxTokens,
				TopP:           target.SettingsPresence.TopP,
				TimeoutSeconds: target.SettingsPresence.TimeoutSeconds,
			},
		},
		Facts:    smokeFactsFromNative(report.Facts),
		Timing:   smokeTimingFromNative(report.Timing),
		Attempts: attempts,
		Usage:    smokeUsageFromNative(report.Usage),
		Failure:  report.Failure,
	}
}

func smokeFactsFromNative(facts promptkit.SmokeTestFacts) smokeFactsDTO {
	return smokeFactsDTO{
		CredentialsSupplied:       facts.CredentialsSupplied,
		EndpointReachable:         facts.EndpointReachable,
		CredentialsAccepted:       facts.CredentialsAccepted,
		ResponseReceived:          facts.ResponseReceived,
		GenerationCompleted:       facts.GenerationCompleted,
		TextReturned:              facts.TextReturned,
		ExpectedTextMatched:       facts.ExpectedTextMatched,
		StructuredOutputRequested: facts.StructuredOutputRequested,
		ValidJSON:                 facts.ValidJSON,
		SchemaMatched:             facts.SchemaMatched,
	}
}

func smokeTimingFromNative(timing promptkit.SmokeTestTiming) smokeTimingDTO {
	return smokeTimingDTO{
		StartedAt:          timing.StartedAt,
		Duration:           timing.Duration.String(),
		CapacityWait:       durationString(timing.CapacityWait),
		RequestDuration:    durationString(timing.RequestDuration),
		RetryDelay:         durationString(timing.RetryDelay),
		EvaluationDuration: durationString(timing.EvaluationDuration),
	}
}

func durationString(duration *time.Duration) *string {
	if duration == nil {
		return nil
	}
	value := duration.String()
	return &value
}

func smokeUsageFromNative(usage promptkit.ObservedTokenUsage) smokeUsageDTO {
	return smokeUsageDTO{
		PromptTokens:     usage.PromptTokens,
		CompletionTokens: usage.CompletionTokens,
		TotalTokens:      usage.TotalTokens,
		CachedTokens:     usage.CachedTokens,
		CacheWriteTokens: usage.CacheWriteTokens,
	}
}