base-layer/commandkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Preserve empty smoke error messages in aggregate reports

Browse Source
This commit is contained in:
Eric Rakestraw
2026-10-18 13:28:29 +00:00
parent 0fe77baa29
commit 1d0085450a
4 changed files with 25 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
integrations/ckprompt/smoke_outcome_test.go
View File

@@ -141,7 +141,7 @@ func (err *smokeTypedError) Error() string { return err.text }
func TestSmokeAggregatePreservesMultipleTypedCauses(t *testing.T) {
firstErr := &smokeTypedError{text: "reachability failed"}
secondErr := &smokeTypedError{text: "authentication failed"}
secondErr := &smokeTypedError{text: ""}
app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{
SmokeTest: func(_ context.Context, request SmokeRequest, _ struct{}, _ struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
var err error
@@ -162,14 +162,14 @@ func TestSmokeAggregatePreservesMultipleTypedCauses(t *testing.T) {
}
var document struct {
Tests []struct {
Kind string `yaml:"kind"`
Error string `yaml:"error"`
Kind string `yaml:"kind"`
Error *string `yaml:"error"`
} `yaml:"tests"`
}
if err := yaml.Unmarshal(output.Bytes(), &document); err != nil {
t.Fatalf("decode aggregate errors %q: %v", output.String(), err)
}
if len(document.Tests) != 4 || document.Tests[0].Kind != string(promptkit.SmokeTestReachability) || document.Tests[0].Error != firstErr.Error() || document.Tests[1].Kind != string(promptkit.SmokeTestAuthentication) || document.Tests[1].Error != secondErr.Error() || document.Tests[2].Error != "" || document.Tests[3].Error != "" {
if len(document.Tests) != 4 || document.Tests[0].Kind != string(promptkit.SmokeTestReachability) || document.Tests[0].Error == nil || *document.Tests[0].Error != firstErr.Error() || document.Tests[1].Kind != string(promptkit.SmokeTestAuthentication) || document.Tests[1].Error == nil || *document.Tests[1].Error != secondErr.Error() || document.Tests[2].Error != nil || document.Tests[3].Error != nil {
t.Fatalf("aggregate error entries = %#v", document.Tests)
}
}

5
integrations/ckprompt/smoke_report.go
View File

@@ -101,7 +101,7 @@ type smokeAggregateDTO struct {
type smokeResultDTO struct {
Kind promptkit.SmokeTestKind `yaml:"kind"`
Report *smokeReportDTO `yaml:"report"`
Error string `yaml:"error,omitempty"`
Error *string `yaml:"error,omitempty"`
}
type smokeResult struct {
@@ -122,7 +122,8 @@ func encodeSmokeResults(results []smokeResult) ([]byte, error) {
Report: smokeReportFromNative(result.Report),
}
if result.Err != nil {
entries[index].Error = result.Err.Error()
message := result.Err.Error()
entries[index].Error = &message
}
}
return encodePromptYAMLBytes(smokeAggregateDTO{Tests: entries})

7
integrations/ckprompt/smoke_report_test.go
View File

@@ -163,6 +163,7 @@ func TestSmokeAggregateYAMLRetainsOrderErrorAndNativeReportKind(t *testing.T) {
encoded, err := encodeSmokeResults([]smokeResult{
{Kind: promptkit.SmokeTestTextInference, Report: report, Err: operationErr},
{Kind: promptkit.SmokeTestAuthentication},
{Kind: promptkit.SmokeTestStructuredOutput, Report: report, Err: errors.New("")},
})
if err != nil {
t.Fatal(err)
@@ -172,7 +173,7 @@ func TestSmokeAggregateYAMLRetainsOrderErrorAndNativeReportKind(t *testing.T) {
t.Fatalf("decode aggregate report: %v\n%s", err, encoded)
}
tests, ok := document["tests"].([]any)
if !ok || len(tests) != 2 {
if !ok || len(tests) != 3 {
t.Fatalf("aggregate entries = %#v", document)
}
first := tests[0].(map[string]any)
@@ -190,6 +191,10 @@ func TestSmokeAggregateYAMLRetainsOrderErrorAndNativeReportKind(t *testing.T) {
if _, exists := second["error"]; exists {
t.Fatalf("nil error should be omitted: %#v", second)
}
third := tests[2].(map[string]any)
if message, exists := third["error"]; !exists || message != "" {
t.Fatalf("empty error message should remain present: %#v", third)
}
}
func containsSmokeKey(value any, want string) bool {