Files
audita/internal/framework/llm/diagnostics_test.go
Eric Rakestraw da5b8913d1
All checks were successful
ci/woodpecker/tag/release Pipeline was successful
Migrate Audita to the v4 module path
2026-11-17 19:57:22 +00:00

89 lines
2.9 KiB
Go

package llm
import (
"encoding/json"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"gitea.maximumdirect.net/eric/audita/v4/internal/framework/structuredoutput"
)
func TestDiagnosticsWriterProducesValidJSONArtifacts(t *testing.T) {
dir := t.TempDir()
writer := NewDiagnosticsWriter(dir)
artifacts, err := writer.WriteInteraction(structuredoutput.Interaction{
Name: "validation:batch-1",
Context: structuredoutput.DiagnosticContext{Role: "validation", ModuleInstance: "grammar_1"},
RequestMetadata: map[string]any{"attempt": 1},
RequestPayload: map[string]any{"messages": []map[string]any{{"role": "user", "content": "hello"}}},
ResponsePayload: map[string]any{"result": "ok"},
})
if err != nil {
t.Fatalf("WriteInteraction: %v", err)
}
if artifacts.ErrorPayloadPath != "" {
t.Fatalf("expected no error payload path, got %q", artifacts.ErrorPayloadPath)
}
assertValidJSONFile(t, artifacts.RequestMetadataPath)
assertValidJSONFile(t, artifacts.RequestPayloadPath)
assertValidJSONFile(t, artifacts.ResponsePayloadPath)
}
func TestDiagnosticsWriterGroupsFailuresAndContinuesCapture(t *testing.T) {
dir := t.TempDir()
writer := NewDiagnosticsWriter(dir)
var workers sync.WaitGroup
for _, name := range []string{"first", "second"} {
blocked := filepath.Join(dir, "grammar_1", name+"-response-payload.json")
if err := os.MkdirAll(blocked, 0o755); err != nil {
t.Fatal(err)
}
workers.Go(func() {
paths, err := writer.WriteInteraction(structuredoutput.Interaction{
Name: name, Context: structuredoutput.DiagnosticContext{Role: "validation", ModuleInstance: "grammar_1"},
ErrorPayload: map[string]string{"error": "model unavailable"},
})
if err == nil || paths.RequestPayloadPath == "" || paths.ResponsePayloadPath != "" || paths.ErrorPayloadPath == "" {
t.Errorf("paths=%+v error=%v", paths, err)
}
})
}
workers.Wait()
warnings := writer.Warnings()
if len(warnings) != 1 || !strings.Contains(warnings[0].Error(), "validation interaction: write response-payload failures (2)") ||
!strings.Contains(warnings[0].Error(), filepath.Join(dir, "grammar_1")) {
t.Fatalf("warnings: %v", warnings)
}
paths, err := writer.WriteInteraction(structuredoutput.Interaction{
Name: "later", Context: structuredoutput.DiagnosticContext{Role: "validation", ModuleInstance: "grammar_1"},
ResponsePayload: map[string]string{"final_content": "{"},
})
if err != nil {
t.Fatal(err)
}
raw, err := os.ReadFile(paths.ResponsePayloadPath)
if err != nil {
t.Fatal(err)
}
var decoded map[string]string
if err := json.Unmarshal(raw, &decoded); err != nil || decoded["final_content"] != "{" {
t.Fatalf("later artifact: %s (%v)", raw, err)
}
}
func assertValidJSONFile(t *testing.T, path string) {
t.Helper()
raw, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read %s: %v", path, err)
}
if !json.Valid(raw) {
t.Fatalf("invalid json in %s", filepath.Base(path))
}
}