2026-05-11 20:09:34 -05:00
|
|
|
package llm
|
|
|
|
|
|
|
|
|
|
import (
|
|
|
|
|
"encoding/json"
|
|
|
|
|
"os"
|
|
|
|
|
"path/filepath"
|
2026-11-16 03:43:29 +00:00
|
|
|
"strings"
|
|
|
|
|
"sync"
|
2026-05-11 20:09:34 -05:00
|
|
|
"testing"
|
2026-11-16 03:43:29 +00:00
|
|
|
|
|
|
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
2026-05-11 20:09:34 -05:00
|
|
|
)
|
|
|
|
|
|
|
|
|
|
func TestDiagnosticsWriterProducesValidJSONArtifacts(t *testing.T) {
|
|
|
|
|
dir := t.TempDir()
|
2026-11-16 03:37:29 +00:00
|
|
|
writer := NewDiagnosticsWriter(dir)
|
2026-05-11 20:09:34 -05:00
|
|
|
|
2026-11-16 03:43:29 +00:00
|
|
|
artifacts, err := writer.WriteInteraction(structuredoutput.Interaction{
|
|
|
|
|
Name: "validation:batch-1",
|
|
|
|
|
Context: structuredoutput.DiagnosticContext{Role: "validation", ModuleInstance: "grammar_1"},
|
|
|
|
|
RequestMetadata: map[string]any{"attempt": 1},
|
|
|
|
|
RequestPayload: map[string]any{"messages": []map[string]any{{"role": "user", "content": "hello"}}},
|
|
|
|
|
ResponsePayload: map[string]any{"result": "ok"},
|
|
|
|
|
})
|
2026-05-11 20:09:34 -05:00
|
|
|
if err != nil {
|
|
|
|
|
t.Fatalf("WriteInteraction: %v", err)
|
|
|
|
|
}
|
|
|
|
|
if artifacts.ErrorPayloadPath != "" {
|
|
|
|
|
t.Fatalf("expected no error payload path, got %q", artifacts.ErrorPayloadPath)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
assertValidJSONFile(t, artifacts.RequestMetadataPath)
|
|
|
|
|
assertValidJSONFile(t, artifacts.RequestPayloadPath)
|
|
|
|
|
assertValidJSONFile(t, artifacts.ResponsePayloadPath)
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-16 03:43:29 +00:00
|
|
|
func TestDiagnosticsWriterGroupsFailuresAndContinuesCapture(t *testing.T) {
|
|
|
|
|
dir := t.TempDir()
|
|
|
|
|
writer := NewDiagnosticsWriter(dir)
|
|
|
|
|
var workers sync.WaitGroup
|
|
|
|
|
for _, name := range []string{"first", "second"} {
|
|
|
|
|
blocked := filepath.Join(dir, "grammar_1", name+"-response-payload.json")
|
|
|
|
|
if err := os.MkdirAll(blocked, 0o755); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
workers.Go(func() {
|
|
|
|
|
paths, err := writer.WriteInteraction(structuredoutput.Interaction{
|
|
|
|
|
Name: name, Context: structuredoutput.DiagnosticContext{Role: "validation", ModuleInstance: "grammar_1"},
|
|
|
|
|
ErrorPayload: map[string]string{"error": "model unavailable"},
|
|
|
|
|
})
|
|
|
|
|
if err == nil || paths.RequestPayloadPath == "" || paths.ResponsePayloadPath != "" || paths.ErrorPayloadPath == "" {
|
|
|
|
|
t.Errorf("paths=%+v error=%v", paths, err)
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
workers.Wait()
|
|
|
|
|
warnings := writer.Warnings()
|
|
|
|
|
if len(warnings) != 1 || !strings.Contains(warnings[0].Error(), "validation interaction: write response-payload failures (2)") ||
|
|
|
|
|
!strings.Contains(warnings[0].Error(), filepath.Join(dir, "grammar_1")) {
|
|
|
|
|
t.Fatalf("warnings: %v", warnings)
|
|
|
|
|
}
|
|
|
|
|
paths, err := writer.WriteInteraction(structuredoutput.Interaction{
|
|
|
|
|
Name: "later", Context: structuredoutput.DiagnosticContext{Role: "validation", ModuleInstance: "grammar_1"},
|
|
|
|
|
ResponsePayload: map[string]string{"final_content": "{"},
|
|
|
|
|
})
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
raw, err := os.ReadFile(paths.ResponsePayloadPath)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
var decoded map[string]string
|
|
|
|
|
if err := json.Unmarshal(raw, &decoded); err != nil || decoded["final_content"] != "{" {
|
|
|
|
|
t.Fatalf("later artifact: %s (%v)", raw, err)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-05-11 20:09:34 -05:00
|
|
|
func assertValidJSONFile(t *testing.T, path string) {
|
|
|
|
|
t.Helper()
|
|
|
|
|
raw, err := os.ReadFile(path)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatalf("read %s: %v", path, err)
|
|
|
|
|
}
|
|
|
|
|
if !json.Valid(raw) {
|
|
|
|
|
t.Fatalf("invalid json in %s", filepath.Base(path))
|
|
|
|
|
}
|
|
|
|
|
}
|