Unify optional interaction capture and warnings

This commit is contained in:
2026-11-16 03:43:29 +00:00
parent ffe0e3491e
commit cea7e44232
22 changed files with 450 additions and 367 deletions

View File

@@ -2,68 +2,120 @@ package llm
import (
"encoding/json"
"errors"
"fmt"
"maps"
"os"
"path/filepath"
"regexp"
"strings"
"sync"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
)
var stageSanitizer = regexp.MustCompile(`[^a-zA-Z0-9_.-]+`)
// InteractionArtifacts contains file paths for written prompt/response
// diagnostics.
type InteractionArtifacts struct {
RequestMetadataPath string
RequestPayloadPath string
ResponsePayloadPath string
ErrorPayloadPath string
}
// DiagnosticsWriter writes generic machine-readable LLM interaction artifacts.
// DiagnosticsWriter writes interaction artifacts under a single run directory.
type DiagnosticsWriter struct {
dir string
dir string
mu sync.Mutex
failures map[interactionFailureKey]*interactionFailure
order []interactionFailureKey
}
type interactionFailureKey struct {
role, action, cause string
}
type interactionFailure struct {
count int
path string
}
// NewDiagnosticsWriter creates a diagnostics writer rooted at dir.
func NewDiagnosticsWriter(dir string) *DiagnosticsWriter {
return &DiagnosticsWriter{dir: dir}
}
// WriteInteraction writes request/response/error artifacts for a generic LLM
// interaction stage.
func (w *DiagnosticsWriter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error) {
func (w *DiagnosticsWriter) WriteInteraction(interaction structuredoutput.Interaction) (structuredoutput.InteractionArtifacts, error) {
var paths structuredoutput.InteractionArtifacts
dir := w.dir
if instance := interaction.Context.ModuleInstance; instance != "" {
dir = filepath.Join(dir, sanitizeStageName(instance))
}
var err error
if strings.TrimSpace(w.dir) == "" {
return InteractionArtifacts{}, fmt.Errorf("diagnostics directory must not be empty")
err = fmt.Errorf("diagnostics directory must not be empty")
} else {
err = os.MkdirAll(dir, 0o755)
}
if err := os.MkdirAll(w.dir, 0o755); err != nil {
return InteractionArtifacts{}, fmt.Errorf("create diagnostics directory: %w", err)
base := sanitizeStageName(interaction.Name)
if err != nil {
w.recordFailure(interaction.Context.Role, "create directory", filepath.Join(dir, base), err)
return paths, fmt.Errorf("create diagnostics directory %q: %w", dir, err)
}
base := sanitizeStageName(stage)
paths := InteractionArtifacts{
RequestMetadataPath: filepath.Join(w.dir, base+"-request-metadata.json"),
RequestPayloadPath: filepath.Join(w.dir, base+"-request-payload.json"),
ResponsePayloadPath: filepath.Join(w.dir, base+"-response-payload.json"),
metadata := maps.Clone(interaction.RequestMetadata)
if metadata == nil {
metadata = make(map[string]any)
}
if err := w.writeJSON(paths.RequestMetadataPath, requestMetadata); err != nil {
return InteractionArtifacts{}, err
metadata["context"] = interaction.Context
type artifactFile struct {
suffix string
payload any
path *string
}
if err := w.writeJSON(paths.RequestPayloadPath, requestPayload); err != nil {
return InteractionArtifacts{}, err
artifacts := []artifactFile{
{"request-metadata", metadata, &paths.RequestMetadataPath},
{"request-payload", interaction.RequestPayload, &paths.RequestPayloadPath},
{"response-payload", interaction.ResponsePayload, &paths.ResponsePayloadPath},
}
if err := w.writeJSON(paths.ResponsePayloadPath, responsePayload); err != nil {
return InteractionArtifacts{}, err
if interaction.ErrorPayload != nil {
artifacts = append(artifacts, artifactFile{"error-payload", interaction.ErrorPayload, &paths.ErrorPayloadPath})
}
if errorPayload != nil {
paths.ErrorPayloadPath = filepath.Join(w.dir, base+"-error-payload.json")
if err := w.writeJSON(paths.ErrorPayloadPath, errorPayload); err != nil {
return InteractionArtifacts{}, err
var failures []error
for _, artifact := range artifacts {
path := filepath.Join(dir, base+"-"+artifact.suffix+".json")
if action, err := writeInteractionJSON(path, artifact.payload); err != nil {
w.recordFailure(interaction.Context.Role, action+" "+artifact.suffix, path, err)
failures = append(failures, fmt.Errorf("%s %q: %w", action, path, err))
continue
}
*artifact.path = path
}
return paths, errors.Join(failures...)
}
return paths, nil
func (w *DiagnosticsWriter) recordFailure(role, action, path string, err error) {
var pathErr *os.PathError
if errors.As(err, &pathErr) {
err = pathErr.Err
}
key := interactionFailureKey{role: role, action: action, cause: err.Error()}
w.mu.Lock()
defer w.mu.Unlock()
if w.failures == nil {
w.failures = make(map[interactionFailureKey]*interactionFailure)
}
group := w.failures[key]
if group == nil {
group = &interactionFailure{path: path}
w.failures[key] = group
w.order = append(w.order, key)
}
group.count++
}
// Warnings summarizes capture failures after the callers join.
func (w *DiagnosticsWriter) Warnings() []error {
w.mu.Lock()
defer w.mu.Unlock()
warnings := make([]error, 0, len(w.order))
for _, key := range w.order {
group := w.failures[key]
warnings = append(warnings, fmt.Errorf("%s interaction: %s failures (%d), first at %q: %s", key.role, key.action, group.count, group.path, key.cause))
}
return warnings
}
func sanitizeStageName(stage string) string {
@@ -79,14 +131,10 @@ func sanitizeStageName(stage string) string {
return safe
}
func (w *DiagnosticsWriter) writeJSON(path string, payload any) error {
func writeInteractionJSON(path string, payload any) (string, error) {
encoded, err := json.Marshal(payload)
if err != nil {
return fmt.Errorf("marshal json artifact %q: %w", filepath.Base(path), err)
return "marshal", err
}
encoded = append(encoded, '\n')
if err := os.WriteFile(path, encoded, 0o644); err != nil {
return fmt.Errorf("write json artifact %q: %w", filepath.Base(path), err)
}
return nil
return "write", os.WriteFile(path, append(encoded, '\n'), 0o644)
}

View File

@@ -4,20 +4,24 @@ import (
"encoding/json"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
)
func TestDiagnosticsWriterProducesValidJSONArtifacts(t *testing.T) {
dir := t.TempDir()
writer := NewDiagnosticsWriter(dir)
artifacts, err := writer.WriteInteraction(
"validation:batch-1",
map[string]any{"stage": "validation", "attempt": 1},
map[string]any{"messages": []map[string]any{{"role": "user", "content": "hello"}}},
map[string]any{"result": "ok"},
nil,
)
artifacts, err := writer.WriteInteraction(structuredoutput.Interaction{
Name: "validation:batch-1",
Context: structuredoutput.DiagnosticContext{Role: "validation", ModuleInstance: "grammar_1"},
RequestMetadata: map[string]any{"attempt": 1},
RequestPayload: map[string]any{"messages": []map[string]any{{"role": "user", "content": "hello"}}},
ResponsePayload: map[string]any{"result": "ok"},
})
if err != nil {
t.Fatalf("WriteInteraction: %v", err)
}
@@ -30,6 +34,48 @@ func TestDiagnosticsWriterProducesValidJSONArtifacts(t *testing.T) {
assertValidJSONFile(t, artifacts.ResponsePayloadPath)
}
func TestDiagnosticsWriterGroupsFailuresAndContinuesCapture(t *testing.T) {
dir := t.TempDir()
writer := NewDiagnosticsWriter(dir)
var workers sync.WaitGroup
for _, name := range []string{"first", "second"} {
blocked := filepath.Join(dir, "grammar_1", name+"-response-payload.json")
if err := os.MkdirAll(blocked, 0o755); err != nil {
t.Fatal(err)
}
workers.Go(func() {
paths, err := writer.WriteInteraction(structuredoutput.Interaction{
Name: name, Context: structuredoutput.DiagnosticContext{Role: "validation", ModuleInstance: "grammar_1"},
ErrorPayload: map[string]string{"error": "model unavailable"},
})
if err == nil || paths.RequestPayloadPath == "" || paths.ResponsePayloadPath != "" || paths.ErrorPayloadPath == "" {
t.Errorf("paths=%+v error=%v", paths, err)
}
})
}
workers.Wait()
warnings := writer.Warnings()
if len(warnings) != 1 || !strings.Contains(warnings[0].Error(), "validation interaction: write response-payload failures (2)") ||
!strings.Contains(warnings[0].Error(), filepath.Join(dir, "grammar_1")) {
t.Fatalf("warnings: %v", warnings)
}
paths, err := writer.WriteInteraction(structuredoutput.Interaction{
Name: "later", Context: structuredoutput.DiagnosticContext{Role: "validation", ModuleInstance: "grammar_1"},
ResponsePayload: map[string]string{"final_content": "{"},
})
if err != nil {
t.Fatal(err)
}
raw, err := os.ReadFile(paths.ResponsePayloadPath)
if err != nil {
t.Fatal(err)
}
var decoded map[string]string
if err := json.Unmarshal(raw, &decoded); err != nil || decoded["final_content"] != "{" {
t.Fatalf("later artifact: %s (%v)", raw, err)
}
}
func assertValidJSONFile(t *testing.T, path string) {
t.Helper()
raw, err := os.ReadFile(path)