Files
audita/internal/framework/llm/diagnostics.go

141 lines
3.9 KiB
Go
Raw Normal View History

package llm
import (
"encoding/json"
"errors"
"fmt"
"maps"
"os"
"path/filepath"
"regexp"
"strings"
"sync"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
)
var stageSanitizer = regexp.MustCompile(`[^a-zA-Z0-9_.-]+`)
// DiagnosticsWriter writes interaction artifacts under a single run directory.
type DiagnosticsWriter struct {
dir string
mu sync.Mutex
failures map[interactionFailureKey]*interactionFailure
order []interactionFailureKey
}
type interactionFailureKey struct {
role, action, cause string
}
type interactionFailure struct {
count int
path string
}
func NewDiagnosticsWriter(dir string) *DiagnosticsWriter {
return &DiagnosticsWriter{dir: dir}
}
func (w *DiagnosticsWriter) WriteInteraction(interaction structuredoutput.Interaction) (structuredoutput.InteractionArtifacts, error) {
var paths structuredoutput.InteractionArtifacts
dir := w.dir
if instance := interaction.Context.ModuleInstance; instance != "" {
dir = filepath.Join(dir, sanitizeStageName(instance))
}
var err error
if strings.TrimSpace(w.dir) == "" {
err = fmt.Errorf("diagnostics directory must not be empty")
} else {
err = os.MkdirAll(dir, 0o755)
}
base := sanitizeStageName(interaction.Name)
if err != nil {
w.recordFailure(interaction.Context.Role, "create directory", filepath.Join(dir, base), err)
return paths, fmt.Errorf("create diagnostics directory %q: %w", dir, err)
}
metadata := maps.Clone(interaction.RequestMetadata)
if metadata == nil {
metadata = make(map[string]any)
}
metadata["context"] = interaction.Context
type artifactFile struct {
suffix string
payload any
path *string
}
artifacts := []artifactFile{
{"request-metadata", metadata, &paths.RequestMetadataPath},
{"request-payload", interaction.RequestPayload, &paths.RequestPayloadPath},
{"response-payload", interaction.ResponsePayload, &paths.ResponsePayloadPath},
}
if interaction.ErrorPayload != nil {
artifacts = append(artifacts, artifactFile{"error-payload", interaction.ErrorPayload, &paths.ErrorPayloadPath})
}
var failures []error
for _, artifact := range artifacts {
path := filepath.Join(dir, base+"-"+artifact.suffix+".json")
if action, err := writeInteractionJSON(path, artifact.payload); err != nil {
w.recordFailure(interaction.Context.Role, action+" "+artifact.suffix, path, err)
failures = append(failures, fmt.Errorf("%s %q: %w", action, path, err))
continue
}
*artifact.path = path
}
return paths, errors.Join(failures...)
}
func (w *DiagnosticsWriter) recordFailure(role, action, path string, err error) {
var pathErr *os.PathError
if errors.As(err, &pathErr) {
err = pathErr.Err
}
key := interactionFailureKey{role: role, action: action, cause: err.Error()}
w.mu.Lock()
defer w.mu.Unlock()
if w.failures == nil {
w.failures = make(map[interactionFailureKey]*interactionFailure)
}
group := w.failures[key]
if group == nil {
group = &interactionFailure{path: path}
w.failures[key] = group
w.order = append(w.order, key)
}
group.count++
}
// Warnings summarizes capture failures after the callers join.
func (w *DiagnosticsWriter) Warnings() []error {
w.mu.Lock()
defer w.mu.Unlock()
warnings := make([]error, 0, len(w.order))
for _, key := range w.order {
group := w.failures[key]
warnings = append(warnings, fmt.Errorf("%s interaction: %s failures (%d), first at %q: %s", key.role, key.action, group.count, group.path, key.cause))
}
return warnings
}
func sanitizeStageName(stage string) string {
stage = strings.TrimSpace(stage)
if stage == "" {
return "llm"
}
safe := stageSanitizer.ReplaceAllString(stage, "_")
safe = strings.Trim(safe, "._-")
if safe == "" {
return "llm"
}
return safe
}
func writeInteractionJSON(path string, payload any) (string, error) {
encoded, err := json.Marshal(payload)
if err != nil {
return "marshal", err
}
return "write", os.WriteFile(path, append(encoded, '\n'), 0o644)
}