Unify optional interaction capture and warnings

This commit is contained in:
2026-11-16 03:43:29 +00:00
parent ffe0e3491e
commit cea7e44232
22 changed files with 450 additions and 367 deletions

View File

@@ -2,68 +2,120 @@ package llm
import (
"encoding/json"
"errors"
"fmt"
"maps"
"os"
"path/filepath"
"regexp"
"strings"
"sync"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
)
var stageSanitizer = regexp.MustCompile(`[^a-zA-Z0-9_.-]+`)
// InteractionArtifacts contains file paths for written prompt/response
// diagnostics.
type InteractionArtifacts struct {
RequestMetadataPath string
RequestPayloadPath string
ResponsePayloadPath string
ErrorPayloadPath string
}
// DiagnosticsWriter writes generic machine-readable LLM interaction artifacts.
// DiagnosticsWriter writes interaction artifacts under a single run directory.
type DiagnosticsWriter struct {
dir string
dir string
mu sync.Mutex
failures map[interactionFailureKey]*interactionFailure
order []interactionFailureKey
}
type interactionFailureKey struct {
role, action, cause string
}
type interactionFailure struct {
count int
path string
}
// NewDiagnosticsWriter creates a diagnostics writer rooted at dir.
func NewDiagnosticsWriter(dir string) *DiagnosticsWriter {
return &DiagnosticsWriter{dir: dir}
}
// WriteInteraction writes request/response/error artifacts for a generic LLM
// interaction stage.
func (w *DiagnosticsWriter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error) {
func (w *DiagnosticsWriter) WriteInteraction(interaction structuredoutput.Interaction) (structuredoutput.InteractionArtifacts, error) {
var paths structuredoutput.InteractionArtifacts
dir := w.dir
if instance := interaction.Context.ModuleInstance; instance != "" {
dir = filepath.Join(dir, sanitizeStageName(instance))
}
var err error
if strings.TrimSpace(w.dir) == "" {
return InteractionArtifacts{}, fmt.Errorf("diagnostics directory must not be empty")
err = fmt.Errorf("diagnostics directory must not be empty")
} else {
err = os.MkdirAll(dir, 0o755)
}
if err := os.MkdirAll(w.dir, 0o755); err != nil {
return InteractionArtifacts{}, fmt.Errorf("create diagnostics directory: %w", err)
base := sanitizeStageName(interaction.Name)
if err != nil {
w.recordFailure(interaction.Context.Role, "create directory", filepath.Join(dir, base), err)
return paths, fmt.Errorf("create diagnostics directory %q: %w", dir, err)
}
base := sanitizeStageName(stage)
paths := InteractionArtifacts{
RequestMetadataPath: filepath.Join(w.dir, base+"-request-metadata.json"),
RequestPayloadPath: filepath.Join(w.dir, base+"-request-payload.json"),
ResponsePayloadPath: filepath.Join(w.dir, base+"-response-payload.json"),
metadata := maps.Clone(interaction.RequestMetadata)
if metadata == nil {
metadata = make(map[string]any)
}
if err := w.writeJSON(paths.RequestMetadataPath, requestMetadata); err != nil {
return InteractionArtifacts{}, err
metadata["context"] = interaction.Context
type artifactFile struct {
suffix string
payload any
path *string
}
if err := w.writeJSON(paths.RequestPayloadPath, requestPayload); err != nil {
return InteractionArtifacts{}, err
artifacts := []artifactFile{
{"request-metadata", metadata, &paths.RequestMetadataPath},
{"request-payload", interaction.RequestPayload, &paths.RequestPayloadPath},
{"response-payload", interaction.ResponsePayload, &paths.ResponsePayloadPath},
}
if err := w.writeJSON(paths.ResponsePayloadPath, responsePayload); err != nil {
return InteractionArtifacts{}, err
if interaction.ErrorPayload != nil {
artifacts = append(artifacts, artifactFile{"error-payload", interaction.ErrorPayload, &paths.ErrorPayloadPath})
}
if errorPayload != nil {
paths.ErrorPayloadPath = filepath.Join(w.dir, base+"-error-payload.json")
if err := w.writeJSON(paths.ErrorPayloadPath, errorPayload); err != nil {
return InteractionArtifacts{}, err
var failures []error
for _, artifact := range artifacts {
path := filepath.Join(dir, base+"-"+artifact.suffix+".json")
if action, err := writeInteractionJSON(path, artifact.payload); err != nil {
w.recordFailure(interaction.Context.Role, action+" "+artifact.suffix, path, err)
failures = append(failures, fmt.Errorf("%s %q: %w", action, path, err))
continue
}
*artifact.path = path
}
return paths, errors.Join(failures...)
}
return paths, nil
func (w *DiagnosticsWriter) recordFailure(role, action, path string, err error) {
var pathErr *os.PathError
if errors.As(err, &pathErr) {
err = pathErr.Err
}
key := interactionFailureKey{role: role, action: action, cause: err.Error()}
w.mu.Lock()
defer w.mu.Unlock()
if w.failures == nil {
w.failures = make(map[interactionFailureKey]*interactionFailure)
}
group := w.failures[key]
if group == nil {
group = &interactionFailure{path: path}
w.failures[key] = group
w.order = append(w.order, key)
}
group.count++
}
// Warnings summarizes capture failures after the callers join.
func (w *DiagnosticsWriter) Warnings() []error {
w.mu.Lock()
defer w.mu.Unlock()
warnings := make([]error, 0, len(w.order))
for _, key := range w.order {
group := w.failures[key]
warnings = append(warnings, fmt.Errorf("%s interaction: %s failures (%d), first at %q: %s", key.role, key.action, group.count, group.path, key.cause))
}
return warnings
}
func sanitizeStageName(stage string) string {
@@ -79,14 +131,10 @@ func sanitizeStageName(stage string) string {
return safe
}
func (w *DiagnosticsWriter) writeJSON(path string, payload any) error {
func writeInteractionJSON(path string, payload any) (string, error) {
encoded, err := json.Marshal(payload)
if err != nil {
return fmt.Errorf("marshal json artifact %q: %w", filepath.Base(path), err)
return "marshal", err
}
encoded = append(encoded, '\n')
if err := os.WriteFile(path, encoded, 0o644); err != nil {
return fmt.Errorf("write json artifact %q: %w", filepath.Base(path), err)
}
return nil
return "write", os.WriteFile(path, append(encoded, '\n'), 0o644)
}