2026-05-11 20:09:34 -05:00
|
|
|
package llm
|
|
|
|
|
|
|
|
|
|
import (
|
2026-11-16 03:37:29 +00:00
|
|
|
"encoding/json"
|
2026-11-16 03:43:29 +00:00
|
|
|
"errors"
|
2026-05-11 20:09:34 -05:00
|
|
|
"fmt"
|
2026-11-16 03:43:29 +00:00
|
|
|
"maps"
|
2026-05-11 20:09:34 -05:00
|
|
|
"os"
|
|
|
|
|
"path/filepath"
|
|
|
|
|
"regexp"
|
|
|
|
|
"strings"
|
2026-11-16 03:43:29 +00:00
|
|
|
"sync"
|
|
|
|
|
|
|
|
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
2026-09-24 15:32:13 +00:00
|
|
|
)
|
2026-05-11 20:09:34 -05:00
|
|
|
|
|
|
|
|
var stageSanitizer = regexp.MustCompile(`[^a-zA-Z0-9_.-]+`)
|
|
|
|
|
|
2026-11-16 03:43:29 +00:00
|
|
|
// DiagnosticsWriter writes interaction artifacts under a single run directory.
|
|
|
|
|
type DiagnosticsWriter struct {
|
|
|
|
|
dir string
|
|
|
|
|
mu sync.Mutex
|
|
|
|
|
failures map[interactionFailureKey]*interactionFailure
|
|
|
|
|
order []interactionFailureKey
|
2026-05-11 20:09:34 -05:00
|
|
|
}
|
|
|
|
|
|
2026-11-16 03:43:29 +00:00
|
|
|
type interactionFailureKey struct {
|
|
|
|
|
role, action, cause string
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
type interactionFailure struct {
|
|
|
|
|
count int
|
|
|
|
|
path string
|
2026-05-11 20:09:34 -05:00
|
|
|
}
|
|
|
|
|
|
2026-11-16 03:37:29 +00:00
|
|
|
func NewDiagnosticsWriter(dir string) *DiagnosticsWriter {
|
|
|
|
|
return &DiagnosticsWriter{dir: dir}
|
2026-05-11 20:09:34 -05:00
|
|
|
}
|
|
|
|
|
|
2026-11-16 03:43:29 +00:00
|
|
|
func (w *DiagnosticsWriter) WriteInteraction(interaction structuredoutput.Interaction) (structuredoutput.InteractionArtifacts, error) {
|
|
|
|
|
var paths structuredoutput.InteractionArtifacts
|
|
|
|
|
dir := w.dir
|
|
|
|
|
if instance := interaction.Context.ModuleInstance; instance != "" {
|
|
|
|
|
dir = filepath.Join(dir, sanitizeStageName(instance))
|
|
|
|
|
}
|
|
|
|
|
var err error
|
2026-05-11 20:09:34 -05:00
|
|
|
if strings.TrimSpace(w.dir) == "" {
|
2026-11-16 03:43:29 +00:00
|
|
|
err = fmt.Errorf("diagnostics directory must not be empty")
|
|
|
|
|
} else {
|
|
|
|
|
err = os.MkdirAll(dir, 0o755)
|
2026-05-11 20:09:34 -05:00
|
|
|
}
|
2026-11-16 03:43:29 +00:00
|
|
|
base := sanitizeStageName(interaction.Name)
|
|
|
|
|
if err != nil {
|
|
|
|
|
w.recordFailure(interaction.Context.Role, "create directory", filepath.Join(dir, base), err)
|
|
|
|
|
return paths, fmt.Errorf("create diagnostics directory %q: %w", dir, err)
|
2026-05-11 20:09:34 -05:00
|
|
|
}
|
|
|
|
|
|
2026-11-16 03:43:29 +00:00
|
|
|
metadata := maps.Clone(interaction.RequestMetadata)
|
|
|
|
|
if metadata == nil {
|
|
|
|
|
metadata = make(map[string]any)
|
2026-05-11 20:09:34 -05:00
|
|
|
}
|
2026-11-16 03:43:29 +00:00
|
|
|
metadata["context"] = interaction.Context
|
|
|
|
|
type artifactFile struct {
|
|
|
|
|
suffix string
|
|
|
|
|
payload any
|
|
|
|
|
path *string
|
2026-05-11 20:09:34 -05:00
|
|
|
}
|
2026-11-16 03:43:29 +00:00
|
|
|
artifacts := []artifactFile{
|
|
|
|
|
{"request-metadata", metadata, &paths.RequestMetadataPath},
|
|
|
|
|
{"request-payload", interaction.RequestPayload, &paths.RequestPayloadPath},
|
|
|
|
|
{"response-payload", interaction.ResponsePayload, &paths.ResponsePayloadPath},
|
2026-05-11 20:09:34 -05:00
|
|
|
}
|
2026-11-16 03:43:29 +00:00
|
|
|
if interaction.ErrorPayload != nil {
|
|
|
|
|
artifacts = append(artifacts, artifactFile{"error-payload", interaction.ErrorPayload, &paths.ErrorPayloadPath})
|
2026-05-11 20:09:34 -05:00
|
|
|
}
|
2026-11-16 03:43:29 +00:00
|
|
|
var failures []error
|
|
|
|
|
for _, artifact := range artifacts {
|
|
|
|
|
path := filepath.Join(dir, base+"-"+artifact.suffix+".json")
|
|
|
|
|
if action, err := writeInteractionJSON(path, artifact.payload); err != nil {
|
|
|
|
|
w.recordFailure(interaction.Context.Role, action+" "+artifact.suffix, path, err)
|
|
|
|
|
failures = append(failures, fmt.Errorf("%s %q: %w", action, path, err))
|
|
|
|
|
continue
|
2026-05-11 20:09:34 -05:00
|
|
|
}
|
2026-11-16 03:43:29 +00:00
|
|
|
*artifact.path = path
|
2026-05-11 20:09:34 -05:00
|
|
|
}
|
2026-11-16 03:43:29 +00:00
|
|
|
return paths, errors.Join(failures...)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (w *DiagnosticsWriter) recordFailure(role, action, path string, err error) {
|
|
|
|
|
var pathErr *os.PathError
|
|
|
|
|
if errors.As(err, &pathErr) {
|
|
|
|
|
err = pathErr.Err
|
|
|
|
|
}
|
|
|
|
|
key := interactionFailureKey{role: role, action: action, cause: err.Error()}
|
|
|
|
|
w.mu.Lock()
|
|
|
|
|
defer w.mu.Unlock()
|
|
|
|
|
if w.failures == nil {
|
|
|
|
|
w.failures = make(map[interactionFailureKey]*interactionFailure)
|
|
|
|
|
}
|
|
|
|
|
group := w.failures[key]
|
|
|
|
|
if group == nil {
|
|
|
|
|
group = &interactionFailure{path: path}
|
|
|
|
|
w.failures[key] = group
|
|
|
|
|
w.order = append(w.order, key)
|
|
|
|
|
}
|
|
|
|
|
group.count++
|
|
|
|
|
}
|
2026-05-11 20:09:34 -05:00
|
|
|
|
2026-11-16 03:43:29 +00:00
|
|
|
// Warnings summarizes capture failures after the callers join.
|
|
|
|
|
func (w *DiagnosticsWriter) Warnings() []error {
|
|
|
|
|
w.mu.Lock()
|
|
|
|
|
defer w.mu.Unlock()
|
|
|
|
|
warnings := make([]error, 0, len(w.order))
|
|
|
|
|
for _, key := range w.order {
|
|
|
|
|
group := w.failures[key]
|
|
|
|
|
warnings = append(warnings, fmt.Errorf("%s interaction: %s failures (%d), first at %q: %s", key.role, key.action, group.count, group.path, key.cause))
|
|
|
|
|
}
|
|
|
|
|
return warnings
|
2026-05-11 20:09:34 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func sanitizeStageName(stage string) string {
|
|
|
|
|
stage = strings.TrimSpace(stage)
|
|
|
|
|
if stage == "" {
|
|
|
|
|
return "llm"
|
|
|
|
|
}
|
|
|
|
|
safe := stageSanitizer.ReplaceAllString(stage, "_")
|
|
|
|
|
safe = strings.Trim(safe, "._-")
|
|
|
|
|
if safe == "" {
|
|
|
|
|
return "llm"
|
|
|
|
|
}
|
|
|
|
|
return safe
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-16 03:43:29 +00:00
|
|
|
func writeInteractionJSON(path string, payload any) (string, error) {
|
2026-11-16 03:37:29 +00:00
|
|
|
encoded, err := json.Marshal(payload)
|
2026-05-11 20:09:34 -05:00
|
|
|
if err != nil {
|
2026-11-16 03:43:29 +00:00
|
|
|
return "marshal", err
|
2026-05-11 20:09:34 -05:00
|
|
|
}
|
2026-11-16 03:43:29 +00:00
|
|
|
return "write", os.WriteFile(path, append(encoded, '\n'), 0o644)
|
2026-05-11 20:09:34 -05:00
|
|
|
}
|