Unify optional interaction capture and warnings
This commit is contained in:
@@ -2,68 +2,120 @@ package llm
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||
)
|
||||
|
||||
var stageSanitizer = regexp.MustCompile(`[^a-zA-Z0-9_.-]+`)
|
||||
|
||||
// InteractionArtifacts contains file paths for written prompt/response
|
||||
// diagnostics.
|
||||
type InteractionArtifacts struct {
|
||||
RequestMetadataPath string
|
||||
RequestPayloadPath string
|
||||
ResponsePayloadPath string
|
||||
ErrorPayloadPath string
|
||||
}
|
||||
|
||||
// DiagnosticsWriter writes generic machine-readable LLM interaction artifacts.
|
||||
// DiagnosticsWriter writes interaction artifacts under a single run directory.
|
||||
type DiagnosticsWriter struct {
|
||||
dir string
|
||||
dir string
|
||||
mu sync.Mutex
|
||||
failures map[interactionFailureKey]*interactionFailure
|
||||
order []interactionFailureKey
|
||||
}
|
||||
|
||||
type interactionFailureKey struct {
|
||||
role, action, cause string
|
||||
}
|
||||
|
||||
type interactionFailure struct {
|
||||
count int
|
||||
path string
|
||||
}
|
||||
|
||||
// NewDiagnosticsWriter creates a diagnostics writer rooted at dir.
|
||||
func NewDiagnosticsWriter(dir string) *DiagnosticsWriter {
|
||||
return &DiagnosticsWriter{dir: dir}
|
||||
}
|
||||
|
||||
// WriteInteraction writes request/response/error artifacts for a generic LLM
|
||||
// interaction stage.
|
||||
func (w *DiagnosticsWriter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error) {
|
||||
func (w *DiagnosticsWriter) WriteInteraction(interaction structuredoutput.Interaction) (structuredoutput.InteractionArtifacts, error) {
|
||||
var paths structuredoutput.InteractionArtifacts
|
||||
dir := w.dir
|
||||
if instance := interaction.Context.ModuleInstance; instance != "" {
|
||||
dir = filepath.Join(dir, sanitizeStageName(instance))
|
||||
}
|
||||
var err error
|
||||
if strings.TrimSpace(w.dir) == "" {
|
||||
return InteractionArtifacts{}, fmt.Errorf("diagnostics directory must not be empty")
|
||||
err = fmt.Errorf("diagnostics directory must not be empty")
|
||||
} else {
|
||||
err = os.MkdirAll(dir, 0o755)
|
||||
}
|
||||
if err := os.MkdirAll(w.dir, 0o755); err != nil {
|
||||
return InteractionArtifacts{}, fmt.Errorf("create diagnostics directory: %w", err)
|
||||
base := sanitizeStageName(interaction.Name)
|
||||
if err != nil {
|
||||
w.recordFailure(interaction.Context.Role, "create directory", filepath.Join(dir, base), err)
|
||||
return paths, fmt.Errorf("create diagnostics directory %q: %w", dir, err)
|
||||
}
|
||||
|
||||
base := sanitizeStageName(stage)
|
||||
paths := InteractionArtifacts{
|
||||
RequestMetadataPath: filepath.Join(w.dir, base+"-request-metadata.json"),
|
||||
RequestPayloadPath: filepath.Join(w.dir, base+"-request-payload.json"),
|
||||
ResponsePayloadPath: filepath.Join(w.dir, base+"-response-payload.json"),
|
||||
metadata := maps.Clone(interaction.RequestMetadata)
|
||||
if metadata == nil {
|
||||
metadata = make(map[string]any)
|
||||
}
|
||||
|
||||
if err := w.writeJSON(paths.RequestMetadataPath, requestMetadata); err != nil {
|
||||
return InteractionArtifacts{}, err
|
||||
metadata["context"] = interaction.Context
|
||||
type artifactFile struct {
|
||||
suffix string
|
||||
payload any
|
||||
path *string
|
||||
}
|
||||
if err := w.writeJSON(paths.RequestPayloadPath, requestPayload); err != nil {
|
||||
return InteractionArtifacts{}, err
|
||||
artifacts := []artifactFile{
|
||||
{"request-metadata", metadata, &paths.RequestMetadataPath},
|
||||
{"request-payload", interaction.RequestPayload, &paths.RequestPayloadPath},
|
||||
{"response-payload", interaction.ResponsePayload, &paths.ResponsePayloadPath},
|
||||
}
|
||||
if err := w.writeJSON(paths.ResponsePayloadPath, responsePayload); err != nil {
|
||||
return InteractionArtifacts{}, err
|
||||
if interaction.ErrorPayload != nil {
|
||||
artifacts = append(artifacts, artifactFile{"error-payload", interaction.ErrorPayload, &paths.ErrorPayloadPath})
|
||||
}
|
||||
if errorPayload != nil {
|
||||
paths.ErrorPayloadPath = filepath.Join(w.dir, base+"-error-payload.json")
|
||||
if err := w.writeJSON(paths.ErrorPayloadPath, errorPayload); err != nil {
|
||||
return InteractionArtifacts{}, err
|
||||
var failures []error
|
||||
for _, artifact := range artifacts {
|
||||
path := filepath.Join(dir, base+"-"+artifact.suffix+".json")
|
||||
if action, err := writeInteractionJSON(path, artifact.payload); err != nil {
|
||||
w.recordFailure(interaction.Context.Role, action+" "+artifact.suffix, path, err)
|
||||
failures = append(failures, fmt.Errorf("%s %q: %w", action, path, err))
|
||||
continue
|
||||
}
|
||||
*artifact.path = path
|
||||
}
|
||||
return paths, errors.Join(failures...)
|
||||
}
|
||||
|
||||
return paths, nil
|
||||
func (w *DiagnosticsWriter) recordFailure(role, action, path string, err error) {
|
||||
var pathErr *os.PathError
|
||||
if errors.As(err, &pathErr) {
|
||||
err = pathErr.Err
|
||||
}
|
||||
key := interactionFailureKey{role: role, action: action, cause: err.Error()}
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.failures == nil {
|
||||
w.failures = make(map[interactionFailureKey]*interactionFailure)
|
||||
}
|
||||
group := w.failures[key]
|
||||
if group == nil {
|
||||
group = &interactionFailure{path: path}
|
||||
w.failures[key] = group
|
||||
w.order = append(w.order, key)
|
||||
}
|
||||
group.count++
|
||||
}
|
||||
|
||||
// Warnings summarizes capture failures after the callers join.
|
||||
func (w *DiagnosticsWriter) Warnings() []error {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
warnings := make([]error, 0, len(w.order))
|
||||
for _, key := range w.order {
|
||||
group := w.failures[key]
|
||||
warnings = append(warnings, fmt.Errorf("%s interaction: %s failures (%d), first at %q: %s", key.role, key.action, group.count, group.path, key.cause))
|
||||
}
|
||||
return warnings
|
||||
}
|
||||
|
||||
func sanitizeStageName(stage string) string {
|
||||
@@ -79,14 +131,10 @@ func sanitizeStageName(stage string) string {
|
||||
return safe
|
||||
}
|
||||
|
||||
func (w *DiagnosticsWriter) writeJSON(path string, payload any) error {
|
||||
func writeInteractionJSON(path string, payload any) (string, error) {
|
||||
encoded, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal json artifact %q: %w", filepath.Base(path), err)
|
||||
return "marshal", err
|
||||
}
|
||||
encoded = append(encoded, '\n')
|
||||
if err := os.WriteFile(path, encoded, 0o644); err != nil {
|
||||
return fmt.Errorf("write json artifact %q: %w", filepath.Base(path), err)
|
||||
}
|
||||
return nil
|
||||
return "write", os.WriteFile(path, append(encoded, '\n'), 0o644)
|
||||
}
|
||||
|
||||
@@ -4,20 +4,24 @@ import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||
)
|
||||
|
||||
func TestDiagnosticsWriterProducesValidJSONArtifacts(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writer := NewDiagnosticsWriter(dir)
|
||||
|
||||
artifacts, err := writer.WriteInteraction(
|
||||
"validation:batch-1",
|
||||
map[string]any{"stage": "validation", "attempt": 1},
|
||||
map[string]any{"messages": []map[string]any{{"role": "user", "content": "hello"}}},
|
||||
map[string]any{"result": "ok"},
|
||||
nil,
|
||||
)
|
||||
artifacts, err := writer.WriteInteraction(structuredoutput.Interaction{
|
||||
Name: "validation:batch-1",
|
||||
Context: structuredoutput.DiagnosticContext{Role: "validation", ModuleInstance: "grammar_1"},
|
||||
RequestMetadata: map[string]any{"attempt": 1},
|
||||
RequestPayload: map[string]any{"messages": []map[string]any{{"role": "user", "content": "hello"}}},
|
||||
ResponsePayload: map[string]any{"result": "ok"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("WriteInteraction: %v", err)
|
||||
}
|
||||
@@ -30,6 +34,48 @@ func TestDiagnosticsWriterProducesValidJSONArtifacts(t *testing.T) {
|
||||
assertValidJSONFile(t, artifacts.ResponsePayloadPath)
|
||||
}
|
||||
|
||||
func TestDiagnosticsWriterGroupsFailuresAndContinuesCapture(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writer := NewDiagnosticsWriter(dir)
|
||||
var workers sync.WaitGroup
|
||||
for _, name := range []string{"first", "second"} {
|
||||
blocked := filepath.Join(dir, "grammar_1", name+"-response-payload.json")
|
||||
if err := os.MkdirAll(blocked, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
workers.Go(func() {
|
||||
paths, err := writer.WriteInteraction(structuredoutput.Interaction{
|
||||
Name: name, Context: structuredoutput.DiagnosticContext{Role: "validation", ModuleInstance: "grammar_1"},
|
||||
ErrorPayload: map[string]string{"error": "model unavailable"},
|
||||
})
|
||||
if err == nil || paths.RequestPayloadPath == "" || paths.ResponsePayloadPath != "" || paths.ErrorPayloadPath == "" {
|
||||
t.Errorf("paths=%+v error=%v", paths, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
workers.Wait()
|
||||
warnings := writer.Warnings()
|
||||
if len(warnings) != 1 || !strings.Contains(warnings[0].Error(), "validation interaction: write response-payload failures (2)") ||
|
||||
!strings.Contains(warnings[0].Error(), filepath.Join(dir, "grammar_1")) {
|
||||
t.Fatalf("warnings: %v", warnings)
|
||||
}
|
||||
paths, err := writer.WriteInteraction(structuredoutput.Interaction{
|
||||
Name: "later", Context: structuredoutput.DiagnosticContext{Role: "validation", ModuleInstance: "grammar_1"},
|
||||
ResponsePayload: map[string]string{"final_content": "{"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, err := os.ReadFile(paths.ResponsePayloadPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var decoded map[string]string
|
||||
if err := json.Unmarshal(raw, &decoded); err != nil || decoded["final_content"] != "{" {
|
||||
t.Fatalf("later artifact: %s (%v)", raw, err)
|
||||
}
|
||||
}
|
||||
|
||||
func assertValidJSONFile(t *testing.T, path string) {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(path)
|
||||
|
||||
Reference in New Issue
Block a user