Unify optional interaction capture and warnings

This commit is contained in:
2026-11-16 03:43:29 +00:00
parent ffe0e3491e
commit cea7e44232
22 changed files with 450 additions and 367 deletions

View File

@@ -32,16 +32,23 @@ On a final write error it attempts other enabled reports and one failed-status
pass; a published transcript is never erased by a later report failure.
Debug is independent. The shared `--debug` flag enables a unique retained child
under `diagnostics.work_dir`. CLI passes empty diagnostics paths and nil writers
when debug is off. With debug enabled, it saves the source bytes as acquired, separate from the parsed and normalized
under `diagnostics.work_dir`. The CLI creates one interaction sink rooted in
that directory and passes it to proposal generation, validation, and repair.
With debug enabled, it saves the source bytes as acquired, separate from the parsed and normalized
transcript snapshots. It also captures invocation metadata, effective settings
rendered by ConfigKit, the correction ledger, LLM execution observations, and
model interactions. Candidate capture includes proposals later
rejected or interrupted. Repair and revalidation artifacts use section, proposal,
attempt, validator, and batch coordinates as applicable. Ledger mapping preserves empty proposed and applied replacements and distinguishes unavailable candidates from interrupted work.
attempt, validator, and batch coordinates as applicable. Request metadata records
these coordinates with the interaction role. Each interaction retains its available
prepared request, response, and failure evidence. Ledger mapping preserves empty
proposed and applied replacements and distinguishes unavailable candidates from
interrupted work.
A concurrency-safe diagnostic callback reports optional write failures without
aborting processing. Debug artifacts can contain sensitive transcript and
The sink groups concurrent write failures by role, artifact operation, and cause.
Warnings report the count and an affected path. The writer still attempts other
artifacts and returns the paths it wrote, and the CLI presents warnings separately
from processing outcomes and required delivery. Debug artifacts can contain sensitive transcript and
prompt content. Execution
position and optional operator ID are carried separately from generated safe
artifact path coordinates.

View File

@@ -141,6 +141,12 @@ Verify role, chunk, batch, proposal, and attempt associations with decoded
artifacts. Exercise a failed optional write followed by later capture. Update
the [diagnostics internals](../internal/diagnostics-reporting.md) for the sink.
**Complete.** The CLI supplies one optional sink for proposal, validation,
repair, and revalidation artifacts. The writer records successful paths even
after a partial write and groups failures by role, artifact operation, and
cause. Decoded-artifact and recovery checks passed, along with the full Go
suite and targeted race checks.
## Stage 4 — Bind publication to checked destinations
Audit #1, #6 (cooperative boundaries), #7, #8, #16, #26, #42.

View File

@@ -47,13 +47,20 @@ func executePipeline(ctx context.Context, loaded *processInputs, prepared *prepa
item.CaptureProposalExchange = true
}
}
debugPath := prepared.DebugPath()
var interactions structuredoutput.InteractionSink
var writer *llm.DiagnosticsWriter
if debug != nil {
writer = llm.NewDiagnosticsWriter(debug.Path())
interactions = writer
}
output, runErr := runner.New(modules.NewFactory()).Run(ctx, runner.RunInput{
Config: cfg, Transcript: normalized, ModuleSpecs: specs,
ProposalDiagnosticsDir: debugPath, ValidationDiagnosticsDir: debugPath,
CaptureCandidates: debugPath != "", ReportDiagnosticError: warn,
Interactions: interactions, CaptureCandidates: debug != nil,
})
if debug != nil {
for _, warning := range writer.Warnings() {
warn(warning)
}
warn(debug.WriteJSONArtifact(diagnostics.ArtifactLLMExecution, processreport.BuildLLMExecution(observer.Snapshot())))
warn(debug.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{RunDirectoryPath: debug.Path(), RunOutput: &output})))
}

View File

@@ -73,17 +73,15 @@ func SectionMetadataFromSection(section chunking.Section) SectionMetadata {
// ExecutionContext carries shared execution state for proposal generation and validation.
type ExecutionContext struct {
StageIndex *int `json:"stage_index,omitempty"`
StageID *string `json:"stage_id,omitempty"`
// ReportDiagnosticError may be called concurrently; it must not affect processing.
ReportDiagnosticError func(error) `json:"-"`
Settings config.Stage `json:"-"`
WorkingTranscript *schema.Transcript `json:"-"`
Glossary []glossary.Entry `json:"-"`
ProtectedTerms []glossary.Term `json:"-"`
GlossaryVariables *promptcontext.GlossaryVariables `json:"-"`
Section *SectionMetadata `json:"section,omitempty"`
DiagnosticsDir string `json:"diagnostics_dir,omitempty"`
StageIndex *int `json:"stage_index,omitempty"`
StageID *string `json:"stage_id,omitempty"`
Settings config.Stage `json:"-"`
WorkingTranscript *schema.Transcript `json:"-"`
Glossary []glossary.Entry `json:"-"`
ProtectedTerms []glossary.Term `json:"-"`
GlossaryVariables *promptcontext.GlossaryVariables `json:"-"`
Section *SectionMetadata `json:"section,omitempty"`
Interactions structuredoutput.InteractionSink `json:"-"`
}
// ProposalRequest is the input to module proposal generation.

View File

@@ -2,68 +2,120 @@ package llm
import (
"encoding/json"
"errors"
"fmt"
"maps"
"os"
"path/filepath"
"regexp"
"strings"
"sync"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
)
var stageSanitizer = regexp.MustCompile(`[^a-zA-Z0-9_.-]+`)
// InteractionArtifacts contains file paths for written prompt/response
// diagnostics.
type InteractionArtifacts struct {
RequestMetadataPath string
RequestPayloadPath string
ResponsePayloadPath string
ErrorPayloadPath string
}
// DiagnosticsWriter writes generic machine-readable LLM interaction artifacts.
// DiagnosticsWriter writes interaction artifacts under a single run directory.
type DiagnosticsWriter struct {
dir string
dir string
mu sync.Mutex
failures map[interactionFailureKey]*interactionFailure
order []interactionFailureKey
}
type interactionFailureKey struct {
role, action, cause string
}
type interactionFailure struct {
count int
path string
}
// NewDiagnosticsWriter creates a diagnostics writer rooted at dir.
func NewDiagnosticsWriter(dir string) *DiagnosticsWriter {
return &DiagnosticsWriter{dir: dir}
}
// WriteInteraction writes request/response/error artifacts for a generic LLM
// interaction stage.
func (w *DiagnosticsWriter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error) {
func (w *DiagnosticsWriter) WriteInteraction(interaction structuredoutput.Interaction) (structuredoutput.InteractionArtifacts, error) {
var paths structuredoutput.InteractionArtifacts
dir := w.dir
if instance := interaction.Context.ModuleInstance; instance != "" {
dir = filepath.Join(dir, sanitizeStageName(instance))
}
var err error
if strings.TrimSpace(w.dir) == "" {
return InteractionArtifacts{}, fmt.Errorf("diagnostics directory must not be empty")
err = fmt.Errorf("diagnostics directory must not be empty")
} else {
err = os.MkdirAll(dir, 0o755)
}
if err := os.MkdirAll(w.dir, 0o755); err != nil {
return InteractionArtifacts{}, fmt.Errorf("create diagnostics directory: %w", err)
base := sanitizeStageName(interaction.Name)
if err != nil {
w.recordFailure(interaction.Context.Role, "create directory", filepath.Join(dir, base), err)
return paths, fmt.Errorf("create diagnostics directory %q: %w", dir, err)
}
base := sanitizeStageName(stage)
paths := InteractionArtifacts{
RequestMetadataPath: filepath.Join(w.dir, base+"-request-metadata.json"),
RequestPayloadPath: filepath.Join(w.dir, base+"-request-payload.json"),
ResponsePayloadPath: filepath.Join(w.dir, base+"-response-payload.json"),
metadata := maps.Clone(interaction.RequestMetadata)
if metadata == nil {
metadata = make(map[string]any)
}
if err := w.writeJSON(paths.RequestMetadataPath, requestMetadata); err != nil {
return InteractionArtifacts{}, err
metadata["context"] = interaction.Context
type artifactFile struct {
suffix string
payload any
path *string
}
if err := w.writeJSON(paths.RequestPayloadPath, requestPayload); err != nil {
return InteractionArtifacts{}, err
artifacts := []artifactFile{
{"request-metadata", metadata, &paths.RequestMetadataPath},
{"request-payload", interaction.RequestPayload, &paths.RequestPayloadPath},
{"response-payload", interaction.ResponsePayload, &paths.ResponsePayloadPath},
}
if err := w.writeJSON(paths.ResponsePayloadPath, responsePayload); err != nil {
return InteractionArtifacts{}, err
if interaction.ErrorPayload != nil {
artifacts = append(artifacts, artifactFile{"error-payload", interaction.ErrorPayload, &paths.ErrorPayloadPath})
}
if errorPayload != nil {
paths.ErrorPayloadPath = filepath.Join(w.dir, base+"-error-payload.json")
if err := w.writeJSON(paths.ErrorPayloadPath, errorPayload); err != nil {
return InteractionArtifacts{}, err
var failures []error
for _, artifact := range artifacts {
path := filepath.Join(dir, base+"-"+artifact.suffix+".json")
if action, err := writeInteractionJSON(path, artifact.payload); err != nil {
w.recordFailure(interaction.Context.Role, action+" "+artifact.suffix, path, err)
failures = append(failures, fmt.Errorf("%s %q: %w", action, path, err))
continue
}
*artifact.path = path
}
return paths, errors.Join(failures...)
}
return paths, nil
func (w *DiagnosticsWriter) recordFailure(role, action, path string, err error) {
var pathErr *os.PathError
if errors.As(err, &pathErr) {
err = pathErr.Err
}
key := interactionFailureKey{role: role, action: action, cause: err.Error()}
w.mu.Lock()
defer w.mu.Unlock()
if w.failures == nil {
w.failures = make(map[interactionFailureKey]*interactionFailure)
}
group := w.failures[key]
if group == nil {
group = &interactionFailure{path: path}
w.failures[key] = group
w.order = append(w.order, key)
}
group.count++
}
// Warnings summarizes capture failures after the callers join.
func (w *DiagnosticsWriter) Warnings() []error {
w.mu.Lock()
defer w.mu.Unlock()
warnings := make([]error, 0, len(w.order))
for _, key := range w.order {
group := w.failures[key]
warnings = append(warnings, fmt.Errorf("%s interaction: %s failures (%d), first at %q: %s", key.role, key.action, group.count, group.path, key.cause))
}
return warnings
}
func sanitizeStageName(stage string) string {
@@ -79,14 +131,10 @@ func sanitizeStageName(stage string) string {
return safe
}
func (w *DiagnosticsWriter) writeJSON(path string, payload any) error {
func writeInteractionJSON(path string, payload any) (string, error) {
encoded, err := json.Marshal(payload)
if err != nil {
return fmt.Errorf("marshal json artifact %q: %w", filepath.Base(path), err)
return "marshal", err
}
encoded = append(encoded, '\n')
if err := os.WriteFile(path, encoded, 0o644); err != nil {
return fmt.Errorf("write json artifact %q: %w", filepath.Base(path), err)
}
return nil
return "write", os.WriteFile(path, append(encoded, '\n'), 0o644)
}

View File

@@ -4,20 +4,24 @@ import (
"encoding/json"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
)
func TestDiagnosticsWriterProducesValidJSONArtifacts(t *testing.T) {
dir := t.TempDir()
writer := NewDiagnosticsWriter(dir)
artifacts, err := writer.WriteInteraction(
"validation:batch-1",
map[string]any{"stage": "validation", "attempt": 1},
map[string]any{"messages": []map[string]any{{"role": "user", "content": "hello"}}},
map[string]any{"result": "ok"},
nil,
)
artifacts, err := writer.WriteInteraction(structuredoutput.Interaction{
Name: "validation:batch-1",
Context: structuredoutput.DiagnosticContext{Role: "validation", ModuleInstance: "grammar_1"},
RequestMetadata: map[string]any{"attempt": 1},
RequestPayload: map[string]any{"messages": []map[string]any{{"role": "user", "content": "hello"}}},
ResponsePayload: map[string]any{"result": "ok"},
})
if err != nil {
t.Fatalf("WriteInteraction: %v", err)
}
@@ -30,6 +34,48 @@ func TestDiagnosticsWriterProducesValidJSONArtifacts(t *testing.T) {
assertValidJSONFile(t, artifacts.ResponsePayloadPath)
}
func TestDiagnosticsWriterGroupsFailuresAndContinuesCapture(t *testing.T) {
dir := t.TempDir()
writer := NewDiagnosticsWriter(dir)
var workers sync.WaitGroup
for _, name := range []string{"first", "second"} {
blocked := filepath.Join(dir, "grammar_1", name+"-response-payload.json")
if err := os.MkdirAll(blocked, 0o755); err != nil {
t.Fatal(err)
}
workers.Go(func() {
paths, err := writer.WriteInteraction(structuredoutput.Interaction{
Name: name, Context: structuredoutput.DiagnosticContext{Role: "validation", ModuleInstance: "grammar_1"},
ErrorPayload: map[string]string{"error": "model unavailable"},
})
if err == nil || paths.RequestPayloadPath == "" || paths.ResponsePayloadPath != "" || paths.ErrorPayloadPath == "" {
t.Errorf("paths=%+v error=%v", paths, err)
}
})
}
workers.Wait()
warnings := writer.Warnings()
if len(warnings) != 1 || !strings.Contains(warnings[0].Error(), "validation interaction: write response-payload failures (2)") ||
!strings.Contains(warnings[0].Error(), filepath.Join(dir, "grammar_1")) {
t.Fatalf("warnings: %v", warnings)
}
paths, err := writer.WriteInteraction(structuredoutput.Interaction{
Name: "later", Context: structuredoutput.DiagnosticContext{Role: "validation", ModuleInstance: "grammar_1"},
ResponsePayload: map[string]string{"final_content": "{"},
})
if err != nil {
t.Fatal(err)
}
raw, err := os.ReadFile(paths.ResponsePayloadPath)
if err != nil {
t.Fatal(err)
}
var decoded map[string]string
if err := json.Unmarshal(raw, &decoded); err != nil || decoded["final_content"] != "{" {
t.Fatalf("later artifact: %s (%v)", raw, err)
}
}
func assertValidJSONFile(t *testing.T, path string) {
t.Helper()
raw, err := os.ReadFile(path)

View File

@@ -7,12 +7,10 @@ import (
"context"
"errors"
"fmt"
"path/filepath"
"strings"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
@@ -20,19 +18,6 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
)
// InteractionDiagnosticsWriter writes machine-readable prompt/response artifacts.
type InteractionDiagnosticsWriter interface {
WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error)
}
// InteractionArtifacts contains written diagnostics paths.
type InteractionArtifacts struct {
RequestMetadataPath string `json:"request_metadata_path,omitempty"`
RequestPayloadPath string `json:"request_payload_path,omitempty"`
ResponsePayloadPath string `json:"response_payload_path,omitempty"`
ErrorPayloadPath string `json:"error_payload_path,omitempty"`
}
// StructuredCorrectionProposal is one LLM response correction payload.
type StructuredCorrectionProposal struct {
SegmentReference int `json:"id"`
@@ -48,23 +33,20 @@ type StructuredCorrectionSet struct {
// Request captures reusable proposal-generation inputs for future modules.
type Request struct {
StageIndex *int `json:"stage_index,omitempty"`
StageID *string `json:"stage_id,omitempty"`
// ReportDiagnosticError may be called concurrently; it must not affect processing.
ReportDiagnosticError func(error) `json:"-"`
ModuleKey string `json:"module_key"`
ModuleInstance string `json:"module_instance"`
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
WorkingTranscript *schema.Transcript `json:"-"`
Section *contracts.SectionMetadata `json:"section,omitempty"`
PromptID string `json:"prompt_id"`
Variables map[string]string `json:"variables"`
StageName string `json:"stage_name,omitempty"`
StartIndex int `json:"start_index"`
CaptureExchange bool
LLMClient contracts.StructuredLLMClient
DiagnosticsDir string
DiagnosticsWriter InteractionDiagnosticsWriter
StageIndex *int `json:"stage_index,omitempty"`
StageID *string `json:"stage_id,omitempty"`
ModuleKey string `json:"module_key"`
ModuleInstance string `json:"module_instance"`
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
WorkingTranscript *schema.Transcript `json:"-"`
Section *contracts.SectionMetadata `json:"section,omitempty"`
PromptID string `json:"prompt_id"`
Variables map[string]string `json:"variables"`
StageName string `json:"stage_name,omitempty"`
StartIndex int `json:"start_index"`
CaptureExchange bool
LLMClient contracts.StructuredLLMClient
Interactions structuredoutput.InteractionSink
}
// Result contains generated candidate proposals and optional diagnostics paths.
@@ -74,7 +56,7 @@ type Result struct {
Corrections []proposals.CorrectionProposal `json:"corrections"`
Enriched []proposals.EnrichedCorrectionProposal `json:"enriched"`
Operations []outcomes.Operation `json:"operations,omitempty"`
Artifacts InteractionArtifacts `json:"artifacts,omitempty"`
Artifacts structuredoutput.InteractionArtifacts `json:"artifacts,omitempty"`
}
// GenerateCandidates executes one structured LLM call and deterministically maps
@@ -105,13 +87,14 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
response StructuredCorrectionSet
completion contracts.StructuredCompletionResponse
callErr error
artifacts InteractionArtifacts
artifacts structuredoutput.InteractionArtifacts
)
coordinates := structuredoutput.DiagnosticContext{StageIndex: req.StageIndex, StageID: req.StageID, Role: "proposal", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: sectionIndexPtr(req.Section)}
completion, callErr = req.LLMClient.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
CaptureExchange: req.CaptureExchange,
PromptID: req.PromptID,
Variables: req.Variables, Schema: responseschema.CorrectionSetKey,
Context: structuredoutput.DiagnosticContext{StageIndex: req.StageIndex, StageID: req.StageID, Role: "proposal", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: sectionIndexPtr(req.Section)},
Context: coordinates,
}, &response)
sectionIndex := 0
if req.Section != nil {
@@ -126,16 +109,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
Counts: completion.Execution,
}
var writer InteractionDiagnosticsWriter
if req.DiagnosticsWriter != nil {
writer = req.DiagnosticsWriter
} else if strings.TrimSpace(req.DiagnosticsDir) != "" {
writer = diagnosticsWriterAdapter{
writer: llm.NewDiagnosticsWriter(filepath.Join(req.DiagnosticsDir, req.ModuleInstance)),
}
}
if writer != nil {
if req.Interactions != nil {
requestMetadata := map[string]any{
"stage_index": req.StageIndex, "stage_id": req.StageID,
"module_key": req.ModuleKey,
@@ -147,17 +121,11 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
"prompt": completion.Prompt,
}
requestMetadata["response_schema"] = responseschema.MustLookup(responseschema.CorrectionSetKey).DiagnosticsMap()
var diagnosticErr error
artifacts, diagnosticErr = writer.WriteInteraction(
stage,
requestMetadata,
completion.Prepared,
map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model},
errPayload(callErr),
)
if diagnosticErr != nil && req.ReportDiagnosticError != nil {
req.ReportDiagnosticError(fmt.Errorf("proposal interaction %s: %w", stage, diagnosticErr))
}
artifacts, _ = req.Interactions.WriteInteraction(structuredoutput.Interaction{
Name: stage, Context: coordinates, RequestMetadata: requestMetadata, RequestPayload: completion.Prepared,
ResponsePayload: map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model},
ErrorPayload: errPayload(callErr),
})
}
if callErr != nil {
@@ -234,23 +202,6 @@ func sectionIndexPtr(section *contracts.SectionMetadata) *int {
return &index
}
type diagnosticsWriterAdapter struct {
writer *llm.DiagnosticsWriter
}
func (a diagnosticsWriterAdapter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error) {
artifacts, err := a.writer.WriteInteraction(stage, requestMetadata, requestPayload, responsePayload, errorPayload)
if err != nil {
return InteractionArtifacts{}, err
}
return InteractionArtifacts{
RequestMetadataPath: artifacts.RequestMetadataPath,
RequestPayloadPath: artifacts.RequestPayloadPath,
ResponsePayloadPath: artifacts.ResponsePayloadPath,
ErrorPayloadPath: artifacts.ErrorPayloadPath,
}, nil
}
// segmentIDs records the request-local reference mapping in debug metadata.
func segmentIDs(transcript *schema.Transcript) []int {
if transcript == nil {

View File

@@ -12,6 +12,7 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
@@ -56,13 +57,11 @@ type captureDiagnosticsWriter struct {
lastRequestPayload any
}
func (w *captureDiagnosticsWriter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error) {
w.lastStage = stage
w.lastRequestMetadata = requestMetadata
w.lastRequestPayload = requestPayload
_ = responsePayload
_ = errorPayload
return InteractionArtifacts{}, w.err
func (w *captureDiagnosticsWriter) WriteInteraction(interaction structuredoutput.Interaction) (structuredoutput.InteractionArtifacts, error) {
w.lastStage = interaction.Name
w.lastRequestMetadata = interaction.RequestMetadata
w.lastRequestPayload = interaction.RequestPayload
return structuredoutput.InteractionArtifacts{}, w.err
}
func defaultRequest(t *testing.T) Request {
@@ -146,7 +145,7 @@ func TestGenerateCandidatesDiagnosticsIncludeSchemaMetadata(t *testing.T) {
diag := &captureDiagnosticsWriter{}
req := defaultRequest(t)
req.LLMClient = client
req.DiagnosticsWriter = diag
req.Interactions = diag
_, err := GenerateCandidates(context.Background(), req)
if err != nil {
@@ -316,8 +315,10 @@ func TestGenerateCandidatesDiagnosticsWritten(t *testing.T) {
}
req := defaultRequest(t)
req.LLMClient = client
req.DiagnosticsDir = t.TempDir()
dir := t.TempDir()
req.Interactions = llm.NewDiagnosticsWriter(dir)
req.Variables = map[string]string{"SectionJSON": "section text"}
req.Section = &contracts.SectionMetadata{Index: 3}
got, err := GenerateCandidates(context.Background(), req)
if err != nil {
@@ -353,13 +354,23 @@ func TestGenerateCandidatesDiagnosticsWritten(t *testing.T) {
if schemaMap["id"] != want.ID || schemaMap["version"] != want.Version || schemaMap["name"] != want.Name || schemaMap["sha256"] != want.SHA256 {
t.Fatalf("unexpected schema metadata in diagnostics: got=%v want=%+v", schemaMap, want)
}
var coordinates struct {
Context structuredoutput.DiagnosticContext `json:"context"`
}
if err := json.Unmarshal(raw, &coordinates); err != nil {
t.Fatal(err)
}
if c := coordinates.Context; c.Role != "proposal" || c.ModuleInstance != req.ModuleInstance || c.SectionIndex == nil || *c.SectionIndex != 3 {
t.Fatalf("proposal metadata: %s", raw)
}
}
func TestGenerateCandidatesClientError(t *testing.T) {
client := &fakeStructuredClient{err: errors.New("boom")}
req := defaultRequest(t)
req.LLMClient = client
req.DiagnosticsDir = t.TempDir()
dir := t.TempDir()
req.Interactions = llm.NewDiagnosticsWriter(dir)
got, err := GenerateCandidates(context.Background(), req)
if err == nil || !strings.Contains(err.Error(), "completion failed") {
@@ -368,12 +379,12 @@ func TestGenerateCandidatesClientError(t *testing.T) {
if len(got.Operations) != 1 || got.Operations[0].FailureReason != outcomes.ReasonInternalFailure || len(got.Corrections) != 0 {
t.Fatalf("expected fatal call facts without proposals, got %+v", got)
}
matches, globErr := filepath.Glob(filepath.Join(req.DiagnosticsDir, req.ModuleInstance, "*error-payload.json"))
matches, globErr := filepath.Glob(filepath.Join(dir, req.ModuleInstance, "*error-payload.json"))
if globErr != nil {
t.Fatalf("glob error: %v", globErr)
}
if len(matches) == 0 {
t.Fatalf("expected error diagnostics artifact under %s", req.DiagnosticsDir)
t.Fatalf("expected error diagnostics artifact under %s", dir)
}
}
@@ -381,11 +392,10 @@ func TestProposalDiagnosticFailureDoesNotChangeOperation(t *testing.T) {
cause := errors.New("diagnostic storage unavailable")
req := defaultRequest(t)
req.LLMClient = &fakeStructuredClient{responses: []StructuredCorrectionSet{{}}}
req.DiagnosticsWriter = &captureDiagnosticsWriter{err: cause}
var reported error
req.ReportDiagnosticError = func(err error) { reported = err }
sink := &captureDiagnosticsWriter{err: cause}
req.Interactions = sink
result, err := GenerateCandidates(context.Background(), req)
if err != nil || !errors.Is(reported, cause) || len(result.Operations) != 1 || !result.Operations[0].Succeeded || len(result.Operations[0].RecoveryReasons) != 0 || result.Artifacts.ResponsePayloadPath != "" {
t.Fatalf("debug failure changed outcome: %+v %v reported=%v", result, err, reported)
if err != nil || sink.lastStage == "" || len(result.Operations) != 1 || !result.Operations[0].Succeeded || len(result.Operations[0].RecoveryReasons) != 0 || result.Artifacts.ResponsePayloadPath != "" {
t.Fatalf("debug failure changed outcome: %+v %v", result, err)
}
}

View File

@@ -54,19 +54,18 @@ func ExecuteModuleProposal(ctx context.Context, req ModuleProposalRequest) (cont
generated, err := GenerateCandidates(ctx, Request{
StageIndex: req.ProposalRequest.StageIndex, StageID: req.ProposalRequest.StageID,
ModuleKey: req.ProposalRequest.RunSpec.ModuleKey,
ModuleInstance: req.ProposalRequest.RunSpec.InstanceName,
ReplacementPolicy: req.ProposalRequest.RunSpec.ReplacementPolicy,
WorkingTranscript: req.ProposalRequest.WorkingTranscript,
Section: req.ProposalRequest.Section,
PromptID: promptMetadata.PromptID,
Variables: variables,
StageName: stagename.ModuleProposal(req.ProposalRequest.RunSpec.InstanceName, sectionIndexPtr(req.ProposalRequest.Section)),
StartIndex: 0,
CaptureExchange: req.ProposalRequest.RunSpec.CaptureProposalExchange,
LLMClient: req.ProposalRequest.LLMClient,
DiagnosticsDir: req.ProposalRequest.DiagnosticsDir,
ReportDiagnosticError: req.ProposalRequest.ReportDiagnosticError,
ModuleKey: req.ProposalRequest.RunSpec.ModuleKey,
ModuleInstance: req.ProposalRequest.RunSpec.InstanceName,
ReplacementPolicy: req.ProposalRequest.RunSpec.ReplacementPolicy,
WorkingTranscript: req.ProposalRequest.WorkingTranscript,
Section: req.ProposalRequest.Section,
PromptID: promptMetadata.PromptID,
Variables: variables,
StageName: stagename.ModuleProposal(req.ProposalRequest.RunSpec.InstanceName, sectionIndexPtr(req.ProposalRequest.Section)),
StartIndex: 0,
CaptureExchange: req.ProposalRequest.RunSpec.CaptureProposalExchange,
LLMClient: req.ProposalRequest.LLMClient,
Interactions: req.ProposalRequest.Interactions,
})
if err != nil {
return contracts.ProposalResult{Operations: generated.Operations}, err

View File

@@ -2,8 +2,10 @@ package runner
import (
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
@@ -11,7 +13,9 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
)
@@ -99,11 +103,12 @@ func TestRevalidationArtifactsUseProposalAndAttemptCoordinates(t *testing.T) {
t.Fatal(err)
}
dir := t.TempDir()
attempt := 1
attempt, section := 1, 4
sink := llm.NewDiagnosticsWriter(dir)
var paths []string
for _, index := range []int{2, 3} {
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar_1", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validator}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, ValidationLLMClient: client, DiagnosticsDir: dir, RepairProposalIndex: &index, RepairAttempt: &attempt}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar_1", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validator}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, ValidationLLMClient: client, Interactions: sink, SectionIndex: &section, RepairProposalIndex: &index, RepairAttempt: &attempt}
result, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
@@ -116,10 +121,29 @@ func TestRevalidationArtifactsUseProposalAndAttemptCoordinates(t *testing.T) {
if paths[0] == paths[1] {
t.Fatalf("colliding paths: %v", paths)
}
for _, path := range paths {
if _, err := os.Stat(path); err != nil {
for i, path := range paths {
base, ok := strings.CutSuffix(filepath.Base(path), "-response-payload.json")
if !ok {
t.Fatalf("response path: %s", path)
}
raw, err := os.ReadFile(filepath.Join(filepath.Dir(path), base+"-request-metadata.json"))
if err != nil {
t.Fatal(err)
}
var metadata struct {
Context structuredoutput.DiagnosticContext `json:"context"`
ProposalIndices []int `json:"proposal_indices"`
}
if err := json.Unmarshal(raw, &metadata); err != nil {
t.Fatal(err)
}
c := metadata.Context
if c.Role != "validation" || c.ValidatorName != "editorial_review" ||
c.BatchIndex == nil || *c.BatchIndex != 0 || c.SectionIndex == nil || *c.SectionIndex != section ||
c.ProposalIndex == nil || *c.ProposalIndex != i+2 || c.RepairAttempt == nil || *c.RepairAttempt != attempt ||
len(metadata.ProposalIndices) != 1 || metadata.ProposalIndices[0] != i+2 {
t.Fatalf("revalidation metadata: %s", raw)
}
}
}

View File

@@ -4,10 +4,8 @@ import (
"context"
"errors"
"fmt"
"path/filepath"
"slices"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
@@ -18,7 +16,7 @@ import (
// RepairAttempt holds optional debug evidence for one semantic request.
type RepairAttempt struct {
Artifacts llm.InteractionArtifacts
Artifacts structuredoutput.InteractionArtifacts
Revision *proposals.EnrichedCorrectionProposal
Decisions []ValidatorDecisionRecord
FailureReason outcomes.ReasonCode
@@ -65,7 +63,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
superseded := make(map[int]bool)
var branches []RepairOutcome
var repairOps []outcomes.Operation
debug := input.DiagnosticsDir != ""
debug := input.Interactions != nil
section := 0
if input.SectionIndex != nil {
section = *input.SectionIndex
@@ -165,13 +163,14 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
if err != nil {
failure = map[string]string{"error": err.Error()}
}
writer := llm.NewDiagnosticsWriter(filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName))
artifacts, writeErr := writer.WriteInteraction(name, map[string]any{"stage_index": input.Spec.Index, "stage_id": input.Spec.ID, "proposal_index": proposalIndex, "repair_attempt": attemptNumber, "prompt_id": promptID, "prompt": completion.Prompt, "feedback": findings}, completion.Prepared, map[string]any{"decoded": response, "final_content": completion.FinalContent, "revision": record.Revision, "decisions": record.Decisions, "rejections": validated.rejected}, failure)
if writeErr == nil {
record.Artifacts = artifacts
} else if input.ReportDiagnosticError != nil {
input.ReportDiagnosticError(fmt.Errorf("repair interaction %s: %w", name, writeErr))
}
record.Artifacts, _ = input.Interactions.WriteInteraction(structuredoutput.Interaction{
Name: name,
Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex, ProposalIndex: &proposalIndex, RepairAttempt: &attemptNumber},
RequestMetadata: map[string]any{"stage_index": input.Spec.Index, "stage_id": input.Spec.ID, "proposal_index": proposalIndex, "repair_attempt": attemptNumber, "prompt_id": promptID, "prompt": completion.Prompt, "feedback": findings},
RequestPayload: completion.Prepared,
ResponsePayload: map[string]any{"decoded": response, "final_content": completion.FinalContent, "revision": record.Revision, "decisions": record.Decisions, "rejections": validated.rejected},
ErrorPayload: failure,
})
branch.Attempts = append(branch.Attempts, record)
}
if callErr != nil {

View File

@@ -291,7 +291,8 @@ func TestRepairArtifactsAreUnique(t *testing.T) {
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}},
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "fox"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}},
}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar_1", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{}, DiagnosticsDir: t.TempDir()}
section := 4
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar_1", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: candidates, SectionIndex: &section, Exchange: &structuredoutput.Exchange{}, Interactions: llm.NewDiagnosticsWriter(t.TempDir())}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
@@ -315,6 +316,21 @@ func TestRepairArtifactsAreUnique(t *testing.T) {
if !json.Valid(data) {
t.Fatalf("invalid JSON artifact %s: %s", filepath.Base(path), data)
}
raw, err := os.ReadFile(branches[i].Attempts[0].Artifacts.RequestMetadataPath)
if err != nil {
t.Fatal(err)
}
var metadata struct {
Context structuredoutput.DiagnosticContext `json:"context"`
}
if err := json.Unmarshal(raw, &metadata); err != nil {
t.Fatal(err)
}
c := metadata.Context
if c.Role != "correction_repair" || c.ModuleInstance != "grammar_1" || c.SectionIndex == nil || *c.SectionIndex != section ||
c.ProposalIndex == nil || *c.ProposalIndex != i || c.RepairAttempt == nil || *c.RepairAttempt != 1 {
t.Fatalf("repair metadata: %s", raw)
}
var request proposal_generation.RepairRequest
if err := json.Unmarshal([]byte(client.calls[i].Variables["RepairRequestJSON"]), &request); err != nil {
t.Fatal(err)
@@ -332,16 +348,16 @@ func TestRepairArtifactFailureDoesNotChangeOutcome(t *testing.T) {
if err := os.WriteFile(path, []byte("x"), 0600); err != nil {
t.Fatal(err)
}
warnings := 0
sink := llm.NewDiagnosticsWriter(path)
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}, DiagnosticsDir: path, ReportDiagnosticError: func(error) { warnings++ }}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}, Interactions: sink}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
}
_, branches, err := repairCandidates(context.Background(), input, []int{1}, initial, "repair")
if err != nil || len(branches) != 1 || branches[0].Outcome != "withdrawn" || warnings != 1 {
t.Fatalf("branches=%+v warnings=%d err=%v", branches, warnings, err)
if err != nil || len(branches) != 1 || branches[0].Outcome != "withdrawn" || len(sink.Warnings()) != 1 {
t.Fatalf("branches=%+v warnings=%v err=%v", branches, sink.Warnings(), err)
}
}

View File

@@ -4,7 +4,6 @@ import (
"context"
"errors"
"fmt"
"path/filepath"
"sort"
"sync"
@@ -13,7 +12,6 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/promptcontext"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
@@ -94,14 +92,11 @@ type ValidatorRejectedChange struct {
// RunInput is the deterministic runner input.
type RunInput struct {
// ReportDiagnosticError may be called concurrently; it must not affect processing.
ReportDiagnosticError func(error) `json:"-"`
Config *config.Config
Transcript *schema.Transcript
ModuleSpecs []contracts.ModuleRunSpec
ProposalDiagnosticsDir string
ValidationDiagnosticsDir string
CaptureCandidates bool
Config *config.Config
Transcript *schema.Transcript
ModuleSpecs []contracts.ModuleRunSpec
Interactions structuredoutput.InteractionSink
CaptureCandidates bool
}
// RunOutput is the deterministic runner output.
@@ -172,17 +167,15 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
}
glossaryView := validators.PrepareGlossaryView(spec.Glossary, spec.ProtectedTerms)
pipelineResult, pipelineErr := runModulePipeline(ctx, collectSectionProposalsInput{
Module: module,
Spec: spec,
Policy: policy,
Sections: sections,
ProposalClient: spec.ProposalClient,
DiagnosticsDir: input.ProposalDiagnosticsDir,
ValidationClient: spec.ValidationClient,
ValidationDiagnosticsDir: input.ValidationDiagnosticsDir,
GlossaryVariables: &glossaryVariables,
GlossaryView: glossaryView,
ReportDiagnosticError: input.ReportDiagnosticError,
Module: module,
Spec: spec,
Policy: policy,
Sections: sections,
ProposalClient: spec.ProposalClient,
Interactions: input.Interactions,
ValidationClient: spec.ValidationClient,
GlossaryVariables: &glossaryVariables,
GlossaryView: glossaryView,
})
if input.CaptureCandidates {
candidates = append(candidates, pipelineResult.Candidates...)
@@ -335,18 +328,16 @@ func applyWithinSections(sections []chunking.Section, approved []proposals.Enric
}
type collectSectionProposalsInput struct {
// ReportDiagnosticError may be called concurrently; it must not affect processing.
ReportDiagnosticError func(error) `json:"-"`
Module contracts.TranscriptModule
Spec contracts.ModuleRunSpec
Policy proposals.ReplacementPolicy
Sections []chunking.Section
ProposalClient contracts.StructuredLLMClient
DiagnosticsDir string
ValidationClient contracts.StructuredLLMClient
ValidationDiagnosticsDir string
GlossaryVariables *promptcontext.GlossaryVariables
GlossaryView *validators.GlossaryView
Interactions structuredoutput.InteractionSink
Module contracts.TranscriptModule
Spec contracts.ModuleRunSpec
Policy proposals.ReplacementPolicy
Sections []chunking.Section
ProposalClient contracts.StructuredLLMClient
ValidationClient contracts.StructuredLLMClient
GlossaryVariables *promptcontext.GlossaryVariables
GlossaryView *validators.GlossaryView
}
type sectionProposalResult struct {
@@ -403,14 +394,13 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI
proposalResult, err := input.Module.Propose(runCtx, contracts.ProposalRequest{
ExecutionContext: contracts.ExecutionContext{
StageIndex: &input.Spec.Index, StageID: input.Spec.ID,
Settings: input.Spec.Settings,
WorkingTranscript: transcriptFromSection(section),
Glossary: input.Spec.Glossary,
ProtectedTerms: input.Spec.ProtectedTerms,
GlossaryVariables: input.GlossaryVariables,
Section: &meta,
DiagnosticsDir: input.DiagnosticsDir,
ReportDiagnosticError: input.ReportDiagnosticError,
Settings: input.Spec.Settings,
WorkingTranscript: transcriptFromSection(section),
Glossary: input.Spec.Glossary,
ProtectedTerms: input.Spec.ProtectedTerms,
GlossaryVariables: input.GlossaryVariables,
Section: &meta,
Interactions: input.Interactions,
},
RunSpec: contracts.ModuleRunSpec{
ModuleKey: input.Spec.ModuleKey,
@@ -548,17 +538,16 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
go func(sectionPos int, enriched []proposals.EnrichedCorrectionProposal, sectionMetadata contracts.SectionMetadata) {
defer vwg.Done()
validationInput := validateSectionCandidatesInput{
Spec: input.Spec,
Policy: input.Policy,
WorkingTranscript: transcriptFromSection(sectionResult.section),
SectionIndex: &sectionMetadata.Index,
Validators: validatorsOrdered,
SectionEnriched: enriched,
Exchange: sectionResult.exchange,
ValidationLLMClient: input.ValidationClient,
DiagnosticsDir: input.ValidationDiagnosticsDir,
GlossaryView: input.GlossaryView,
ReportDiagnosticError: input.ReportDiagnosticError,
Spec: input.Spec,
Policy: input.Policy,
WorkingTranscript: transcriptFromSection(sectionResult.section),
SectionIndex: &sectionMetadata.Index,
Validators: validatorsOrdered,
SectionEnriched: enriched,
Exchange: sectionResult.exchange,
ValidationLLMClient: input.ValidationClient,
GlossaryView: input.GlossaryView,
Interactions: input.Interactions,
}
if input.Spec.Settings.LLM.CorrectionRepairAttempts > 0 && len(enriched) > 0 {
validationInput.Section = validators.NewSectionLookup(validationInput.WorkingTranscript)
@@ -656,17 +645,15 @@ type validateSectionCandidatesInput struct {
RepairProposalIndex *int
RepairAttempt *int
Exchange *structuredoutput.Exchange
// ReportDiagnosticError may be called concurrently; it must not affect processing.
ReportDiagnosticError func(error) `json:"-"`
Spec contracts.ModuleRunSpec
Policy proposals.ReplacementPolicy
WorkingTranscript *schema.Transcript
SectionIndex *int
Validators []contracts.Validator
SectionEnriched []proposals.EnrichedCorrectionProposal
ValidationLLMClient contracts.StructuredLLMClient
DiagnosticsDir string
GlossaryView *validators.GlossaryView
Spec contracts.ModuleRunSpec
Policy proposals.ReplacementPolicy
WorkingTranscript *schema.Transcript
SectionIndex *int
Validators []contracts.Validator
SectionEnriched []proposals.EnrichedCorrectionProposal
ValidationLLMClient contracts.StructuredLLMClient
Interactions structuredoutput.InteractionSink
GlossaryView *validators.GlossaryView
}
type validateSectionCandidatesResult struct {
@@ -693,21 +680,11 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
evidence.Section = validators.NewSectionLookup(input.WorkingTranscript)
}
}
var interactionWriter *llmDiagnosticsWriterAdapter
for validatorOrder, validator := range input.Validators {
if err := ctx.Err(); err != nil {
return validateSectionCandidatesResult{decisions: decisions, rejected: rejected, operations: operations, filtered: filtered}, err
}
var diagnosticsWriter validators.InteractionDiagnosticsWriter
_, directLLMValidator := validator.(*validators.LLMBackedValidator)
if input.DiagnosticsDir != "" && (directLLMValidator || validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked) {
if interactionWriter == nil {
interactionWriter = &llmDiagnosticsWriterAdapter{dir: filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName)}
}
diagnosticsWriter = interactionWriter
}
submitted := eligible
if feedback {
submitted = feedbackCandidates(input, validator, noEffect, evidence)
@@ -721,15 +698,14 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
ModuleInstance: input.Spec.InstanceName,
SectionIndex: input.SectionIndex,
IndependentEvidence: feedback, RepairProposalIndex: input.RepairProposalIndex, RepairAttempt: input.RepairAttempt,
ValidatorOrder: &validatorOrder,
ReplacementPolicy: input.Policy,
Glossary: input.Spec.Glossary,
ProtectedTerms: input.Spec.ProtectedTerms,
GlossaryView: input.GlossaryView,
Settings: input.Spec.Settings,
LLMClient: input.ValidationLLMClient,
DiagnosticsWriter: diagnosticsWriter,
ReportDiagnosticError: input.ReportDiagnosticError,
ValidatorOrder: &validatorOrder,
ReplacementPolicy: input.Policy,
Glossary: input.Spec.Glossary,
ProtectedTerms: input.Spec.ProtectedTerms,
GlossaryView: input.GlossaryView,
Settings: input.Spec.Settings,
LLMClient: input.ValidationLLMClient,
Interactions: input.Interactions,
})
operations = append(operations, vResult.Operations...)
@@ -985,20 +961,3 @@ func cloneTranscript(t *schema.Transcript) *schema.Transcript {
}
return &schema.Transcript{Segments: segments}
}
type llmDiagnosticsWriterAdapter struct {
dir string
}
func (a *llmDiagnosticsWriterAdapter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (validators.InteractionArtifacts, error) {
art, err := llm.NewDiagnosticsWriter(a.dir).WriteInteraction(stage, requestMetadata, requestPayload, responsePayload, errorPayload)
if err != nil {
return validators.InteractionArtifacts{}, err
}
return validators.InteractionArtifacts{
RequestMetadataPath: art.RequestMetadataPath,
RequestPayloadPath: art.RequestPayloadPath,
ResponsePayloadPath: art.ResponsePayloadPath,
ErrorPayloadPath: art.ErrorPayloadPath,
}, nil
}

View File

@@ -16,6 +16,7 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
@@ -1163,10 +1164,10 @@ func TestRunnerLLMValidatorDiagnosticsWritten(t *testing.T) {
}},
}})
out, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}},
ValidationDiagnosticsDir: diagDir,
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}},
Interactions: llm.NewDiagnosticsWriter(diagDir),
})
if err != nil {
t.Fatalf("Run error: %v", err)
@@ -1217,7 +1218,7 @@ func (m proposalGenerationModule) Propose(ctx context.Context, req contracts.Pro
PromptID: "modules.grammar.proposal",
Variables: map[string]string{"TranscriptDescriptionBlock": "", "GlossaryJSON": "[]", "SectionJSON": "[]"},
LLMClient: req.LLMClient,
DiagnosticsDir: req.DiagnosticsDir,
Interactions: req.Interactions,
})
if err != nil {
return contracts.ProposalResult{}, err
@@ -1263,10 +1264,10 @@ func TestRunnerProposalGenerationHelperFlowsThroughPipeline(t *testing.T) {
}})
out, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ProposalClient: client, Settings: config.Stage{}}},
ProposalDiagnosticsDir: diagDir,
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ProposalClient: client, Settings: config.Stage{}}},
Interactions: llm.NewDiagnosticsWriter(diagDir),
})
if err != nil {
t.Fatalf("unexpected run error: %v", err)

View File

@@ -0,0 +1,24 @@
package structuredoutput
// InteractionSink captures optional diagnostics. Calls can occur concurrently.
// It returns paths for the artifacts that were written even when others fail.
// A sink can collect write errors for presentation outside the processing result.
type InteractionSink interface {
WriteInteraction(Interaction) (InteractionArtifacts, error)
}
type Interaction struct {
Name string
Context DiagnosticContext
RequestMetadata map[string]any
RequestPayload any
ResponsePayload any
ErrorPayload any
}
type InteractionArtifacts struct {
RequestMetadataPath string `json:"request_metadata_path,omitempty"`
RequestPayloadPath string `json:"request_payload_path,omitempty"`
ResponsePayloadPath string `json:"response_payload_path,omitempty"`
ErrorPayloadPath string `json:"error_payload_path,omitempty"`
}

View File

@@ -123,10 +123,11 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
var response LLMValidationResponse
var completion StructuredCompletionResponse
promptMetadata := validatorPromptMetadata(v.validatorType)
coordinates := structuredoutput.DiagnosticContext{StageIndex: req.StageIndex, StageID: req.StageID, Role: "validation", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: req.SectionIndex, BatchIndex: &batch.BatchIndex, ValidatorName: v.name, ProposalIndex: req.RepairProposalIndex, RepairAttempt: req.RepairAttempt}
completion, err = req.LLMClient.CompleteStructured(ctx, StructuredCompletionRequest{
PromptID: promptMetadata.PromptID,
Variables: variables, Schema: responseschema.ValidatorDecisionSetKey,
Context: structuredoutput.DiagnosticContext{StageIndex: req.StageIndex, StageID: req.StageID, Role: "validation", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: req.SectionIndex, BatchIndex: &batch.BatchIndex, ValidatorName: v.name, ProposalIndex: req.RepairProposalIndex, RepairAttempt: req.RepairAttempt},
Context: coordinates,
}, &response)
batchIndex := batch.BatchIndex
operation := outcomes.Operation{
@@ -134,9 +135,9 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
Counts: completion.Execution,
}
writeInteraction := func(reconciled *validationReconciliation, callErr error) InteractionArtifacts {
if req.DiagnosticsWriter == nil {
return InteractionArtifacts{}
writeInteraction := func(reconciled *validationReconciliation, callErr error) structuredoutput.InteractionArtifacts {
if req.Interactions == nil {
return structuredoutput.InteractionArtifacts{}
}
stageInstance := req.ModuleInstance
if req.SectionIndex != nil {
@@ -151,10 +152,8 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
if reconciled != nil {
responsePayload["reconciliation"] = *reconciled
}
var diagnosticErr error
artifacts, diagnosticErr := req.DiagnosticsWriter.WriteInteraction(
stage,
map[string]any{
artifacts, _ := req.Interactions.WriteInteraction(structuredoutput.Interaction{
Name: stage, Context: coordinates, RequestMetadata: map[string]any{
"stage_index": req.StageIndex, "stage_id": req.StageID, "module_key": req.ModuleKey,
"validator_name": v.name,
"validator_type": v.validatorType,
@@ -164,17 +163,14 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
"prompt": completion.Prompt,
"response_schema": responseSchema.DiagnosticsMap(),
},
completion.Prepared,
responsePayload,
errPayload(callErr),
)
if diagnosticErr != nil && req.ReportDiagnosticError != nil {
req.ReportDiagnosticError(fmt.Errorf("validation interaction %s: %w", stage, diagnosticErr))
}
RequestPayload: completion.Prepared,
ResponsePayload: responsePayload,
ErrorPayload: errPayload(callErr),
})
return artifacts
}
if err != nil {
writeInteraction(nil, err)
artifacts := writeInteraction(nil, err)
var failure *structuredoutput.Failure
if errors.As(err, &failure) {
operation.FailureReason = failure.Reason
@@ -185,7 +181,11 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
operation.ProposalIndices = batchProposalIndices(batch.Items)
operations = append(operations, operation)
if structuredoutput.IsRecoverable(err) && failure != nil && failure.Reason != "" {
llmDecisions = append(llmDecisions, rejectBatch(batch.Items, string(failure.Reason), "model validation was unavailable")...)
decisions := rejectBatch(batch.Items, string(failure.Reason), "model validation was unavailable")
for i := range decisions {
decisions[i].DiagnosticArtifactPath = artifacts.ResponsePayloadPath
}
llmDecisions = append(llmDecisions, decisions...)
continue
}
return partialResult(), fmt.Errorf("LLM validator %q completion failed: %w", v.name, err)

View File

@@ -30,13 +30,10 @@ type captureValidationDiagnosticsWriter struct {
lastResponsePayload any
}
func (w *captureValidationDiagnosticsWriter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error) {
_ = stage
_ = requestPayload
_ = errorPayload
w.lastRequestMetadata = requestMetadata
w.lastResponsePayload = responsePayload
return InteractionArtifacts{ResponsePayloadPath: w.path}, w.err
func (w *captureValidationDiagnosticsWriter) WriteInteraction(interaction structuredoutput.Interaction) (structuredoutput.InteractionArtifacts, error) {
w.lastRequestMetadata = interaction.RequestMetadata
w.lastResponsePayload = interaction.ResponsePayload
return structuredoutput.InteractionArtifacts{ResponsePayloadPath: w.path}, w.err
}
func (f *fakeStructuredLLMClient) CompleteStructured(ctx context.Context, req StructuredCompletionRequest, out any) (StructuredCompletionResponse, error) {
@@ -237,7 +234,7 @@ func TestLLMBackedValidatorReconcilesDecisionIdentities(t *testing.T) {
v, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(7, "gestures", "Jesters"), mk(11, "gestures", "actors"), mk(18, "gestures", "figures")})
req.LLMClient = client
req.DiagnosticsWriter = writer
req.Interactions = writer
result, err := v.Validate(context.Background(), req)
if err != nil || len(client.calls) != 1 || len(result.Decisions) != len(tc.want) || len(result.Operations) != 1 {
t.Fatalf("result=%+v calls=%d err=%v", result, len(client.calls), err)
@@ -343,7 +340,7 @@ func TestLLMBackedValidatorDiagnosticsIncludeSchemaMetadata(t *testing.T) {
}
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = client
req.DiagnosticsWriter = writer
req.Interactions = writer
_, err = v.Validate(context.Background(), req)
if err != nil {
t.Fatalf("validate error: %v", err)
@@ -399,12 +396,11 @@ func TestValidationDiagnosticFailureDoesNotChangeDecision(t *testing.T) {
cause := errors.New("diagnostic storage unavailable")
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}}}
req.DiagnosticsWriter = &captureValidationDiagnosticsWriter{err: cause}
var reported error
req.ReportDiagnosticError = func(err error) { reported = err }
sink := &captureValidationDiagnosticsWriter{err: cause}
req.Interactions = sink
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
result, err := validator.Validate(context.Background(), req)
if err != nil || !errors.Is(reported, cause) || len(result.Decisions) != 1 || !result.Decisions[0].Approved || result.Decisions[0].DiagnosticArtifactPath != "" || len(result.Operations) != 1 || !result.Operations[0].Succeeded || len(result.Operations[0].RecoveryReasons) != 0 {
t.Fatalf("debug failure changed outcome: %+v %v reported=%v", result, err, reported)
if err != nil || sink.lastRequestMetadata == nil || len(result.Decisions) != 1 || !result.Decisions[0].Approved || result.Decisions[0].DiagnosticArtifactPath != "" || len(result.Operations) != 1 || !result.Operations[0].Succeeded || len(result.Operations[0].RecoveryReasons) != 0 {
t.Fatalf("debug failure changed outcome: %+v %v", result, err)
}
}

View File

@@ -10,6 +10,7 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
)
const (
@@ -29,28 +30,26 @@ const (
// Request is the runtime input shared by deterministic validators.
type Request struct {
StageIndex *int `json:"stage_index,omitempty"`
StageID *string `json:"stage_id,omitempty"`
// ReportDiagnosticError may be called concurrently; it must not affect processing.
ReportDiagnosticError func(error) `json:"-"`
IndependentEvidence bool `json:"-"`
RepairProposalIndex *int `json:"repair_proposal_index,omitempty"`
RepairAttempt *int `json:"repair_attempt,omitempty"`
SectionIndex *int `json:"section_index,omitempty"`
WorkingTranscript *schema.Transcript `json:"-"`
Section *SectionLookup `json:"-"`
Previews map[int]proposals.SegmentPreviewResult `json:"-"`
CandidateProposal []proposals.EnrichedCorrectionProposal `json:"candidate_proposals"`
ModuleKey string `json:"module_key"`
ModuleInstance string `json:"module_instance"`
ValidatorOrder *int `json:"validator_order,omitempty"`
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
Glossary []glossary.Entry `json:"-"`
ProtectedTerms []glossary.Term `json:"-"`
GlossaryView *GlossaryView `json:"-"`
Settings config.Stage `json:"-"`
LLMClient StructuredLLMClient `json:"-"`
DiagnosticsWriter InteractionDiagnosticsWriter `json:"-"`
StageIndex *int `json:"stage_index,omitempty"`
StageID *string `json:"stage_id,omitempty"`
IndependentEvidence bool `json:"-"`
RepairProposalIndex *int `json:"repair_proposal_index,omitempty"`
RepairAttempt *int `json:"repair_attempt,omitempty"`
SectionIndex *int `json:"section_index,omitempty"`
WorkingTranscript *schema.Transcript `json:"-"`
Section *SectionLookup `json:"-"`
Previews map[int]proposals.SegmentPreviewResult `json:"-"`
CandidateProposal []proposals.EnrichedCorrectionProposal `json:"candidate_proposals"`
ModuleKey string `json:"module_key"`
ModuleInstance string `json:"module_instance"`
ValidatorOrder *int `json:"validator_order,omitempty"`
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
Glossary []glossary.Entry `json:"-"`
ProtectedTerms []glossary.Term `json:"-"`
GlossaryView *GlossaryView `json:"-"`
Settings config.Stage `json:"-"`
LLMClient StructuredLLMClient `json:"-"`
Interactions structuredoutput.InteractionSink `json:"-"`
}
// Decision is one validator decision for one proposal index.
@@ -81,17 +80,6 @@ type Result struct {
Operations []outcomes.Operation `json:"operations,omitempty"`
}
type InteractionDiagnosticsWriter interface {
WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error)
}
type InteractionArtifacts struct {
RequestMetadataPath string `json:"request_metadata_path,omitempty"`
RequestPayloadPath string `json:"request_payload_path,omitempty"`
ResponsePayloadPath string `json:"response_payload_path,omitempty"`
ErrorPayloadPath string `json:"error_payload_path,omitempty"`
}
// Validator is the deterministic runtime validator interface.
type Validator interface {
Name() string

View File

@@ -11,6 +11,7 @@ import (
domain "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
@@ -105,7 +106,7 @@ func TestGlossaryModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T)
ExecutionContext: contracts.ExecutionContext{
WorkingTranscript: tinyTranscript(),
Glossary: tinyGlossary(),
DiagnosticsDir: diagDir,
Interactions: llm.NewDiagnosticsWriter(diagDir),
},
RunSpec: contracts.ModuleRunSpec{
ModuleKey: "glossary",

View File

@@ -11,6 +11,7 @@ import (
domain "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
@@ -114,7 +115,7 @@ func TestGrammarModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T) {
ExecutionContext: contracts.ExecutionContext{
WorkingTranscript: tinyTranscript(),
Glossary: tinyGlossary(),
DiagnosticsDir: diagDir,
Interactions: llm.NewDiagnosticsWriter(diagDir),
},
RunSpec: contracts.ModuleRunSpec{
ModuleKey: "grammar",

View File

@@ -11,6 +11,7 @@ import (
domain "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
@@ -106,7 +107,7 @@ func TestHomophonesModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T
ExecutionContext: contracts.ExecutionContext{
WorkingTranscript: tinyTranscript(),
Glossary: tinyGlossary(),
DiagnosticsDir: diagDir,
Interactions: llm.NewDiagnosticsWriter(diagDir),
},
RunSpec: contracts.ModuleRunSpec{
ModuleKey: "homophones",

View File

@@ -11,6 +11,7 @@ import (
domain "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
@@ -106,7 +107,7 @@ func TestSpokenWordModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T
ExecutionContext: contracts.ExecutionContext{
WorkingTranscript: tinyTranscript(),
Glossary: tinyGlossary(),
DiagnosticsDir: diagDir,
Interactions: llm.NewDiagnosticsWriter(diagDir),
},
RunSpec: contracts.ModuleRunSpec{
ModuleKey: "spoken_word",