Unify optional interaction capture and warnings
This commit is contained in:
@@ -73,17 +73,15 @@ func SectionMetadataFromSection(section chunking.Section) SectionMetadata {
|
||||
|
||||
// ExecutionContext carries shared execution state for proposal generation and validation.
|
||||
type ExecutionContext struct {
|
||||
StageIndex *int `json:"stage_index,omitempty"`
|
||||
StageID *string `json:"stage_id,omitempty"`
|
||||
// ReportDiagnosticError may be called concurrently; it must not affect processing.
|
||||
ReportDiagnosticError func(error) `json:"-"`
|
||||
Settings config.Stage `json:"-"`
|
||||
WorkingTranscript *schema.Transcript `json:"-"`
|
||||
Glossary []glossary.Entry `json:"-"`
|
||||
ProtectedTerms []glossary.Term `json:"-"`
|
||||
GlossaryVariables *promptcontext.GlossaryVariables `json:"-"`
|
||||
Section *SectionMetadata `json:"section,omitempty"`
|
||||
DiagnosticsDir string `json:"diagnostics_dir,omitempty"`
|
||||
StageIndex *int `json:"stage_index,omitempty"`
|
||||
StageID *string `json:"stage_id,omitempty"`
|
||||
Settings config.Stage `json:"-"`
|
||||
WorkingTranscript *schema.Transcript `json:"-"`
|
||||
Glossary []glossary.Entry `json:"-"`
|
||||
ProtectedTerms []glossary.Term `json:"-"`
|
||||
GlossaryVariables *promptcontext.GlossaryVariables `json:"-"`
|
||||
Section *SectionMetadata `json:"section,omitempty"`
|
||||
Interactions structuredoutput.InteractionSink `json:"-"`
|
||||
}
|
||||
|
||||
// ProposalRequest is the input to module proposal generation.
|
||||
|
||||
@@ -2,68 +2,120 @@ package llm
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||
)
|
||||
|
||||
var stageSanitizer = regexp.MustCompile(`[^a-zA-Z0-9_.-]+`)
|
||||
|
||||
// InteractionArtifacts contains file paths for written prompt/response
|
||||
// diagnostics.
|
||||
type InteractionArtifacts struct {
|
||||
RequestMetadataPath string
|
||||
RequestPayloadPath string
|
||||
ResponsePayloadPath string
|
||||
ErrorPayloadPath string
|
||||
}
|
||||
|
||||
// DiagnosticsWriter writes generic machine-readable LLM interaction artifacts.
|
||||
// DiagnosticsWriter writes interaction artifacts under a single run directory.
|
||||
type DiagnosticsWriter struct {
|
||||
dir string
|
||||
dir string
|
||||
mu sync.Mutex
|
||||
failures map[interactionFailureKey]*interactionFailure
|
||||
order []interactionFailureKey
|
||||
}
|
||||
|
||||
type interactionFailureKey struct {
|
||||
role, action, cause string
|
||||
}
|
||||
|
||||
type interactionFailure struct {
|
||||
count int
|
||||
path string
|
||||
}
|
||||
|
||||
// NewDiagnosticsWriter creates a diagnostics writer rooted at dir.
|
||||
func NewDiagnosticsWriter(dir string) *DiagnosticsWriter {
|
||||
return &DiagnosticsWriter{dir: dir}
|
||||
}
|
||||
|
||||
// WriteInteraction writes request/response/error artifacts for a generic LLM
|
||||
// interaction stage.
|
||||
func (w *DiagnosticsWriter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error) {
|
||||
func (w *DiagnosticsWriter) WriteInteraction(interaction structuredoutput.Interaction) (structuredoutput.InteractionArtifacts, error) {
|
||||
var paths structuredoutput.InteractionArtifacts
|
||||
dir := w.dir
|
||||
if instance := interaction.Context.ModuleInstance; instance != "" {
|
||||
dir = filepath.Join(dir, sanitizeStageName(instance))
|
||||
}
|
||||
var err error
|
||||
if strings.TrimSpace(w.dir) == "" {
|
||||
return InteractionArtifacts{}, fmt.Errorf("diagnostics directory must not be empty")
|
||||
err = fmt.Errorf("diagnostics directory must not be empty")
|
||||
} else {
|
||||
err = os.MkdirAll(dir, 0o755)
|
||||
}
|
||||
if err := os.MkdirAll(w.dir, 0o755); err != nil {
|
||||
return InteractionArtifacts{}, fmt.Errorf("create diagnostics directory: %w", err)
|
||||
base := sanitizeStageName(interaction.Name)
|
||||
if err != nil {
|
||||
w.recordFailure(interaction.Context.Role, "create directory", filepath.Join(dir, base), err)
|
||||
return paths, fmt.Errorf("create diagnostics directory %q: %w", dir, err)
|
||||
}
|
||||
|
||||
base := sanitizeStageName(stage)
|
||||
paths := InteractionArtifacts{
|
||||
RequestMetadataPath: filepath.Join(w.dir, base+"-request-metadata.json"),
|
||||
RequestPayloadPath: filepath.Join(w.dir, base+"-request-payload.json"),
|
||||
ResponsePayloadPath: filepath.Join(w.dir, base+"-response-payload.json"),
|
||||
metadata := maps.Clone(interaction.RequestMetadata)
|
||||
if metadata == nil {
|
||||
metadata = make(map[string]any)
|
||||
}
|
||||
|
||||
if err := w.writeJSON(paths.RequestMetadataPath, requestMetadata); err != nil {
|
||||
return InteractionArtifacts{}, err
|
||||
metadata["context"] = interaction.Context
|
||||
type artifactFile struct {
|
||||
suffix string
|
||||
payload any
|
||||
path *string
|
||||
}
|
||||
if err := w.writeJSON(paths.RequestPayloadPath, requestPayload); err != nil {
|
||||
return InteractionArtifacts{}, err
|
||||
artifacts := []artifactFile{
|
||||
{"request-metadata", metadata, &paths.RequestMetadataPath},
|
||||
{"request-payload", interaction.RequestPayload, &paths.RequestPayloadPath},
|
||||
{"response-payload", interaction.ResponsePayload, &paths.ResponsePayloadPath},
|
||||
}
|
||||
if err := w.writeJSON(paths.ResponsePayloadPath, responsePayload); err != nil {
|
||||
return InteractionArtifacts{}, err
|
||||
if interaction.ErrorPayload != nil {
|
||||
artifacts = append(artifacts, artifactFile{"error-payload", interaction.ErrorPayload, &paths.ErrorPayloadPath})
|
||||
}
|
||||
if errorPayload != nil {
|
||||
paths.ErrorPayloadPath = filepath.Join(w.dir, base+"-error-payload.json")
|
||||
if err := w.writeJSON(paths.ErrorPayloadPath, errorPayload); err != nil {
|
||||
return InteractionArtifacts{}, err
|
||||
var failures []error
|
||||
for _, artifact := range artifacts {
|
||||
path := filepath.Join(dir, base+"-"+artifact.suffix+".json")
|
||||
if action, err := writeInteractionJSON(path, artifact.payload); err != nil {
|
||||
w.recordFailure(interaction.Context.Role, action+" "+artifact.suffix, path, err)
|
||||
failures = append(failures, fmt.Errorf("%s %q: %w", action, path, err))
|
||||
continue
|
||||
}
|
||||
*artifact.path = path
|
||||
}
|
||||
return paths, errors.Join(failures...)
|
||||
}
|
||||
|
||||
return paths, nil
|
||||
func (w *DiagnosticsWriter) recordFailure(role, action, path string, err error) {
|
||||
var pathErr *os.PathError
|
||||
if errors.As(err, &pathErr) {
|
||||
err = pathErr.Err
|
||||
}
|
||||
key := interactionFailureKey{role: role, action: action, cause: err.Error()}
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.failures == nil {
|
||||
w.failures = make(map[interactionFailureKey]*interactionFailure)
|
||||
}
|
||||
group := w.failures[key]
|
||||
if group == nil {
|
||||
group = &interactionFailure{path: path}
|
||||
w.failures[key] = group
|
||||
w.order = append(w.order, key)
|
||||
}
|
||||
group.count++
|
||||
}
|
||||
|
||||
// Warnings summarizes capture failures after the callers join.
|
||||
func (w *DiagnosticsWriter) Warnings() []error {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
warnings := make([]error, 0, len(w.order))
|
||||
for _, key := range w.order {
|
||||
group := w.failures[key]
|
||||
warnings = append(warnings, fmt.Errorf("%s interaction: %s failures (%d), first at %q: %s", key.role, key.action, group.count, group.path, key.cause))
|
||||
}
|
||||
return warnings
|
||||
}
|
||||
|
||||
func sanitizeStageName(stage string) string {
|
||||
@@ -79,14 +131,10 @@ func sanitizeStageName(stage string) string {
|
||||
return safe
|
||||
}
|
||||
|
||||
func (w *DiagnosticsWriter) writeJSON(path string, payload any) error {
|
||||
func writeInteractionJSON(path string, payload any) (string, error) {
|
||||
encoded, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal json artifact %q: %w", filepath.Base(path), err)
|
||||
return "marshal", err
|
||||
}
|
||||
encoded = append(encoded, '\n')
|
||||
if err := os.WriteFile(path, encoded, 0o644); err != nil {
|
||||
return fmt.Errorf("write json artifact %q: %w", filepath.Base(path), err)
|
||||
}
|
||||
return nil
|
||||
return "write", os.WriteFile(path, append(encoded, '\n'), 0o644)
|
||||
}
|
||||
|
||||
@@ -4,20 +4,24 @@ import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||
)
|
||||
|
||||
func TestDiagnosticsWriterProducesValidJSONArtifacts(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writer := NewDiagnosticsWriter(dir)
|
||||
|
||||
artifacts, err := writer.WriteInteraction(
|
||||
"validation:batch-1",
|
||||
map[string]any{"stage": "validation", "attempt": 1},
|
||||
map[string]any{"messages": []map[string]any{{"role": "user", "content": "hello"}}},
|
||||
map[string]any{"result": "ok"},
|
||||
nil,
|
||||
)
|
||||
artifacts, err := writer.WriteInteraction(structuredoutput.Interaction{
|
||||
Name: "validation:batch-1",
|
||||
Context: structuredoutput.DiagnosticContext{Role: "validation", ModuleInstance: "grammar_1"},
|
||||
RequestMetadata: map[string]any{"attempt": 1},
|
||||
RequestPayload: map[string]any{"messages": []map[string]any{{"role": "user", "content": "hello"}}},
|
||||
ResponsePayload: map[string]any{"result": "ok"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("WriteInteraction: %v", err)
|
||||
}
|
||||
@@ -30,6 +34,48 @@ func TestDiagnosticsWriterProducesValidJSONArtifacts(t *testing.T) {
|
||||
assertValidJSONFile(t, artifacts.ResponsePayloadPath)
|
||||
}
|
||||
|
||||
func TestDiagnosticsWriterGroupsFailuresAndContinuesCapture(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writer := NewDiagnosticsWriter(dir)
|
||||
var workers sync.WaitGroup
|
||||
for _, name := range []string{"first", "second"} {
|
||||
blocked := filepath.Join(dir, "grammar_1", name+"-response-payload.json")
|
||||
if err := os.MkdirAll(blocked, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
workers.Go(func() {
|
||||
paths, err := writer.WriteInteraction(structuredoutput.Interaction{
|
||||
Name: name, Context: structuredoutput.DiagnosticContext{Role: "validation", ModuleInstance: "grammar_1"},
|
||||
ErrorPayload: map[string]string{"error": "model unavailable"},
|
||||
})
|
||||
if err == nil || paths.RequestPayloadPath == "" || paths.ResponsePayloadPath != "" || paths.ErrorPayloadPath == "" {
|
||||
t.Errorf("paths=%+v error=%v", paths, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
workers.Wait()
|
||||
warnings := writer.Warnings()
|
||||
if len(warnings) != 1 || !strings.Contains(warnings[0].Error(), "validation interaction: write response-payload failures (2)") ||
|
||||
!strings.Contains(warnings[0].Error(), filepath.Join(dir, "grammar_1")) {
|
||||
t.Fatalf("warnings: %v", warnings)
|
||||
}
|
||||
paths, err := writer.WriteInteraction(structuredoutput.Interaction{
|
||||
Name: "later", Context: structuredoutput.DiagnosticContext{Role: "validation", ModuleInstance: "grammar_1"},
|
||||
ResponsePayload: map[string]string{"final_content": "{"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, err := os.ReadFile(paths.ResponsePayloadPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var decoded map[string]string
|
||||
if err := json.Unmarshal(raw, &decoded); err != nil || decoded["final_content"] != "{" {
|
||||
t.Fatalf("later artifact: %s (%v)", raw, err)
|
||||
}
|
||||
}
|
||||
|
||||
func assertValidJSONFile(t *testing.T, path string) {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(path)
|
||||
|
||||
@@ -7,12 +7,10 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
|
||||
@@ -20,19 +18,6 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||
)
|
||||
|
||||
// InteractionDiagnosticsWriter writes machine-readable prompt/response artifacts.
|
||||
type InteractionDiagnosticsWriter interface {
|
||||
WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error)
|
||||
}
|
||||
|
||||
// InteractionArtifacts contains written diagnostics paths.
|
||||
type InteractionArtifacts struct {
|
||||
RequestMetadataPath string `json:"request_metadata_path,omitempty"`
|
||||
RequestPayloadPath string `json:"request_payload_path,omitempty"`
|
||||
ResponsePayloadPath string `json:"response_payload_path,omitempty"`
|
||||
ErrorPayloadPath string `json:"error_payload_path,omitempty"`
|
||||
}
|
||||
|
||||
// StructuredCorrectionProposal is one LLM response correction payload.
|
||||
type StructuredCorrectionProposal struct {
|
||||
SegmentReference int `json:"id"`
|
||||
@@ -48,23 +33,20 @@ type StructuredCorrectionSet struct {
|
||||
|
||||
// Request captures reusable proposal-generation inputs for future modules.
|
||||
type Request struct {
|
||||
StageIndex *int `json:"stage_index,omitempty"`
|
||||
StageID *string `json:"stage_id,omitempty"`
|
||||
// ReportDiagnosticError may be called concurrently; it must not affect processing.
|
||||
ReportDiagnosticError func(error) `json:"-"`
|
||||
ModuleKey string `json:"module_key"`
|
||||
ModuleInstance string `json:"module_instance"`
|
||||
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
|
||||
WorkingTranscript *schema.Transcript `json:"-"`
|
||||
Section *contracts.SectionMetadata `json:"section,omitempty"`
|
||||
PromptID string `json:"prompt_id"`
|
||||
Variables map[string]string `json:"variables"`
|
||||
StageName string `json:"stage_name,omitempty"`
|
||||
StartIndex int `json:"start_index"`
|
||||
CaptureExchange bool
|
||||
LLMClient contracts.StructuredLLMClient
|
||||
DiagnosticsDir string
|
||||
DiagnosticsWriter InteractionDiagnosticsWriter
|
||||
StageIndex *int `json:"stage_index,omitempty"`
|
||||
StageID *string `json:"stage_id,omitempty"`
|
||||
ModuleKey string `json:"module_key"`
|
||||
ModuleInstance string `json:"module_instance"`
|
||||
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
|
||||
WorkingTranscript *schema.Transcript `json:"-"`
|
||||
Section *contracts.SectionMetadata `json:"section,omitempty"`
|
||||
PromptID string `json:"prompt_id"`
|
||||
Variables map[string]string `json:"variables"`
|
||||
StageName string `json:"stage_name,omitempty"`
|
||||
StartIndex int `json:"start_index"`
|
||||
CaptureExchange bool
|
||||
LLMClient contracts.StructuredLLMClient
|
||||
Interactions structuredoutput.InteractionSink
|
||||
}
|
||||
|
||||
// Result contains generated candidate proposals and optional diagnostics paths.
|
||||
@@ -74,7 +56,7 @@ type Result struct {
|
||||
Corrections []proposals.CorrectionProposal `json:"corrections"`
|
||||
Enriched []proposals.EnrichedCorrectionProposal `json:"enriched"`
|
||||
Operations []outcomes.Operation `json:"operations,omitempty"`
|
||||
Artifacts InteractionArtifacts `json:"artifacts,omitempty"`
|
||||
Artifacts structuredoutput.InteractionArtifacts `json:"artifacts,omitempty"`
|
||||
}
|
||||
|
||||
// GenerateCandidates executes one structured LLM call and deterministically maps
|
||||
@@ -105,13 +87,14 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
response StructuredCorrectionSet
|
||||
completion contracts.StructuredCompletionResponse
|
||||
callErr error
|
||||
artifacts InteractionArtifacts
|
||||
artifacts structuredoutput.InteractionArtifacts
|
||||
)
|
||||
coordinates := structuredoutput.DiagnosticContext{StageIndex: req.StageIndex, StageID: req.StageID, Role: "proposal", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: sectionIndexPtr(req.Section)}
|
||||
completion, callErr = req.LLMClient.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||
CaptureExchange: req.CaptureExchange,
|
||||
PromptID: req.PromptID,
|
||||
Variables: req.Variables, Schema: responseschema.CorrectionSetKey,
|
||||
Context: structuredoutput.DiagnosticContext{StageIndex: req.StageIndex, StageID: req.StageID, Role: "proposal", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: sectionIndexPtr(req.Section)},
|
||||
Context: coordinates,
|
||||
}, &response)
|
||||
sectionIndex := 0
|
||||
if req.Section != nil {
|
||||
@@ -126,16 +109,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
Counts: completion.Execution,
|
||||
}
|
||||
|
||||
var writer InteractionDiagnosticsWriter
|
||||
if req.DiagnosticsWriter != nil {
|
||||
writer = req.DiagnosticsWriter
|
||||
} else if strings.TrimSpace(req.DiagnosticsDir) != "" {
|
||||
writer = diagnosticsWriterAdapter{
|
||||
writer: llm.NewDiagnosticsWriter(filepath.Join(req.DiagnosticsDir, req.ModuleInstance)),
|
||||
}
|
||||
}
|
||||
|
||||
if writer != nil {
|
||||
if req.Interactions != nil {
|
||||
requestMetadata := map[string]any{
|
||||
"stage_index": req.StageIndex, "stage_id": req.StageID,
|
||||
"module_key": req.ModuleKey,
|
||||
@@ -147,17 +121,11 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
"prompt": completion.Prompt,
|
||||
}
|
||||
requestMetadata["response_schema"] = responseschema.MustLookup(responseschema.CorrectionSetKey).DiagnosticsMap()
|
||||
var diagnosticErr error
|
||||
artifacts, diagnosticErr = writer.WriteInteraction(
|
||||
stage,
|
||||
requestMetadata,
|
||||
completion.Prepared,
|
||||
map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model},
|
||||
errPayload(callErr),
|
||||
)
|
||||
if diagnosticErr != nil && req.ReportDiagnosticError != nil {
|
||||
req.ReportDiagnosticError(fmt.Errorf("proposal interaction %s: %w", stage, diagnosticErr))
|
||||
}
|
||||
artifacts, _ = req.Interactions.WriteInteraction(structuredoutput.Interaction{
|
||||
Name: stage, Context: coordinates, RequestMetadata: requestMetadata, RequestPayload: completion.Prepared,
|
||||
ResponsePayload: map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model},
|
||||
ErrorPayload: errPayload(callErr),
|
||||
})
|
||||
}
|
||||
|
||||
if callErr != nil {
|
||||
@@ -234,23 +202,6 @@ func sectionIndexPtr(section *contracts.SectionMetadata) *int {
|
||||
return &index
|
||||
}
|
||||
|
||||
type diagnosticsWriterAdapter struct {
|
||||
writer *llm.DiagnosticsWriter
|
||||
}
|
||||
|
||||
func (a diagnosticsWriterAdapter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error) {
|
||||
artifacts, err := a.writer.WriteInteraction(stage, requestMetadata, requestPayload, responsePayload, errorPayload)
|
||||
if err != nil {
|
||||
return InteractionArtifacts{}, err
|
||||
}
|
||||
return InteractionArtifacts{
|
||||
RequestMetadataPath: artifacts.RequestMetadataPath,
|
||||
RequestPayloadPath: artifacts.RequestPayloadPath,
|
||||
ResponsePayloadPath: artifacts.ResponsePayloadPath,
|
||||
ErrorPayloadPath: artifacts.ErrorPayloadPath,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// segmentIDs records the request-local reference mapping in debug metadata.
|
||||
func segmentIDs(transcript *schema.Transcript) []int {
|
||||
if transcript == nil {
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
|
||||
@@ -56,13 +57,11 @@ type captureDiagnosticsWriter struct {
|
||||
lastRequestPayload any
|
||||
}
|
||||
|
||||
func (w *captureDiagnosticsWriter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error) {
|
||||
w.lastStage = stage
|
||||
w.lastRequestMetadata = requestMetadata
|
||||
w.lastRequestPayload = requestPayload
|
||||
_ = responsePayload
|
||||
_ = errorPayload
|
||||
return InteractionArtifacts{}, w.err
|
||||
func (w *captureDiagnosticsWriter) WriteInteraction(interaction structuredoutput.Interaction) (structuredoutput.InteractionArtifacts, error) {
|
||||
w.lastStage = interaction.Name
|
||||
w.lastRequestMetadata = interaction.RequestMetadata
|
||||
w.lastRequestPayload = interaction.RequestPayload
|
||||
return structuredoutput.InteractionArtifacts{}, w.err
|
||||
}
|
||||
|
||||
func defaultRequest(t *testing.T) Request {
|
||||
@@ -146,7 +145,7 @@ func TestGenerateCandidatesDiagnosticsIncludeSchemaMetadata(t *testing.T) {
|
||||
diag := &captureDiagnosticsWriter{}
|
||||
req := defaultRequest(t)
|
||||
req.LLMClient = client
|
||||
req.DiagnosticsWriter = diag
|
||||
req.Interactions = diag
|
||||
|
||||
_, err := GenerateCandidates(context.Background(), req)
|
||||
if err != nil {
|
||||
@@ -316,8 +315,10 @@ func TestGenerateCandidatesDiagnosticsWritten(t *testing.T) {
|
||||
}
|
||||
req := defaultRequest(t)
|
||||
req.LLMClient = client
|
||||
req.DiagnosticsDir = t.TempDir()
|
||||
dir := t.TempDir()
|
||||
req.Interactions = llm.NewDiagnosticsWriter(dir)
|
||||
req.Variables = map[string]string{"SectionJSON": "section text"}
|
||||
req.Section = &contracts.SectionMetadata{Index: 3}
|
||||
|
||||
got, err := GenerateCandidates(context.Background(), req)
|
||||
if err != nil {
|
||||
@@ -353,13 +354,23 @@ func TestGenerateCandidatesDiagnosticsWritten(t *testing.T) {
|
||||
if schemaMap["id"] != want.ID || schemaMap["version"] != want.Version || schemaMap["name"] != want.Name || schemaMap["sha256"] != want.SHA256 {
|
||||
t.Fatalf("unexpected schema metadata in diagnostics: got=%v want=%+v", schemaMap, want)
|
||||
}
|
||||
var coordinates struct {
|
||||
Context structuredoutput.DiagnosticContext `json:"context"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &coordinates); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c := coordinates.Context; c.Role != "proposal" || c.ModuleInstance != req.ModuleInstance || c.SectionIndex == nil || *c.SectionIndex != 3 {
|
||||
t.Fatalf("proposal metadata: %s", raw)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateCandidatesClientError(t *testing.T) {
|
||||
client := &fakeStructuredClient{err: errors.New("boom")}
|
||||
req := defaultRequest(t)
|
||||
req.LLMClient = client
|
||||
req.DiagnosticsDir = t.TempDir()
|
||||
dir := t.TempDir()
|
||||
req.Interactions = llm.NewDiagnosticsWriter(dir)
|
||||
|
||||
got, err := GenerateCandidates(context.Background(), req)
|
||||
if err == nil || !strings.Contains(err.Error(), "completion failed") {
|
||||
@@ -368,12 +379,12 @@ func TestGenerateCandidatesClientError(t *testing.T) {
|
||||
if len(got.Operations) != 1 || got.Operations[0].FailureReason != outcomes.ReasonInternalFailure || len(got.Corrections) != 0 {
|
||||
t.Fatalf("expected fatal call facts without proposals, got %+v", got)
|
||||
}
|
||||
matches, globErr := filepath.Glob(filepath.Join(req.DiagnosticsDir, req.ModuleInstance, "*error-payload.json"))
|
||||
matches, globErr := filepath.Glob(filepath.Join(dir, req.ModuleInstance, "*error-payload.json"))
|
||||
if globErr != nil {
|
||||
t.Fatalf("glob error: %v", globErr)
|
||||
}
|
||||
if len(matches) == 0 {
|
||||
t.Fatalf("expected error diagnostics artifact under %s", req.DiagnosticsDir)
|
||||
t.Fatalf("expected error diagnostics artifact under %s", dir)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -381,11 +392,10 @@ func TestProposalDiagnosticFailureDoesNotChangeOperation(t *testing.T) {
|
||||
cause := errors.New("diagnostic storage unavailable")
|
||||
req := defaultRequest(t)
|
||||
req.LLMClient = &fakeStructuredClient{responses: []StructuredCorrectionSet{{}}}
|
||||
req.DiagnosticsWriter = &captureDiagnosticsWriter{err: cause}
|
||||
var reported error
|
||||
req.ReportDiagnosticError = func(err error) { reported = err }
|
||||
sink := &captureDiagnosticsWriter{err: cause}
|
||||
req.Interactions = sink
|
||||
result, err := GenerateCandidates(context.Background(), req)
|
||||
if err != nil || !errors.Is(reported, cause) || len(result.Operations) != 1 || !result.Operations[0].Succeeded || len(result.Operations[0].RecoveryReasons) != 0 || result.Artifacts.ResponsePayloadPath != "" {
|
||||
t.Fatalf("debug failure changed outcome: %+v %v reported=%v", result, err, reported)
|
||||
if err != nil || sink.lastStage == "" || len(result.Operations) != 1 || !result.Operations[0].Succeeded || len(result.Operations[0].RecoveryReasons) != 0 || result.Artifacts.ResponsePayloadPath != "" {
|
||||
t.Fatalf("debug failure changed outcome: %+v %v", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -54,19 +54,18 @@ func ExecuteModuleProposal(ctx context.Context, req ModuleProposalRequest) (cont
|
||||
|
||||
generated, err := GenerateCandidates(ctx, Request{
|
||||
StageIndex: req.ProposalRequest.StageIndex, StageID: req.ProposalRequest.StageID,
|
||||
ModuleKey: req.ProposalRequest.RunSpec.ModuleKey,
|
||||
ModuleInstance: req.ProposalRequest.RunSpec.InstanceName,
|
||||
ReplacementPolicy: req.ProposalRequest.RunSpec.ReplacementPolicy,
|
||||
WorkingTranscript: req.ProposalRequest.WorkingTranscript,
|
||||
Section: req.ProposalRequest.Section,
|
||||
PromptID: promptMetadata.PromptID,
|
||||
Variables: variables,
|
||||
StageName: stagename.ModuleProposal(req.ProposalRequest.RunSpec.InstanceName, sectionIndexPtr(req.ProposalRequest.Section)),
|
||||
StartIndex: 0,
|
||||
CaptureExchange: req.ProposalRequest.RunSpec.CaptureProposalExchange,
|
||||
LLMClient: req.ProposalRequest.LLMClient,
|
||||
DiagnosticsDir: req.ProposalRequest.DiagnosticsDir,
|
||||
ReportDiagnosticError: req.ProposalRequest.ReportDiagnosticError,
|
||||
ModuleKey: req.ProposalRequest.RunSpec.ModuleKey,
|
||||
ModuleInstance: req.ProposalRequest.RunSpec.InstanceName,
|
||||
ReplacementPolicy: req.ProposalRequest.RunSpec.ReplacementPolicy,
|
||||
WorkingTranscript: req.ProposalRequest.WorkingTranscript,
|
||||
Section: req.ProposalRequest.Section,
|
||||
PromptID: promptMetadata.PromptID,
|
||||
Variables: variables,
|
||||
StageName: stagename.ModuleProposal(req.ProposalRequest.RunSpec.InstanceName, sectionIndexPtr(req.ProposalRequest.Section)),
|
||||
StartIndex: 0,
|
||||
CaptureExchange: req.ProposalRequest.RunSpec.CaptureProposalExchange,
|
||||
LLMClient: req.ProposalRequest.LLMClient,
|
||||
Interactions: req.ProposalRequest.Interactions,
|
||||
})
|
||||
if err != nil {
|
||||
return contracts.ProposalResult{Operations: generated.Operations}, err
|
||||
|
||||
@@ -2,8 +2,10 @@ package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -11,7 +13,9 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
||||
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
|
||||
)
|
||||
@@ -99,11 +103,12 @@ func TestRevalidationArtifactsUseProposalAndAttemptCoordinates(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dir := t.TempDir()
|
||||
attempt := 1
|
||||
attempt, section := 1, 4
|
||||
sink := llm.NewDiagnosticsWriter(dir)
|
||||
var paths []string
|
||||
for _, index := range []int{2, 3} {
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar_1", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validator}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, ValidationLLMClient: client, DiagnosticsDir: dir, RepairProposalIndex: &index, RepairAttempt: &attempt}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar_1", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validator}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, ValidationLLMClient: client, Interactions: sink, SectionIndex: §ion, RepairProposalIndex: &index, RepairAttempt: &attempt}
|
||||
result, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -116,10 +121,29 @@ func TestRevalidationArtifactsUseProposalAndAttemptCoordinates(t *testing.T) {
|
||||
if paths[0] == paths[1] {
|
||||
t.Fatalf("colliding paths: %v", paths)
|
||||
}
|
||||
for _, path := range paths {
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
for i, path := range paths {
|
||||
base, ok := strings.CutSuffix(filepath.Base(path), "-response-payload.json")
|
||||
if !ok {
|
||||
t.Fatalf("response path: %s", path)
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(filepath.Dir(path), base+"-request-metadata.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var metadata struct {
|
||||
Context structuredoutput.DiagnosticContext `json:"context"`
|
||||
ProposalIndices []int `json:"proposal_indices"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &metadata); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c := metadata.Context
|
||||
if c.Role != "validation" || c.ValidatorName != "editorial_review" ||
|
||||
c.BatchIndex == nil || *c.BatchIndex != 0 || c.SectionIndex == nil || *c.SectionIndex != section ||
|
||||
c.ProposalIndex == nil || *c.ProposalIndex != i+2 || c.RepairAttempt == nil || *c.RepairAttempt != attempt ||
|
||||
len(metadata.ProposalIndices) != 1 || metadata.ProposalIndices[0] != i+2 {
|
||||
t.Fatalf("revalidation metadata: %s", raw)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -4,10 +4,8 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||
@@ -18,7 +16,7 @@ import (
|
||||
|
||||
// RepairAttempt holds optional debug evidence for one semantic request.
|
||||
type RepairAttempt struct {
|
||||
Artifacts llm.InteractionArtifacts
|
||||
Artifacts structuredoutput.InteractionArtifacts
|
||||
Revision *proposals.EnrichedCorrectionProposal
|
||||
Decisions []ValidatorDecisionRecord
|
||||
FailureReason outcomes.ReasonCode
|
||||
@@ -65,7 +63,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
||||
superseded := make(map[int]bool)
|
||||
var branches []RepairOutcome
|
||||
var repairOps []outcomes.Operation
|
||||
debug := input.DiagnosticsDir != ""
|
||||
debug := input.Interactions != nil
|
||||
section := 0
|
||||
if input.SectionIndex != nil {
|
||||
section = *input.SectionIndex
|
||||
@@ -165,13 +163,14 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
||||
if err != nil {
|
||||
failure = map[string]string{"error": err.Error()}
|
||||
}
|
||||
writer := llm.NewDiagnosticsWriter(filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName))
|
||||
artifacts, writeErr := writer.WriteInteraction(name, map[string]any{"stage_index": input.Spec.Index, "stage_id": input.Spec.ID, "proposal_index": proposalIndex, "repair_attempt": attemptNumber, "prompt_id": promptID, "prompt": completion.Prompt, "feedback": findings}, completion.Prepared, map[string]any{"decoded": response, "final_content": completion.FinalContent, "revision": record.Revision, "decisions": record.Decisions, "rejections": validated.rejected}, failure)
|
||||
if writeErr == nil {
|
||||
record.Artifacts = artifacts
|
||||
} else if input.ReportDiagnosticError != nil {
|
||||
input.ReportDiagnosticError(fmt.Errorf("repair interaction %s: %w", name, writeErr))
|
||||
}
|
||||
record.Artifacts, _ = input.Interactions.WriteInteraction(structuredoutput.Interaction{
|
||||
Name: name,
|
||||
Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex, ProposalIndex: &proposalIndex, RepairAttempt: &attemptNumber},
|
||||
RequestMetadata: map[string]any{"stage_index": input.Spec.Index, "stage_id": input.Spec.ID, "proposal_index": proposalIndex, "repair_attempt": attemptNumber, "prompt_id": promptID, "prompt": completion.Prompt, "feedback": findings},
|
||||
RequestPayload: completion.Prepared,
|
||||
ResponsePayload: map[string]any{"decoded": response, "final_content": completion.FinalContent, "revision": record.Revision, "decisions": record.Decisions, "rejections": validated.rejected},
|
||||
ErrorPayload: failure,
|
||||
})
|
||||
branch.Attempts = append(branch.Attempts, record)
|
||||
}
|
||||
if callErr != nil {
|
||||
|
||||
@@ -291,7 +291,8 @@ func TestRepairArtifactsAreUnique(t *testing.T) {
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}},
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "fox"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}},
|
||||
}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar_1", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{}, DiagnosticsDir: t.TempDir()}
|
||||
section := 4
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar_1", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: candidates, SectionIndex: §ion, Exchange: &structuredoutput.Exchange{}, Interactions: llm.NewDiagnosticsWriter(t.TempDir())}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -315,6 +316,21 @@ func TestRepairArtifactsAreUnique(t *testing.T) {
|
||||
if !json.Valid(data) {
|
||||
t.Fatalf("invalid JSON artifact %s: %s", filepath.Base(path), data)
|
||||
}
|
||||
raw, err := os.ReadFile(branches[i].Attempts[0].Artifacts.RequestMetadataPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var metadata struct {
|
||||
Context structuredoutput.DiagnosticContext `json:"context"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &metadata); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c := metadata.Context
|
||||
if c.Role != "correction_repair" || c.ModuleInstance != "grammar_1" || c.SectionIndex == nil || *c.SectionIndex != section ||
|
||||
c.ProposalIndex == nil || *c.ProposalIndex != i || c.RepairAttempt == nil || *c.RepairAttempt != 1 {
|
||||
t.Fatalf("repair metadata: %s", raw)
|
||||
}
|
||||
var request proposal_generation.RepairRequest
|
||||
if err := json.Unmarshal([]byte(client.calls[i].Variables["RepairRequestJSON"]), &request); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -332,16 +348,16 @@ func TestRepairArtifactFailureDoesNotChangeOutcome(t *testing.T) {
|
||||
if err := os.WriteFile(path, []byte("x"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
warnings := 0
|
||||
sink := llm.NewDiagnosticsWriter(path)
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}, DiagnosticsDir: path, ReportDiagnosticError: func(error) { warnings++ }}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}, Interactions: sink}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, branches, err := repairCandidates(context.Background(), input, []int{1}, initial, "repair")
|
||||
if err != nil || len(branches) != 1 || branches[0].Outcome != "withdrawn" || warnings != 1 {
|
||||
t.Fatalf("branches=%+v warnings=%d err=%v", branches, warnings, err)
|
||||
if err != nil || len(branches) != 1 || branches[0].Outcome != "withdrawn" || len(sink.Warnings()) != 1 {
|
||||
t.Fatalf("branches=%+v warnings=%v err=%v", branches, sink.Warnings(), err)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"sync"
|
||||
|
||||
@@ -13,7 +12,6 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/promptcontext"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||
@@ -94,14 +92,11 @@ type ValidatorRejectedChange struct {
|
||||
|
||||
// RunInput is the deterministic runner input.
|
||||
type RunInput struct {
|
||||
// ReportDiagnosticError may be called concurrently; it must not affect processing.
|
||||
ReportDiagnosticError func(error) `json:"-"`
|
||||
Config *config.Config
|
||||
Transcript *schema.Transcript
|
||||
ModuleSpecs []contracts.ModuleRunSpec
|
||||
ProposalDiagnosticsDir string
|
||||
ValidationDiagnosticsDir string
|
||||
CaptureCandidates bool
|
||||
Config *config.Config
|
||||
Transcript *schema.Transcript
|
||||
ModuleSpecs []contracts.ModuleRunSpec
|
||||
Interactions structuredoutput.InteractionSink
|
||||
CaptureCandidates bool
|
||||
}
|
||||
|
||||
// RunOutput is the deterministic runner output.
|
||||
@@ -172,17 +167,15 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
|
||||
}
|
||||
glossaryView := validators.PrepareGlossaryView(spec.Glossary, spec.ProtectedTerms)
|
||||
pipelineResult, pipelineErr := runModulePipeline(ctx, collectSectionProposalsInput{
|
||||
Module: module,
|
||||
Spec: spec,
|
||||
Policy: policy,
|
||||
Sections: sections,
|
||||
ProposalClient: spec.ProposalClient,
|
||||
DiagnosticsDir: input.ProposalDiagnosticsDir,
|
||||
ValidationClient: spec.ValidationClient,
|
||||
ValidationDiagnosticsDir: input.ValidationDiagnosticsDir,
|
||||
GlossaryVariables: &glossaryVariables,
|
||||
GlossaryView: glossaryView,
|
||||
ReportDiagnosticError: input.ReportDiagnosticError,
|
||||
Module: module,
|
||||
Spec: spec,
|
||||
Policy: policy,
|
||||
Sections: sections,
|
||||
ProposalClient: spec.ProposalClient,
|
||||
Interactions: input.Interactions,
|
||||
ValidationClient: spec.ValidationClient,
|
||||
GlossaryVariables: &glossaryVariables,
|
||||
GlossaryView: glossaryView,
|
||||
})
|
||||
if input.CaptureCandidates {
|
||||
candidates = append(candidates, pipelineResult.Candidates...)
|
||||
@@ -335,18 +328,16 @@ func applyWithinSections(sections []chunking.Section, approved []proposals.Enric
|
||||
}
|
||||
|
||||
type collectSectionProposalsInput struct {
|
||||
// ReportDiagnosticError may be called concurrently; it must not affect processing.
|
||||
ReportDiagnosticError func(error) `json:"-"`
|
||||
Module contracts.TranscriptModule
|
||||
Spec contracts.ModuleRunSpec
|
||||
Policy proposals.ReplacementPolicy
|
||||
Sections []chunking.Section
|
||||
ProposalClient contracts.StructuredLLMClient
|
||||
DiagnosticsDir string
|
||||
ValidationClient contracts.StructuredLLMClient
|
||||
ValidationDiagnosticsDir string
|
||||
GlossaryVariables *promptcontext.GlossaryVariables
|
||||
GlossaryView *validators.GlossaryView
|
||||
Interactions structuredoutput.InteractionSink
|
||||
Module contracts.TranscriptModule
|
||||
Spec contracts.ModuleRunSpec
|
||||
Policy proposals.ReplacementPolicy
|
||||
Sections []chunking.Section
|
||||
ProposalClient contracts.StructuredLLMClient
|
||||
|
||||
ValidationClient contracts.StructuredLLMClient
|
||||
GlossaryVariables *promptcontext.GlossaryVariables
|
||||
GlossaryView *validators.GlossaryView
|
||||
}
|
||||
|
||||
type sectionProposalResult struct {
|
||||
@@ -403,14 +394,13 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI
|
||||
proposalResult, err := input.Module.Propose(runCtx, contracts.ProposalRequest{
|
||||
ExecutionContext: contracts.ExecutionContext{
|
||||
StageIndex: &input.Spec.Index, StageID: input.Spec.ID,
|
||||
Settings: input.Spec.Settings,
|
||||
WorkingTranscript: transcriptFromSection(section),
|
||||
Glossary: input.Spec.Glossary,
|
||||
ProtectedTerms: input.Spec.ProtectedTerms,
|
||||
GlossaryVariables: input.GlossaryVariables,
|
||||
Section: &meta,
|
||||
DiagnosticsDir: input.DiagnosticsDir,
|
||||
ReportDiagnosticError: input.ReportDiagnosticError,
|
||||
Settings: input.Spec.Settings,
|
||||
WorkingTranscript: transcriptFromSection(section),
|
||||
Glossary: input.Spec.Glossary,
|
||||
ProtectedTerms: input.Spec.ProtectedTerms,
|
||||
GlossaryVariables: input.GlossaryVariables,
|
||||
Section: &meta,
|
||||
Interactions: input.Interactions,
|
||||
},
|
||||
RunSpec: contracts.ModuleRunSpec{
|
||||
ModuleKey: input.Spec.ModuleKey,
|
||||
@@ -548,17 +538,16 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
||||
go func(sectionPos int, enriched []proposals.EnrichedCorrectionProposal, sectionMetadata contracts.SectionMetadata) {
|
||||
defer vwg.Done()
|
||||
validationInput := validateSectionCandidatesInput{
|
||||
Spec: input.Spec,
|
||||
Policy: input.Policy,
|
||||
WorkingTranscript: transcriptFromSection(sectionResult.section),
|
||||
SectionIndex: §ionMetadata.Index,
|
||||
Validators: validatorsOrdered,
|
||||
SectionEnriched: enriched,
|
||||
Exchange: sectionResult.exchange,
|
||||
ValidationLLMClient: input.ValidationClient,
|
||||
DiagnosticsDir: input.ValidationDiagnosticsDir,
|
||||
GlossaryView: input.GlossaryView,
|
||||
ReportDiagnosticError: input.ReportDiagnosticError,
|
||||
Spec: input.Spec,
|
||||
Policy: input.Policy,
|
||||
WorkingTranscript: transcriptFromSection(sectionResult.section),
|
||||
SectionIndex: §ionMetadata.Index,
|
||||
Validators: validatorsOrdered,
|
||||
SectionEnriched: enriched,
|
||||
Exchange: sectionResult.exchange,
|
||||
ValidationLLMClient: input.ValidationClient,
|
||||
GlossaryView: input.GlossaryView,
|
||||
Interactions: input.Interactions,
|
||||
}
|
||||
if input.Spec.Settings.LLM.CorrectionRepairAttempts > 0 && len(enriched) > 0 {
|
||||
validationInput.Section = validators.NewSectionLookup(validationInput.WorkingTranscript)
|
||||
@@ -656,17 +645,15 @@ type validateSectionCandidatesInput struct {
|
||||
RepairProposalIndex *int
|
||||
RepairAttempt *int
|
||||
Exchange *structuredoutput.Exchange
|
||||
// ReportDiagnosticError may be called concurrently; it must not affect processing.
|
||||
ReportDiagnosticError func(error) `json:"-"`
|
||||
Spec contracts.ModuleRunSpec
|
||||
Policy proposals.ReplacementPolicy
|
||||
WorkingTranscript *schema.Transcript
|
||||
SectionIndex *int
|
||||
Validators []contracts.Validator
|
||||
SectionEnriched []proposals.EnrichedCorrectionProposal
|
||||
ValidationLLMClient contracts.StructuredLLMClient
|
||||
DiagnosticsDir string
|
||||
GlossaryView *validators.GlossaryView
|
||||
Spec contracts.ModuleRunSpec
|
||||
Policy proposals.ReplacementPolicy
|
||||
WorkingTranscript *schema.Transcript
|
||||
SectionIndex *int
|
||||
Validators []contracts.Validator
|
||||
SectionEnriched []proposals.EnrichedCorrectionProposal
|
||||
ValidationLLMClient contracts.StructuredLLMClient
|
||||
Interactions structuredoutput.InteractionSink
|
||||
GlossaryView *validators.GlossaryView
|
||||
}
|
||||
|
||||
type validateSectionCandidatesResult struct {
|
||||
@@ -693,21 +680,11 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
||||
evidence.Section = validators.NewSectionLookup(input.WorkingTranscript)
|
||||
}
|
||||
}
|
||||
var interactionWriter *llmDiagnosticsWriterAdapter
|
||||
|
||||
for validatorOrder, validator := range input.Validators {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return validateSectionCandidatesResult{decisions: decisions, rejected: rejected, operations: operations, filtered: filtered}, err
|
||||
}
|
||||
var diagnosticsWriter validators.InteractionDiagnosticsWriter
|
||||
_, directLLMValidator := validator.(*validators.LLMBackedValidator)
|
||||
if input.DiagnosticsDir != "" && (directLLMValidator || validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked) {
|
||||
if interactionWriter == nil {
|
||||
interactionWriter = &llmDiagnosticsWriterAdapter{dir: filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName)}
|
||||
}
|
||||
diagnosticsWriter = interactionWriter
|
||||
}
|
||||
|
||||
submitted := eligible
|
||||
if feedback {
|
||||
submitted = feedbackCandidates(input, validator, noEffect, evidence)
|
||||
@@ -721,15 +698,14 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
||||
ModuleInstance: input.Spec.InstanceName,
|
||||
SectionIndex: input.SectionIndex,
|
||||
IndependentEvidence: feedback, RepairProposalIndex: input.RepairProposalIndex, RepairAttempt: input.RepairAttempt,
|
||||
ValidatorOrder: &validatorOrder,
|
||||
ReplacementPolicy: input.Policy,
|
||||
Glossary: input.Spec.Glossary,
|
||||
ProtectedTerms: input.Spec.ProtectedTerms,
|
||||
GlossaryView: input.GlossaryView,
|
||||
Settings: input.Spec.Settings,
|
||||
LLMClient: input.ValidationLLMClient,
|
||||
DiagnosticsWriter: diagnosticsWriter,
|
||||
ReportDiagnosticError: input.ReportDiagnosticError,
|
||||
ValidatorOrder: &validatorOrder,
|
||||
ReplacementPolicy: input.Policy,
|
||||
Glossary: input.Spec.Glossary,
|
||||
ProtectedTerms: input.Spec.ProtectedTerms,
|
||||
GlossaryView: input.GlossaryView,
|
||||
Settings: input.Spec.Settings,
|
||||
LLMClient: input.ValidationLLMClient,
|
||||
Interactions: input.Interactions,
|
||||
})
|
||||
operations = append(operations, vResult.Operations...)
|
||||
|
||||
@@ -985,20 +961,3 @@ func cloneTranscript(t *schema.Transcript) *schema.Transcript {
|
||||
}
|
||||
return &schema.Transcript{Segments: segments}
|
||||
}
|
||||
|
||||
type llmDiagnosticsWriterAdapter struct {
|
||||
dir string
|
||||
}
|
||||
|
||||
func (a *llmDiagnosticsWriterAdapter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (validators.InteractionArtifacts, error) {
|
||||
art, err := llm.NewDiagnosticsWriter(a.dir).WriteInteraction(stage, requestMetadata, requestPayload, responsePayload, errorPayload)
|
||||
if err != nil {
|
||||
return validators.InteractionArtifacts{}, err
|
||||
}
|
||||
return validators.InteractionArtifacts{
|
||||
RequestMetadataPath: art.RequestMetadataPath,
|
||||
RequestPayloadPath: art.RequestPayloadPath,
|
||||
ResponsePayloadPath: art.ResponsePayloadPath,
|
||||
ErrorPayloadPath: art.ErrorPayloadPath,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||
@@ -1163,10 +1164,10 @@ func TestRunnerLLMValidatorDiagnosticsWritten(t *testing.T) {
|
||||
}},
|
||||
}})
|
||||
out, err := r.Run(context.Background(), RunInput{
|
||||
Config: &cfg,
|
||||
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}},
|
||||
ValidationDiagnosticsDir: diagDir,
|
||||
Config: &cfg,
|
||||
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}},
|
||||
Interactions: llm.NewDiagnosticsWriter(diagDir),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run error: %v", err)
|
||||
@@ -1217,7 +1218,7 @@ func (m proposalGenerationModule) Propose(ctx context.Context, req contracts.Pro
|
||||
PromptID: "modules.grammar.proposal",
|
||||
Variables: map[string]string{"TranscriptDescriptionBlock": "", "GlossaryJSON": "[]", "SectionJSON": "[]"},
|
||||
LLMClient: req.LLMClient,
|
||||
DiagnosticsDir: req.DiagnosticsDir,
|
||||
Interactions: req.Interactions,
|
||||
})
|
||||
if err != nil {
|
||||
return contracts.ProposalResult{}, err
|
||||
@@ -1263,10 +1264,10 @@ func TestRunnerProposalGenerationHelperFlowsThroughPipeline(t *testing.T) {
|
||||
}})
|
||||
|
||||
out, err := r.Run(context.Background(), RunInput{
|
||||
Config: &cfg,
|
||||
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}},
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ProposalClient: client, Settings: config.Stage{}}},
|
||||
ProposalDiagnosticsDir: diagDir,
|
||||
Config: &cfg,
|
||||
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}},
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ProposalClient: client, Settings: config.Stage{}}},
|
||||
Interactions: llm.NewDiagnosticsWriter(diagDir),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected run error: %v", err)
|
||||
|
||||
24
internal/framework/structuredoutput/diagnostics.go
Normal file
24
internal/framework/structuredoutput/diagnostics.go
Normal file
@@ -0,0 +1,24 @@
|
||||
package structuredoutput
|
||||
|
||||
// InteractionSink captures optional diagnostics. Calls can occur concurrently.
|
||||
// It returns paths for the artifacts that were written even when others fail.
|
||||
// A sink can collect write errors for presentation outside the processing result.
|
||||
type InteractionSink interface {
|
||||
WriteInteraction(Interaction) (InteractionArtifacts, error)
|
||||
}
|
||||
|
||||
type Interaction struct {
|
||||
Name string
|
||||
Context DiagnosticContext
|
||||
RequestMetadata map[string]any
|
||||
RequestPayload any
|
||||
ResponsePayload any
|
||||
ErrorPayload any
|
||||
}
|
||||
|
||||
type InteractionArtifacts struct {
|
||||
RequestMetadataPath string `json:"request_metadata_path,omitempty"`
|
||||
RequestPayloadPath string `json:"request_payload_path,omitempty"`
|
||||
ResponsePayloadPath string `json:"response_payload_path,omitempty"`
|
||||
ErrorPayloadPath string `json:"error_payload_path,omitempty"`
|
||||
}
|
||||
@@ -123,10 +123,11 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
var response LLMValidationResponse
|
||||
var completion StructuredCompletionResponse
|
||||
promptMetadata := validatorPromptMetadata(v.validatorType)
|
||||
coordinates := structuredoutput.DiagnosticContext{StageIndex: req.StageIndex, StageID: req.StageID, Role: "validation", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: req.SectionIndex, BatchIndex: &batch.BatchIndex, ValidatorName: v.name, ProposalIndex: req.RepairProposalIndex, RepairAttempt: req.RepairAttempt}
|
||||
completion, err = req.LLMClient.CompleteStructured(ctx, StructuredCompletionRequest{
|
||||
PromptID: promptMetadata.PromptID,
|
||||
Variables: variables, Schema: responseschema.ValidatorDecisionSetKey,
|
||||
Context: structuredoutput.DiagnosticContext{StageIndex: req.StageIndex, StageID: req.StageID, Role: "validation", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: req.SectionIndex, BatchIndex: &batch.BatchIndex, ValidatorName: v.name, ProposalIndex: req.RepairProposalIndex, RepairAttempt: req.RepairAttempt},
|
||||
Context: coordinates,
|
||||
}, &response)
|
||||
batchIndex := batch.BatchIndex
|
||||
operation := outcomes.Operation{
|
||||
@@ -134,9 +135,9 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
Counts: completion.Execution,
|
||||
}
|
||||
|
||||
writeInteraction := func(reconciled *validationReconciliation, callErr error) InteractionArtifacts {
|
||||
if req.DiagnosticsWriter == nil {
|
||||
return InteractionArtifacts{}
|
||||
writeInteraction := func(reconciled *validationReconciliation, callErr error) structuredoutput.InteractionArtifacts {
|
||||
if req.Interactions == nil {
|
||||
return structuredoutput.InteractionArtifacts{}
|
||||
}
|
||||
stageInstance := req.ModuleInstance
|
||||
if req.SectionIndex != nil {
|
||||
@@ -151,10 +152,8 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
if reconciled != nil {
|
||||
responsePayload["reconciliation"] = *reconciled
|
||||
}
|
||||
var diagnosticErr error
|
||||
artifacts, diagnosticErr := req.DiagnosticsWriter.WriteInteraction(
|
||||
stage,
|
||||
map[string]any{
|
||||
artifacts, _ := req.Interactions.WriteInteraction(structuredoutput.Interaction{
|
||||
Name: stage, Context: coordinates, RequestMetadata: map[string]any{
|
||||
"stage_index": req.StageIndex, "stage_id": req.StageID, "module_key": req.ModuleKey,
|
||||
"validator_name": v.name,
|
||||
"validator_type": v.validatorType,
|
||||
@@ -164,17 +163,14 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
"prompt": completion.Prompt,
|
||||
"response_schema": responseSchema.DiagnosticsMap(),
|
||||
},
|
||||
completion.Prepared,
|
||||
responsePayload,
|
||||
errPayload(callErr),
|
||||
)
|
||||
if diagnosticErr != nil && req.ReportDiagnosticError != nil {
|
||||
req.ReportDiagnosticError(fmt.Errorf("validation interaction %s: %w", stage, diagnosticErr))
|
||||
}
|
||||
RequestPayload: completion.Prepared,
|
||||
ResponsePayload: responsePayload,
|
||||
ErrorPayload: errPayload(callErr),
|
||||
})
|
||||
return artifacts
|
||||
}
|
||||
if err != nil {
|
||||
writeInteraction(nil, err)
|
||||
artifacts := writeInteraction(nil, err)
|
||||
var failure *structuredoutput.Failure
|
||||
if errors.As(err, &failure) {
|
||||
operation.FailureReason = failure.Reason
|
||||
@@ -185,7 +181,11 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
operation.ProposalIndices = batchProposalIndices(batch.Items)
|
||||
operations = append(operations, operation)
|
||||
if structuredoutput.IsRecoverable(err) && failure != nil && failure.Reason != "" {
|
||||
llmDecisions = append(llmDecisions, rejectBatch(batch.Items, string(failure.Reason), "model validation was unavailable")...)
|
||||
decisions := rejectBatch(batch.Items, string(failure.Reason), "model validation was unavailable")
|
||||
for i := range decisions {
|
||||
decisions[i].DiagnosticArtifactPath = artifacts.ResponsePayloadPath
|
||||
}
|
||||
llmDecisions = append(llmDecisions, decisions...)
|
||||
continue
|
||||
}
|
||||
return partialResult(), fmt.Errorf("LLM validator %q completion failed: %w", v.name, err)
|
||||
|
||||
@@ -30,13 +30,10 @@ type captureValidationDiagnosticsWriter struct {
|
||||
lastResponsePayload any
|
||||
}
|
||||
|
||||
func (w *captureValidationDiagnosticsWriter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error) {
|
||||
_ = stage
|
||||
_ = requestPayload
|
||||
_ = errorPayload
|
||||
w.lastRequestMetadata = requestMetadata
|
||||
w.lastResponsePayload = responsePayload
|
||||
return InteractionArtifacts{ResponsePayloadPath: w.path}, w.err
|
||||
func (w *captureValidationDiagnosticsWriter) WriteInteraction(interaction structuredoutput.Interaction) (structuredoutput.InteractionArtifacts, error) {
|
||||
w.lastRequestMetadata = interaction.RequestMetadata
|
||||
w.lastResponsePayload = interaction.ResponsePayload
|
||||
return structuredoutput.InteractionArtifacts{ResponsePayloadPath: w.path}, w.err
|
||||
}
|
||||
|
||||
func (f *fakeStructuredLLMClient) CompleteStructured(ctx context.Context, req StructuredCompletionRequest, out any) (StructuredCompletionResponse, error) {
|
||||
@@ -237,7 +234,7 @@ func TestLLMBackedValidatorReconcilesDecisionIdentities(t *testing.T) {
|
||||
v, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
|
||||
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(7, "gestures", "Jesters"), mk(11, "gestures", "actors"), mk(18, "gestures", "figures")})
|
||||
req.LLMClient = client
|
||||
req.DiagnosticsWriter = writer
|
||||
req.Interactions = writer
|
||||
result, err := v.Validate(context.Background(), req)
|
||||
if err != nil || len(client.calls) != 1 || len(result.Decisions) != len(tc.want) || len(result.Operations) != 1 {
|
||||
t.Fatalf("result=%+v calls=%d err=%v", result, len(client.calls), err)
|
||||
@@ -343,7 +340,7 @@ func TestLLMBackedValidatorDiagnosticsIncludeSchemaMetadata(t *testing.T) {
|
||||
}
|
||||
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
||||
req.LLMClient = client
|
||||
req.DiagnosticsWriter = writer
|
||||
req.Interactions = writer
|
||||
_, err = v.Validate(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("validate error: %v", err)
|
||||
@@ -399,12 +396,11 @@ func TestValidationDiagnosticFailureDoesNotChangeDecision(t *testing.T) {
|
||||
cause := errors.New("diagnostic storage unavailable")
|
||||
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
||||
req.LLMClient = &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}}}
|
||||
req.DiagnosticsWriter = &captureValidationDiagnosticsWriter{err: cause}
|
||||
var reported error
|
||||
req.ReportDiagnosticError = func(err error) { reported = err }
|
||||
sink := &captureValidationDiagnosticsWriter{err: cause}
|
||||
req.Interactions = sink
|
||||
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
|
||||
result, err := validator.Validate(context.Background(), req)
|
||||
if err != nil || !errors.Is(reported, cause) || len(result.Decisions) != 1 || !result.Decisions[0].Approved || result.Decisions[0].DiagnosticArtifactPath != "" || len(result.Operations) != 1 || !result.Operations[0].Succeeded || len(result.Operations[0].RecoveryReasons) != 0 {
|
||||
t.Fatalf("debug failure changed outcome: %+v %v reported=%v", result, err, reported)
|
||||
if err != nil || sink.lastRequestMetadata == nil || len(result.Decisions) != 1 || !result.Decisions[0].Approved || result.Decisions[0].DiagnosticArtifactPath != "" || len(result.Operations) != 1 || !result.Operations[0].Succeeded || len(result.Operations[0].RecoveryReasons) != 0 {
|
||||
t.Fatalf("debug failure changed outcome: %+v %v", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -29,28 +30,26 @@ const (
|
||||
|
||||
// Request is the runtime input shared by deterministic validators.
|
||||
type Request struct {
|
||||
StageIndex *int `json:"stage_index,omitempty"`
|
||||
StageID *string `json:"stage_id,omitempty"`
|
||||
// ReportDiagnosticError may be called concurrently; it must not affect processing.
|
||||
ReportDiagnosticError func(error) `json:"-"`
|
||||
IndependentEvidence bool `json:"-"`
|
||||
RepairProposalIndex *int `json:"repair_proposal_index,omitempty"`
|
||||
RepairAttempt *int `json:"repair_attempt,omitempty"`
|
||||
SectionIndex *int `json:"section_index,omitempty"`
|
||||
WorkingTranscript *schema.Transcript `json:"-"`
|
||||
Section *SectionLookup `json:"-"`
|
||||
Previews map[int]proposals.SegmentPreviewResult `json:"-"`
|
||||
CandidateProposal []proposals.EnrichedCorrectionProposal `json:"candidate_proposals"`
|
||||
ModuleKey string `json:"module_key"`
|
||||
ModuleInstance string `json:"module_instance"`
|
||||
ValidatorOrder *int `json:"validator_order,omitempty"`
|
||||
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
|
||||
Glossary []glossary.Entry `json:"-"`
|
||||
ProtectedTerms []glossary.Term `json:"-"`
|
||||
GlossaryView *GlossaryView `json:"-"`
|
||||
Settings config.Stage `json:"-"`
|
||||
LLMClient StructuredLLMClient `json:"-"`
|
||||
DiagnosticsWriter InteractionDiagnosticsWriter `json:"-"`
|
||||
StageIndex *int `json:"stage_index,omitempty"`
|
||||
StageID *string `json:"stage_id,omitempty"`
|
||||
IndependentEvidence bool `json:"-"`
|
||||
RepairProposalIndex *int `json:"repair_proposal_index,omitempty"`
|
||||
RepairAttempt *int `json:"repair_attempt,omitempty"`
|
||||
SectionIndex *int `json:"section_index,omitempty"`
|
||||
WorkingTranscript *schema.Transcript `json:"-"`
|
||||
Section *SectionLookup `json:"-"`
|
||||
Previews map[int]proposals.SegmentPreviewResult `json:"-"`
|
||||
CandidateProposal []proposals.EnrichedCorrectionProposal `json:"candidate_proposals"`
|
||||
ModuleKey string `json:"module_key"`
|
||||
ModuleInstance string `json:"module_instance"`
|
||||
ValidatorOrder *int `json:"validator_order,omitempty"`
|
||||
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
|
||||
Glossary []glossary.Entry `json:"-"`
|
||||
ProtectedTerms []glossary.Term `json:"-"`
|
||||
GlossaryView *GlossaryView `json:"-"`
|
||||
Settings config.Stage `json:"-"`
|
||||
LLMClient StructuredLLMClient `json:"-"`
|
||||
Interactions structuredoutput.InteractionSink `json:"-"`
|
||||
}
|
||||
|
||||
// Decision is one validator decision for one proposal index.
|
||||
@@ -81,17 +80,6 @@ type Result struct {
|
||||
Operations []outcomes.Operation `json:"operations,omitempty"`
|
||||
}
|
||||
|
||||
type InteractionDiagnosticsWriter interface {
|
||||
WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error)
|
||||
}
|
||||
|
||||
type InteractionArtifacts struct {
|
||||
RequestMetadataPath string `json:"request_metadata_path,omitempty"`
|
||||
RequestPayloadPath string `json:"request_payload_path,omitempty"`
|
||||
ResponsePayloadPath string `json:"response_payload_path,omitempty"`
|
||||
ErrorPayloadPath string `json:"error_payload_path,omitempty"`
|
||||
}
|
||||
|
||||
// Validator is the deterministic runtime validator interface.
|
||||
type Validator interface {
|
||||
Name() string
|
||||
|
||||
Reference in New Issue
Block a user