From cea7e442321f9b4a787cdcb0f78978f24c1a39f0 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Mon, 16 Nov 2026 03:43:29 +0000 Subject: [PATCH] Unify optional interaction capture and warnings --- docs/internal/diagnostics-reporting.md | 17 +- docs/roadmap/implementation.md | 6 + internal/cli/process_pipeline.go | 13 +- internal/framework/contracts/contracts.go | 20 +-- internal/framework/llm/diagnostics.go | 132 ++++++++++----- internal/framework/llm/diagnostics_test.go | 60 ++++++- .../framework/proposal_generation/generate.go | 97 +++-------- .../proposal_generation/generate_test.go | 44 +++-- .../proposal_generation/module_proposal.go | 25 ++- internal/framework/runner/feedback_test.go | 32 +++- internal/framework/runner/repair.go | 21 ++- internal/framework/runner/repair_test.go | 26 ++- internal/framework/runner/runner.go | 157 +++++++----------- internal/framework/runner/runner_test.go | 19 ++- .../framework/structuredoutput/diagnostics.go | 24 +++ .../framework/validators/llm_validators.go | 34 ++-- .../validators/llm_validators_test.go | 24 ++- internal/framework/validators/models.go | 54 +++--- internal/modules/glossary/module_test.go | 3 +- internal/modules/grammar/module_test.go | 3 +- internal/modules/homophones/module_test.go | 3 +- internal/modules/spoken_word/module_test.go | 3 +- 22 files changed, 450 insertions(+), 367 deletions(-) create mode 100644 internal/framework/structuredoutput/diagnostics.go diff --git a/docs/internal/diagnostics-reporting.md b/docs/internal/diagnostics-reporting.md index 674dcfd..33d135b 100644 --- a/docs/internal/diagnostics-reporting.md +++ b/docs/internal/diagnostics-reporting.md @@ -32,16 +32,23 @@ On a final write error it attempts other enabled reports and one failed-status pass; a published transcript is never erased by a later report failure. Debug is independent. The shared `--debug` flag enables a unique retained child -under `diagnostics.work_dir`. CLI passes empty diagnostics paths and nil writers -when debug is off. With debug enabled, it saves the source bytes as acquired, separate from the parsed and normalized +under `diagnostics.work_dir`. The CLI creates one interaction sink rooted in +that directory and passes it to proposal generation, validation, and repair. +With debug enabled, it saves the source bytes as acquired, separate from the parsed and normalized transcript snapshots. It also captures invocation metadata, effective settings rendered by ConfigKit, the correction ledger, LLM execution observations, and model interactions. Candidate capture includes proposals later rejected or interrupted. Repair and revalidation artifacts use section, proposal, -attempt, validator, and batch coordinates as applicable. Ledger mapping preserves empty proposed and applied replacements and distinguishes unavailable candidates from interrupted work. +attempt, validator, and batch coordinates as applicable. Request metadata records +these coordinates with the interaction role. Each interaction retains its available +prepared request, response, and failure evidence. Ledger mapping preserves empty +proposed and applied replacements and distinguishes unavailable candidates from +interrupted work. -A concurrency-safe diagnostic callback reports optional write failures without -aborting processing. Debug artifacts can contain sensitive transcript and +The sink groups concurrent write failures by role, artifact operation, and cause. +Warnings report the count and an affected path. The writer still attempts other +artifacts and returns the paths it wrote, and the CLI presents warnings separately +from processing outcomes and required delivery. Debug artifacts can contain sensitive transcript and prompt content. Execution position and optional operator ID are carried separately from generated safe artifact path coordinates. diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 8dadd92..710a723 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -141,6 +141,12 @@ Verify role, chunk, batch, proposal, and attempt associations with decoded artifacts. Exercise a failed optional write followed by later capture. Update the [diagnostics internals](../internal/diagnostics-reporting.md) for the sink. +**Complete.** The CLI supplies one optional sink for proposal, validation, +repair, and revalidation artifacts. The writer records successful paths even +after a partial write and groups failures by role, artifact operation, and +cause. Decoded-artifact and recovery checks passed, along with the full Go +suite and targeted race checks. + ## Stage 4 — Bind publication to checked destinations Audit #1, #6 (cooperative boundaries), #7, #8, #16, #26, #42. diff --git a/internal/cli/process_pipeline.go b/internal/cli/process_pipeline.go index 6908e59..0b7271d 100644 --- a/internal/cli/process_pipeline.go +++ b/internal/cli/process_pipeline.go @@ -47,13 +47,20 @@ func executePipeline(ctx context.Context, loaded *processInputs, prepared *prepa item.CaptureProposalExchange = true } } - debugPath := prepared.DebugPath() + var interactions structuredoutput.InteractionSink + var writer *llm.DiagnosticsWriter + if debug != nil { + writer = llm.NewDiagnosticsWriter(debug.Path()) + interactions = writer + } output, runErr := runner.New(modules.NewFactory()).Run(ctx, runner.RunInput{ Config: cfg, Transcript: normalized, ModuleSpecs: specs, - ProposalDiagnosticsDir: debugPath, ValidationDiagnosticsDir: debugPath, - CaptureCandidates: debugPath != "", ReportDiagnosticError: warn, + Interactions: interactions, CaptureCandidates: debug != nil, }) if debug != nil { + for _, warning := range writer.Warnings() { + warn(warning) + } warn(debug.WriteJSONArtifact(diagnostics.ArtifactLLMExecution, processreport.BuildLLMExecution(observer.Snapshot()))) warn(debug.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{RunDirectoryPath: debug.Path(), RunOutput: &output}))) } diff --git a/internal/framework/contracts/contracts.go b/internal/framework/contracts/contracts.go index 694d894..c6c6e44 100644 --- a/internal/framework/contracts/contracts.go +++ b/internal/framework/contracts/contracts.go @@ -73,17 +73,15 @@ func SectionMetadataFromSection(section chunking.Section) SectionMetadata { // ExecutionContext carries shared execution state for proposal generation and validation. type ExecutionContext struct { - StageIndex *int `json:"stage_index,omitempty"` - StageID *string `json:"stage_id,omitempty"` - // ReportDiagnosticError may be called concurrently; it must not affect processing. - ReportDiagnosticError func(error) `json:"-"` - Settings config.Stage `json:"-"` - WorkingTranscript *schema.Transcript `json:"-"` - Glossary []glossary.Entry `json:"-"` - ProtectedTerms []glossary.Term `json:"-"` - GlossaryVariables *promptcontext.GlossaryVariables `json:"-"` - Section *SectionMetadata `json:"section,omitempty"` - DiagnosticsDir string `json:"diagnostics_dir,omitempty"` + StageIndex *int `json:"stage_index,omitempty"` + StageID *string `json:"stage_id,omitempty"` + Settings config.Stage `json:"-"` + WorkingTranscript *schema.Transcript `json:"-"` + Glossary []glossary.Entry `json:"-"` + ProtectedTerms []glossary.Term `json:"-"` + GlossaryVariables *promptcontext.GlossaryVariables `json:"-"` + Section *SectionMetadata `json:"section,omitempty"` + Interactions structuredoutput.InteractionSink `json:"-"` } // ProposalRequest is the input to module proposal generation. diff --git a/internal/framework/llm/diagnostics.go b/internal/framework/llm/diagnostics.go index 4e1c2a7..9c49d00 100644 --- a/internal/framework/llm/diagnostics.go +++ b/internal/framework/llm/diagnostics.go @@ -2,68 +2,120 @@ package llm import ( "encoding/json" + "errors" "fmt" + "maps" "os" "path/filepath" "regexp" "strings" + "sync" + + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput" ) var stageSanitizer = regexp.MustCompile(`[^a-zA-Z0-9_.-]+`) -// InteractionArtifacts contains file paths for written prompt/response -// diagnostics. -type InteractionArtifacts struct { - RequestMetadataPath string - RequestPayloadPath string - ResponsePayloadPath string - ErrorPayloadPath string -} - -// DiagnosticsWriter writes generic machine-readable LLM interaction artifacts. +// DiagnosticsWriter writes interaction artifacts under a single run directory. type DiagnosticsWriter struct { - dir string + dir string + mu sync.Mutex + failures map[interactionFailureKey]*interactionFailure + order []interactionFailureKey +} + +type interactionFailureKey struct { + role, action, cause string +} + +type interactionFailure struct { + count int + path string } -// NewDiagnosticsWriter creates a diagnostics writer rooted at dir. func NewDiagnosticsWriter(dir string) *DiagnosticsWriter { return &DiagnosticsWriter{dir: dir} } -// WriteInteraction writes request/response/error artifacts for a generic LLM -// interaction stage. -func (w *DiagnosticsWriter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error) { +func (w *DiagnosticsWriter) WriteInteraction(interaction structuredoutput.Interaction) (structuredoutput.InteractionArtifacts, error) { + var paths structuredoutput.InteractionArtifacts + dir := w.dir + if instance := interaction.Context.ModuleInstance; instance != "" { + dir = filepath.Join(dir, sanitizeStageName(instance)) + } + var err error if strings.TrimSpace(w.dir) == "" { - return InteractionArtifacts{}, fmt.Errorf("diagnostics directory must not be empty") + err = fmt.Errorf("diagnostics directory must not be empty") + } else { + err = os.MkdirAll(dir, 0o755) } - if err := os.MkdirAll(w.dir, 0o755); err != nil { - return InteractionArtifacts{}, fmt.Errorf("create diagnostics directory: %w", err) + base := sanitizeStageName(interaction.Name) + if err != nil { + w.recordFailure(interaction.Context.Role, "create directory", filepath.Join(dir, base), err) + return paths, fmt.Errorf("create diagnostics directory %q: %w", dir, err) } - base := sanitizeStageName(stage) - paths := InteractionArtifacts{ - RequestMetadataPath: filepath.Join(w.dir, base+"-request-metadata.json"), - RequestPayloadPath: filepath.Join(w.dir, base+"-request-payload.json"), - ResponsePayloadPath: filepath.Join(w.dir, base+"-response-payload.json"), + metadata := maps.Clone(interaction.RequestMetadata) + if metadata == nil { + metadata = make(map[string]any) } - - if err := w.writeJSON(paths.RequestMetadataPath, requestMetadata); err != nil { - return InteractionArtifacts{}, err + metadata["context"] = interaction.Context + type artifactFile struct { + suffix string + payload any + path *string } - if err := w.writeJSON(paths.RequestPayloadPath, requestPayload); err != nil { - return InteractionArtifacts{}, err + artifacts := []artifactFile{ + {"request-metadata", metadata, &paths.RequestMetadataPath}, + {"request-payload", interaction.RequestPayload, &paths.RequestPayloadPath}, + {"response-payload", interaction.ResponsePayload, &paths.ResponsePayloadPath}, } - if err := w.writeJSON(paths.ResponsePayloadPath, responsePayload); err != nil { - return InteractionArtifacts{}, err + if interaction.ErrorPayload != nil { + artifacts = append(artifacts, artifactFile{"error-payload", interaction.ErrorPayload, &paths.ErrorPayloadPath}) } - if errorPayload != nil { - paths.ErrorPayloadPath = filepath.Join(w.dir, base+"-error-payload.json") - if err := w.writeJSON(paths.ErrorPayloadPath, errorPayload); err != nil { - return InteractionArtifacts{}, err + var failures []error + for _, artifact := range artifacts { + path := filepath.Join(dir, base+"-"+artifact.suffix+".json") + if action, err := writeInteractionJSON(path, artifact.payload); err != nil { + w.recordFailure(interaction.Context.Role, action+" "+artifact.suffix, path, err) + failures = append(failures, fmt.Errorf("%s %q: %w", action, path, err)) + continue } + *artifact.path = path } + return paths, errors.Join(failures...) +} - return paths, nil +func (w *DiagnosticsWriter) recordFailure(role, action, path string, err error) { + var pathErr *os.PathError + if errors.As(err, &pathErr) { + err = pathErr.Err + } + key := interactionFailureKey{role: role, action: action, cause: err.Error()} + w.mu.Lock() + defer w.mu.Unlock() + if w.failures == nil { + w.failures = make(map[interactionFailureKey]*interactionFailure) + } + group := w.failures[key] + if group == nil { + group = &interactionFailure{path: path} + w.failures[key] = group + w.order = append(w.order, key) + } + group.count++ +} + +// Warnings summarizes capture failures after the callers join. +func (w *DiagnosticsWriter) Warnings() []error { + w.mu.Lock() + defer w.mu.Unlock() + warnings := make([]error, 0, len(w.order)) + for _, key := range w.order { + group := w.failures[key] + warnings = append(warnings, fmt.Errorf("%s interaction: %s failures (%d), first at %q: %s", key.role, key.action, group.count, group.path, key.cause)) + } + return warnings } func sanitizeStageName(stage string) string { @@ -79,14 +131,10 @@ func sanitizeStageName(stage string) string { return safe } -func (w *DiagnosticsWriter) writeJSON(path string, payload any) error { +func writeInteractionJSON(path string, payload any) (string, error) { encoded, err := json.Marshal(payload) if err != nil { - return fmt.Errorf("marshal json artifact %q: %w", filepath.Base(path), err) + return "marshal", err } - encoded = append(encoded, '\n') - if err := os.WriteFile(path, encoded, 0o644); err != nil { - return fmt.Errorf("write json artifact %q: %w", filepath.Base(path), err) - } - return nil + return "write", os.WriteFile(path, append(encoded, '\n'), 0o644) } diff --git a/internal/framework/llm/diagnostics_test.go b/internal/framework/llm/diagnostics_test.go index aaa1041..9aaf1b1 100644 --- a/internal/framework/llm/diagnostics_test.go +++ b/internal/framework/llm/diagnostics_test.go @@ -4,20 +4,24 @@ import ( "encoding/json" "os" "path/filepath" + "strings" + "sync" "testing" + + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput" ) func TestDiagnosticsWriterProducesValidJSONArtifacts(t *testing.T) { dir := t.TempDir() writer := NewDiagnosticsWriter(dir) - artifacts, err := writer.WriteInteraction( - "validation:batch-1", - map[string]any{"stage": "validation", "attempt": 1}, - map[string]any{"messages": []map[string]any{{"role": "user", "content": "hello"}}}, - map[string]any{"result": "ok"}, - nil, - ) + artifacts, err := writer.WriteInteraction(structuredoutput.Interaction{ + Name: "validation:batch-1", + Context: structuredoutput.DiagnosticContext{Role: "validation", ModuleInstance: "grammar_1"}, + RequestMetadata: map[string]any{"attempt": 1}, + RequestPayload: map[string]any{"messages": []map[string]any{{"role": "user", "content": "hello"}}}, + ResponsePayload: map[string]any{"result": "ok"}, + }) if err != nil { t.Fatalf("WriteInteraction: %v", err) } @@ -30,6 +34,48 @@ func TestDiagnosticsWriterProducesValidJSONArtifacts(t *testing.T) { assertValidJSONFile(t, artifacts.ResponsePayloadPath) } +func TestDiagnosticsWriterGroupsFailuresAndContinuesCapture(t *testing.T) { + dir := t.TempDir() + writer := NewDiagnosticsWriter(dir) + var workers sync.WaitGroup + for _, name := range []string{"first", "second"} { + blocked := filepath.Join(dir, "grammar_1", name+"-response-payload.json") + if err := os.MkdirAll(blocked, 0o755); err != nil { + t.Fatal(err) + } + workers.Go(func() { + paths, err := writer.WriteInteraction(structuredoutput.Interaction{ + Name: name, Context: structuredoutput.DiagnosticContext{Role: "validation", ModuleInstance: "grammar_1"}, + ErrorPayload: map[string]string{"error": "model unavailable"}, + }) + if err == nil || paths.RequestPayloadPath == "" || paths.ResponsePayloadPath != "" || paths.ErrorPayloadPath == "" { + t.Errorf("paths=%+v error=%v", paths, err) + } + }) + } + workers.Wait() + warnings := writer.Warnings() + if len(warnings) != 1 || !strings.Contains(warnings[0].Error(), "validation interaction: write response-payload failures (2)") || + !strings.Contains(warnings[0].Error(), filepath.Join(dir, "grammar_1")) { + t.Fatalf("warnings: %v", warnings) + } + paths, err := writer.WriteInteraction(structuredoutput.Interaction{ + Name: "later", Context: structuredoutput.DiagnosticContext{Role: "validation", ModuleInstance: "grammar_1"}, + ResponsePayload: map[string]string{"final_content": "{"}, + }) + if err != nil { + t.Fatal(err) + } + raw, err := os.ReadFile(paths.ResponsePayloadPath) + if err != nil { + t.Fatal(err) + } + var decoded map[string]string + if err := json.Unmarshal(raw, &decoded); err != nil || decoded["final_content"] != "{" { + t.Fatalf("later artifact: %s (%v)", raw, err) + } +} + func assertValidJSONFile(t *testing.T, path string) { t.Helper() raw, err := os.ReadFile(path) diff --git a/internal/framework/proposal_generation/generate.go b/internal/framework/proposal_generation/generate.go index 6d97523..96b5ae6 100644 --- a/internal/framework/proposal_generation/generate.go +++ b/internal/framework/proposal_generation/generate.go @@ -7,12 +7,10 @@ import ( "context" "errors" "fmt" - "path/filepath" "strings" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" - "gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema" @@ -20,19 +18,6 @@ import ( "gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput" ) -// InteractionDiagnosticsWriter writes machine-readable prompt/response artifacts. -type InteractionDiagnosticsWriter interface { - WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error) -} - -// InteractionArtifacts contains written diagnostics paths. -type InteractionArtifacts struct { - RequestMetadataPath string `json:"request_metadata_path,omitempty"` - RequestPayloadPath string `json:"request_payload_path,omitempty"` - ResponsePayloadPath string `json:"response_payload_path,omitempty"` - ErrorPayloadPath string `json:"error_payload_path,omitempty"` -} - // StructuredCorrectionProposal is one LLM response correction payload. type StructuredCorrectionProposal struct { SegmentReference int `json:"id"` @@ -48,23 +33,20 @@ type StructuredCorrectionSet struct { // Request captures reusable proposal-generation inputs for future modules. type Request struct { - StageIndex *int `json:"stage_index,omitempty"` - StageID *string `json:"stage_id,omitempty"` - // ReportDiagnosticError may be called concurrently; it must not affect processing. - ReportDiagnosticError func(error) `json:"-"` - ModuleKey string `json:"module_key"` - ModuleInstance string `json:"module_instance"` - ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"` - WorkingTranscript *schema.Transcript `json:"-"` - Section *contracts.SectionMetadata `json:"section,omitempty"` - PromptID string `json:"prompt_id"` - Variables map[string]string `json:"variables"` - StageName string `json:"stage_name,omitempty"` - StartIndex int `json:"start_index"` - CaptureExchange bool - LLMClient contracts.StructuredLLMClient - DiagnosticsDir string - DiagnosticsWriter InteractionDiagnosticsWriter + StageIndex *int `json:"stage_index,omitempty"` + StageID *string `json:"stage_id,omitempty"` + ModuleKey string `json:"module_key"` + ModuleInstance string `json:"module_instance"` + ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"` + WorkingTranscript *schema.Transcript `json:"-"` + Section *contracts.SectionMetadata `json:"section,omitempty"` + PromptID string `json:"prompt_id"` + Variables map[string]string `json:"variables"` + StageName string `json:"stage_name,omitempty"` + StartIndex int `json:"start_index"` + CaptureExchange bool + LLMClient contracts.StructuredLLMClient + Interactions structuredoutput.InteractionSink } // Result contains generated candidate proposals and optional diagnostics paths. @@ -74,7 +56,7 @@ type Result struct { Corrections []proposals.CorrectionProposal `json:"corrections"` Enriched []proposals.EnrichedCorrectionProposal `json:"enriched"` Operations []outcomes.Operation `json:"operations,omitempty"` - Artifacts InteractionArtifacts `json:"artifacts,omitempty"` + Artifacts structuredoutput.InteractionArtifacts `json:"artifacts,omitempty"` } // GenerateCandidates executes one structured LLM call and deterministically maps @@ -105,13 +87,14 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) { response StructuredCorrectionSet completion contracts.StructuredCompletionResponse callErr error - artifacts InteractionArtifacts + artifacts structuredoutput.InteractionArtifacts ) + coordinates := structuredoutput.DiagnosticContext{StageIndex: req.StageIndex, StageID: req.StageID, Role: "proposal", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: sectionIndexPtr(req.Section)} completion, callErr = req.LLMClient.CompleteStructured(ctx, contracts.StructuredCompletionRequest{ CaptureExchange: req.CaptureExchange, PromptID: req.PromptID, Variables: req.Variables, Schema: responseschema.CorrectionSetKey, - Context: structuredoutput.DiagnosticContext{StageIndex: req.StageIndex, StageID: req.StageID, Role: "proposal", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: sectionIndexPtr(req.Section)}, + Context: coordinates, }, &response) sectionIndex := 0 if req.Section != nil { @@ -126,16 +109,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) { Counts: completion.Execution, } - var writer InteractionDiagnosticsWriter - if req.DiagnosticsWriter != nil { - writer = req.DiagnosticsWriter - } else if strings.TrimSpace(req.DiagnosticsDir) != "" { - writer = diagnosticsWriterAdapter{ - writer: llm.NewDiagnosticsWriter(filepath.Join(req.DiagnosticsDir, req.ModuleInstance)), - } - } - - if writer != nil { + if req.Interactions != nil { requestMetadata := map[string]any{ "stage_index": req.StageIndex, "stage_id": req.StageID, "module_key": req.ModuleKey, @@ -147,17 +121,11 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) { "prompt": completion.Prompt, } requestMetadata["response_schema"] = responseschema.MustLookup(responseschema.CorrectionSetKey).DiagnosticsMap() - var diagnosticErr error - artifacts, diagnosticErr = writer.WriteInteraction( - stage, - requestMetadata, - completion.Prepared, - map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model}, - errPayload(callErr), - ) - if diagnosticErr != nil && req.ReportDiagnosticError != nil { - req.ReportDiagnosticError(fmt.Errorf("proposal interaction %s: %w", stage, diagnosticErr)) - } + artifacts, _ = req.Interactions.WriteInteraction(structuredoutput.Interaction{ + Name: stage, Context: coordinates, RequestMetadata: requestMetadata, RequestPayload: completion.Prepared, + ResponsePayload: map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model}, + ErrorPayload: errPayload(callErr), + }) } if callErr != nil { @@ -234,23 +202,6 @@ func sectionIndexPtr(section *contracts.SectionMetadata) *int { return &index } -type diagnosticsWriterAdapter struct { - writer *llm.DiagnosticsWriter -} - -func (a diagnosticsWriterAdapter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error) { - artifacts, err := a.writer.WriteInteraction(stage, requestMetadata, requestPayload, responsePayload, errorPayload) - if err != nil { - return InteractionArtifacts{}, err - } - return InteractionArtifacts{ - RequestMetadataPath: artifacts.RequestMetadataPath, - RequestPayloadPath: artifacts.RequestPayloadPath, - ResponsePayloadPath: artifacts.ResponsePayloadPath, - ErrorPayloadPath: artifacts.ErrorPayloadPath, - }, nil -} - // segmentIDs records the request-local reference mapping in debug metadata. func segmentIDs(transcript *schema.Transcript) []int { if transcript == nil { diff --git a/internal/framework/proposal_generation/generate_test.go b/internal/framework/proposal_generation/generate_test.go index 434dda5..2613c2d 100644 --- a/internal/framework/proposal_generation/generate_test.go +++ b/internal/framework/proposal_generation/generate_test.go @@ -12,6 +12,7 @@ import ( "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema" @@ -56,13 +57,11 @@ type captureDiagnosticsWriter struct { lastRequestPayload any } -func (w *captureDiagnosticsWriter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error) { - w.lastStage = stage - w.lastRequestMetadata = requestMetadata - w.lastRequestPayload = requestPayload - _ = responsePayload - _ = errorPayload - return InteractionArtifacts{}, w.err +func (w *captureDiagnosticsWriter) WriteInteraction(interaction structuredoutput.Interaction) (structuredoutput.InteractionArtifacts, error) { + w.lastStage = interaction.Name + w.lastRequestMetadata = interaction.RequestMetadata + w.lastRequestPayload = interaction.RequestPayload + return structuredoutput.InteractionArtifacts{}, w.err } func defaultRequest(t *testing.T) Request { @@ -146,7 +145,7 @@ func TestGenerateCandidatesDiagnosticsIncludeSchemaMetadata(t *testing.T) { diag := &captureDiagnosticsWriter{} req := defaultRequest(t) req.LLMClient = client - req.DiagnosticsWriter = diag + req.Interactions = diag _, err := GenerateCandidates(context.Background(), req) if err != nil { @@ -316,8 +315,10 @@ func TestGenerateCandidatesDiagnosticsWritten(t *testing.T) { } req := defaultRequest(t) req.LLMClient = client - req.DiagnosticsDir = t.TempDir() + dir := t.TempDir() + req.Interactions = llm.NewDiagnosticsWriter(dir) req.Variables = map[string]string{"SectionJSON": "section text"} + req.Section = &contracts.SectionMetadata{Index: 3} got, err := GenerateCandidates(context.Background(), req) if err != nil { @@ -353,13 +354,23 @@ func TestGenerateCandidatesDiagnosticsWritten(t *testing.T) { if schemaMap["id"] != want.ID || schemaMap["version"] != want.Version || schemaMap["name"] != want.Name || schemaMap["sha256"] != want.SHA256 { t.Fatalf("unexpected schema metadata in diagnostics: got=%v want=%+v", schemaMap, want) } + var coordinates struct { + Context structuredoutput.DiagnosticContext `json:"context"` + } + if err := json.Unmarshal(raw, &coordinates); err != nil { + t.Fatal(err) + } + if c := coordinates.Context; c.Role != "proposal" || c.ModuleInstance != req.ModuleInstance || c.SectionIndex == nil || *c.SectionIndex != 3 { + t.Fatalf("proposal metadata: %s", raw) + } } func TestGenerateCandidatesClientError(t *testing.T) { client := &fakeStructuredClient{err: errors.New("boom")} req := defaultRequest(t) req.LLMClient = client - req.DiagnosticsDir = t.TempDir() + dir := t.TempDir() + req.Interactions = llm.NewDiagnosticsWriter(dir) got, err := GenerateCandidates(context.Background(), req) if err == nil || !strings.Contains(err.Error(), "completion failed") { @@ -368,12 +379,12 @@ func TestGenerateCandidatesClientError(t *testing.T) { if len(got.Operations) != 1 || got.Operations[0].FailureReason != outcomes.ReasonInternalFailure || len(got.Corrections) != 0 { t.Fatalf("expected fatal call facts without proposals, got %+v", got) } - matches, globErr := filepath.Glob(filepath.Join(req.DiagnosticsDir, req.ModuleInstance, "*error-payload.json")) + matches, globErr := filepath.Glob(filepath.Join(dir, req.ModuleInstance, "*error-payload.json")) if globErr != nil { t.Fatalf("glob error: %v", globErr) } if len(matches) == 0 { - t.Fatalf("expected error diagnostics artifact under %s", req.DiagnosticsDir) + t.Fatalf("expected error diagnostics artifact under %s", dir) } } @@ -381,11 +392,10 @@ func TestProposalDiagnosticFailureDoesNotChangeOperation(t *testing.T) { cause := errors.New("diagnostic storage unavailable") req := defaultRequest(t) req.LLMClient = &fakeStructuredClient{responses: []StructuredCorrectionSet{{}}} - req.DiagnosticsWriter = &captureDiagnosticsWriter{err: cause} - var reported error - req.ReportDiagnosticError = func(err error) { reported = err } + sink := &captureDiagnosticsWriter{err: cause} + req.Interactions = sink result, err := GenerateCandidates(context.Background(), req) - if err != nil || !errors.Is(reported, cause) || len(result.Operations) != 1 || !result.Operations[0].Succeeded || len(result.Operations[0].RecoveryReasons) != 0 || result.Artifacts.ResponsePayloadPath != "" { - t.Fatalf("debug failure changed outcome: %+v %v reported=%v", result, err, reported) + if err != nil || sink.lastStage == "" || len(result.Operations) != 1 || !result.Operations[0].Succeeded || len(result.Operations[0].RecoveryReasons) != 0 || result.Artifacts.ResponsePayloadPath != "" { + t.Fatalf("debug failure changed outcome: %+v %v", result, err) } } diff --git a/internal/framework/proposal_generation/module_proposal.go b/internal/framework/proposal_generation/module_proposal.go index baa89aa..517164a 100644 --- a/internal/framework/proposal_generation/module_proposal.go +++ b/internal/framework/proposal_generation/module_proposal.go @@ -54,19 +54,18 @@ func ExecuteModuleProposal(ctx context.Context, req ModuleProposalRequest) (cont generated, err := GenerateCandidates(ctx, Request{ StageIndex: req.ProposalRequest.StageIndex, StageID: req.ProposalRequest.StageID, - ModuleKey: req.ProposalRequest.RunSpec.ModuleKey, - ModuleInstance: req.ProposalRequest.RunSpec.InstanceName, - ReplacementPolicy: req.ProposalRequest.RunSpec.ReplacementPolicy, - WorkingTranscript: req.ProposalRequest.WorkingTranscript, - Section: req.ProposalRequest.Section, - PromptID: promptMetadata.PromptID, - Variables: variables, - StageName: stagename.ModuleProposal(req.ProposalRequest.RunSpec.InstanceName, sectionIndexPtr(req.ProposalRequest.Section)), - StartIndex: 0, - CaptureExchange: req.ProposalRequest.RunSpec.CaptureProposalExchange, - LLMClient: req.ProposalRequest.LLMClient, - DiagnosticsDir: req.ProposalRequest.DiagnosticsDir, - ReportDiagnosticError: req.ProposalRequest.ReportDiagnosticError, + ModuleKey: req.ProposalRequest.RunSpec.ModuleKey, + ModuleInstance: req.ProposalRequest.RunSpec.InstanceName, + ReplacementPolicy: req.ProposalRequest.RunSpec.ReplacementPolicy, + WorkingTranscript: req.ProposalRequest.WorkingTranscript, + Section: req.ProposalRequest.Section, + PromptID: promptMetadata.PromptID, + Variables: variables, + StageName: stagename.ModuleProposal(req.ProposalRequest.RunSpec.InstanceName, sectionIndexPtr(req.ProposalRequest.Section)), + StartIndex: 0, + CaptureExchange: req.ProposalRequest.RunSpec.CaptureProposalExchange, + LLMClient: req.ProposalRequest.LLMClient, + Interactions: req.ProposalRequest.Interactions, }) if err != nil { return contracts.ProposalResult{Operations: generated.Operations}, err diff --git a/internal/framework/runner/feedback_test.go b/internal/framework/runner/feedback_test.go index 679cb50..087195a 100644 --- a/internal/framework/runner/feedback_test.go +++ b/internal/framework/runner/feedback_test.go @@ -2,8 +2,10 @@ package runner import ( "context" + "encoding/json" "fmt" "os" + "path/filepath" "strings" "testing" @@ -11,7 +13,9 @@ import ( "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators" validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata" ) @@ -99,11 +103,12 @@ func TestRevalidationArtifactsUseProposalAndAttemptCoordinates(t *testing.T) { t.Fatal(err) } dir := t.TempDir() - attempt := 1 + attempt, section := 1, 4 + sink := llm.NewDiagnosticsWriter(dir) var paths []string for _, index := range []int{2, 3} { candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index}} - input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar_1", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validator}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, ValidationLLMClient: client, DiagnosticsDir: dir, RepairProposalIndex: &index, RepairAttempt: &attempt} + input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar_1", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validator}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, ValidationLLMClient: client, Interactions: sink, SectionIndex: §ion, RepairProposalIndex: &index, RepairAttempt: &attempt} result, err := validateSectionCandidates(context.Background(), input) if err != nil { t.Fatal(err) @@ -116,10 +121,29 @@ func TestRevalidationArtifactsUseProposalAndAttemptCoordinates(t *testing.T) { if paths[0] == paths[1] { t.Fatalf("colliding paths: %v", paths) } - for _, path := range paths { - if _, err := os.Stat(path); err != nil { + for i, path := range paths { + base, ok := strings.CutSuffix(filepath.Base(path), "-response-payload.json") + if !ok { + t.Fatalf("response path: %s", path) + } + raw, err := os.ReadFile(filepath.Join(filepath.Dir(path), base+"-request-metadata.json")) + if err != nil { t.Fatal(err) } + var metadata struct { + Context structuredoutput.DiagnosticContext `json:"context"` + ProposalIndices []int `json:"proposal_indices"` + } + if err := json.Unmarshal(raw, &metadata); err != nil { + t.Fatal(err) + } + c := metadata.Context + if c.Role != "validation" || c.ValidatorName != "editorial_review" || + c.BatchIndex == nil || *c.BatchIndex != 0 || c.SectionIndex == nil || *c.SectionIndex != section || + c.ProposalIndex == nil || *c.ProposalIndex != i+2 || c.RepairAttempt == nil || *c.RepairAttempt != attempt || + len(metadata.ProposalIndices) != 1 || metadata.ProposalIndices[0] != i+2 { + t.Fatalf("revalidation metadata: %s", raw) + } } } diff --git a/internal/framework/runner/repair.go b/internal/framework/runner/repair.go index ff1d44f..21561e8 100644 --- a/internal/framework/runner/repair.go +++ b/internal/framework/runner/repair.go @@ -4,10 +4,8 @@ import ( "context" "errors" "fmt" - "path/filepath" "slices" - "gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" @@ -18,7 +16,7 @@ import ( // RepairAttempt holds optional debug evidence for one semantic request. type RepairAttempt struct { - Artifacts llm.InteractionArtifacts + Artifacts structuredoutput.InteractionArtifacts Revision *proposals.EnrichedCorrectionProposal Decisions []ValidatorDecisionRecord FailureReason outcomes.ReasonCode @@ -65,7 +63,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput, superseded := make(map[int]bool) var branches []RepairOutcome var repairOps []outcomes.Operation - debug := input.DiagnosticsDir != "" + debug := input.Interactions != nil section := 0 if input.SectionIndex != nil { section = *input.SectionIndex @@ -165,13 +163,14 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput, if err != nil { failure = map[string]string{"error": err.Error()} } - writer := llm.NewDiagnosticsWriter(filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName)) - artifacts, writeErr := writer.WriteInteraction(name, map[string]any{"stage_index": input.Spec.Index, "stage_id": input.Spec.ID, "proposal_index": proposalIndex, "repair_attempt": attemptNumber, "prompt_id": promptID, "prompt": completion.Prompt, "feedback": findings}, completion.Prepared, map[string]any{"decoded": response, "final_content": completion.FinalContent, "revision": record.Revision, "decisions": record.Decisions, "rejections": validated.rejected}, failure) - if writeErr == nil { - record.Artifacts = artifacts - } else if input.ReportDiagnosticError != nil { - input.ReportDiagnosticError(fmt.Errorf("repair interaction %s: %w", name, writeErr)) - } + record.Artifacts, _ = input.Interactions.WriteInteraction(structuredoutput.Interaction{ + Name: name, + Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex, ProposalIndex: &proposalIndex, RepairAttempt: &attemptNumber}, + RequestMetadata: map[string]any{"stage_index": input.Spec.Index, "stage_id": input.Spec.ID, "proposal_index": proposalIndex, "repair_attempt": attemptNumber, "prompt_id": promptID, "prompt": completion.Prompt, "feedback": findings}, + RequestPayload: completion.Prepared, + ResponsePayload: map[string]any{"decoded": response, "final_content": completion.FinalContent, "revision": record.Revision, "decisions": record.Decisions, "rejections": validated.rejected}, + ErrorPayload: failure, + }) branch.Attempts = append(branch.Attempts, record) } if callErr != nil { diff --git a/internal/framework/runner/repair_test.go b/internal/framework/runner/repair_test.go index adda090..7136da0 100644 --- a/internal/framework/runner/repair_test.go +++ b/internal/framework/runner/repair_test.go @@ -291,7 +291,8 @@ func TestRepairArtifactsAreUnique(t *testing.T) { {CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}, {CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "fox"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}}, } - input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar_1", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{}, DiagnosticsDir: t.TempDir()} + section := 4 + input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar_1", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: candidates, SectionIndex: §ion, Exchange: &structuredoutput.Exchange{}, Interactions: llm.NewDiagnosticsWriter(t.TempDir())} initial, err := validateSectionCandidates(context.Background(), input) if err != nil { t.Fatal(err) @@ -315,6 +316,21 @@ func TestRepairArtifactsAreUnique(t *testing.T) { if !json.Valid(data) { t.Fatalf("invalid JSON artifact %s: %s", filepath.Base(path), data) } + raw, err := os.ReadFile(branches[i].Attempts[0].Artifacts.RequestMetadataPath) + if err != nil { + t.Fatal(err) + } + var metadata struct { + Context structuredoutput.DiagnosticContext `json:"context"` + } + if err := json.Unmarshal(raw, &metadata); err != nil { + t.Fatal(err) + } + c := metadata.Context + if c.Role != "correction_repair" || c.ModuleInstance != "grammar_1" || c.SectionIndex == nil || *c.SectionIndex != section || + c.ProposalIndex == nil || *c.ProposalIndex != i || c.RepairAttempt == nil || *c.RepairAttempt != 1 { + t.Fatalf("repair metadata: %s", raw) + } var request proposal_generation.RepairRequest if err := json.Unmarshal([]byte(client.calls[i].Variables["RepairRequestJSON"]), &request); err != nil { t.Fatal(err) @@ -332,16 +348,16 @@ func TestRepairArtifactFailureDoesNotChangeOutcome(t *testing.T) { if err := os.WriteFile(path, []byte("x"), 0600); err != nil { t.Fatal(err) } - warnings := 0 + sink := llm.NewDiagnosticsWriter(path) candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}} - input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}, DiagnosticsDir: path, ReportDiagnosticError: func(error) { warnings++ }} + input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}, Interactions: sink} initial, err := validateSectionCandidates(context.Background(), input) if err != nil { t.Fatal(err) } _, branches, err := repairCandidates(context.Background(), input, []int{1}, initial, "repair") - if err != nil || len(branches) != 1 || branches[0].Outcome != "withdrawn" || warnings != 1 { - t.Fatalf("branches=%+v warnings=%d err=%v", branches, warnings, err) + if err != nil || len(branches) != 1 || branches[0].Outcome != "withdrawn" || len(sink.Warnings()) != 1 { + t.Fatalf("branches=%+v warnings=%v err=%v", branches, sink.Warnings(), err) } } diff --git a/internal/framework/runner/runner.go b/internal/framework/runner/runner.go index 5f47fd5..07def1e 100644 --- a/internal/framework/runner/runner.go +++ b/internal/framework/runner/runner.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "path/filepath" "sort" "sync" @@ -13,7 +12,6 @@ import ( "gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" - "gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/promptcontext" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" @@ -94,14 +92,11 @@ type ValidatorRejectedChange struct { // RunInput is the deterministic runner input. type RunInput struct { - // ReportDiagnosticError may be called concurrently; it must not affect processing. - ReportDiagnosticError func(error) `json:"-"` - Config *config.Config - Transcript *schema.Transcript - ModuleSpecs []contracts.ModuleRunSpec - ProposalDiagnosticsDir string - ValidationDiagnosticsDir string - CaptureCandidates bool + Config *config.Config + Transcript *schema.Transcript + ModuleSpecs []contracts.ModuleRunSpec + Interactions structuredoutput.InteractionSink + CaptureCandidates bool } // RunOutput is the deterministic runner output. @@ -172,17 +167,15 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) { } glossaryView := validators.PrepareGlossaryView(spec.Glossary, spec.ProtectedTerms) pipelineResult, pipelineErr := runModulePipeline(ctx, collectSectionProposalsInput{ - Module: module, - Spec: spec, - Policy: policy, - Sections: sections, - ProposalClient: spec.ProposalClient, - DiagnosticsDir: input.ProposalDiagnosticsDir, - ValidationClient: spec.ValidationClient, - ValidationDiagnosticsDir: input.ValidationDiagnosticsDir, - GlossaryVariables: &glossaryVariables, - GlossaryView: glossaryView, - ReportDiagnosticError: input.ReportDiagnosticError, + Module: module, + Spec: spec, + Policy: policy, + Sections: sections, + ProposalClient: spec.ProposalClient, + Interactions: input.Interactions, + ValidationClient: spec.ValidationClient, + GlossaryVariables: &glossaryVariables, + GlossaryView: glossaryView, }) if input.CaptureCandidates { candidates = append(candidates, pipelineResult.Candidates...) @@ -335,18 +328,16 @@ func applyWithinSections(sections []chunking.Section, approved []proposals.Enric } type collectSectionProposalsInput struct { - // ReportDiagnosticError may be called concurrently; it must not affect processing. - ReportDiagnosticError func(error) `json:"-"` - Module contracts.TranscriptModule - Spec contracts.ModuleRunSpec - Policy proposals.ReplacementPolicy - Sections []chunking.Section - ProposalClient contracts.StructuredLLMClient - DiagnosticsDir string - ValidationClient contracts.StructuredLLMClient - ValidationDiagnosticsDir string - GlossaryVariables *promptcontext.GlossaryVariables - GlossaryView *validators.GlossaryView + Interactions structuredoutput.InteractionSink + Module contracts.TranscriptModule + Spec contracts.ModuleRunSpec + Policy proposals.ReplacementPolicy + Sections []chunking.Section + ProposalClient contracts.StructuredLLMClient + + ValidationClient contracts.StructuredLLMClient + GlossaryVariables *promptcontext.GlossaryVariables + GlossaryView *validators.GlossaryView } type sectionProposalResult struct { @@ -403,14 +394,13 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI proposalResult, err := input.Module.Propose(runCtx, contracts.ProposalRequest{ ExecutionContext: contracts.ExecutionContext{ StageIndex: &input.Spec.Index, StageID: input.Spec.ID, - Settings: input.Spec.Settings, - WorkingTranscript: transcriptFromSection(section), - Glossary: input.Spec.Glossary, - ProtectedTerms: input.Spec.ProtectedTerms, - GlossaryVariables: input.GlossaryVariables, - Section: &meta, - DiagnosticsDir: input.DiagnosticsDir, - ReportDiagnosticError: input.ReportDiagnosticError, + Settings: input.Spec.Settings, + WorkingTranscript: transcriptFromSection(section), + Glossary: input.Spec.Glossary, + ProtectedTerms: input.Spec.ProtectedTerms, + GlossaryVariables: input.GlossaryVariables, + Section: &meta, + Interactions: input.Interactions, }, RunSpec: contracts.ModuleRunSpec{ ModuleKey: input.Spec.ModuleKey, @@ -548,17 +538,16 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) go func(sectionPos int, enriched []proposals.EnrichedCorrectionProposal, sectionMetadata contracts.SectionMetadata) { defer vwg.Done() validationInput := validateSectionCandidatesInput{ - Spec: input.Spec, - Policy: input.Policy, - WorkingTranscript: transcriptFromSection(sectionResult.section), - SectionIndex: §ionMetadata.Index, - Validators: validatorsOrdered, - SectionEnriched: enriched, - Exchange: sectionResult.exchange, - ValidationLLMClient: input.ValidationClient, - DiagnosticsDir: input.ValidationDiagnosticsDir, - GlossaryView: input.GlossaryView, - ReportDiagnosticError: input.ReportDiagnosticError, + Spec: input.Spec, + Policy: input.Policy, + WorkingTranscript: transcriptFromSection(sectionResult.section), + SectionIndex: §ionMetadata.Index, + Validators: validatorsOrdered, + SectionEnriched: enriched, + Exchange: sectionResult.exchange, + ValidationLLMClient: input.ValidationClient, + GlossaryView: input.GlossaryView, + Interactions: input.Interactions, } if input.Spec.Settings.LLM.CorrectionRepairAttempts > 0 && len(enriched) > 0 { validationInput.Section = validators.NewSectionLookup(validationInput.WorkingTranscript) @@ -656,17 +645,15 @@ type validateSectionCandidatesInput struct { RepairProposalIndex *int RepairAttempt *int Exchange *structuredoutput.Exchange - // ReportDiagnosticError may be called concurrently; it must not affect processing. - ReportDiagnosticError func(error) `json:"-"` - Spec contracts.ModuleRunSpec - Policy proposals.ReplacementPolicy - WorkingTranscript *schema.Transcript - SectionIndex *int - Validators []contracts.Validator - SectionEnriched []proposals.EnrichedCorrectionProposal - ValidationLLMClient contracts.StructuredLLMClient - DiagnosticsDir string - GlossaryView *validators.GlossaryView + Spec contracts.ModuleRunSpec + Policy proposals.ReplacementPolicy + WorkingTranscript *schema.Transcript + SectionIndex *int + Validators []contracts.Validator + SectionEnriched []proposals.EnrichedCorrectionProposal + ValidationLLMClient contracts.StructuredLLMClient + Interactions structuredoutput.InteractionSink + GlossaryView *validators.GlossaryView } type validateSectionCandidatesResult struct { @@ -693,21 +680,11 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida evidence.Section = validators.NewSectionLookup(input.WorkingTranscript) } } - var interactionWriter *llmDiagnosticsWriterAdapter for validatorOrder, validator := range input.Validators { if err := ctx.Err(); err != nil { return validateSectionCandidatesResult{decisions: decisions, rejected: rejected, operations: operations, filtered: filtered}, err } - var diagnosticsWriter validators.InteractionDiagnosticsWriter - _, directLLMValidator := validator.(*validators.LLMBackedValidator) - if input.DiagnosticsDir != "" && (directLLMValidator || validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked) { - if interactionWriter == nil { - interactionWriter = &llmDiagnosticsWriterAdapter{dir: filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName)} - } - diagnosticsWriter = interactionWriter - } - submitted := eligible if feedback { submitted = feedbackCandidates(input, validator, noEffect, evidence) @@ -721,15 +698,14 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex, IndependentEvidence: feedback, RepairProposalIndex: input.RepairProposalIndex, RepairAttempt: input.RepairAttempt, - ValidatorOrder: &validatorOrder, - ReplacementPolicy: input.Policy, - Glossary: input.Spec.Glossary, - ProtectedTerms: input.Spec.ProtectedTerms, - GlossaryView: input.GlossaryView, - Settings: input.Spec.Settings, - LLMClient: input.ValidationLLMClient, - DiagnosticsWriter: diagnosticsWriter, - ReportDiagnosticError: input.ReportDiagnosticError, + ValidatorOrder: &validatorOrder, + ReplacementPolicy: input.Policy, + Glossary: input.Spec.Glossary, + ProtectedTerms: input.Spec.ProtectedTerms, + GlossaryView: input.GlossaryView, + Settings: input.Spec.Settings, + LLMClient: input.ValidationLLMClient, + Interactions: input.Interactions, }) operations = append(operations, vResult.Operations...) @@ -985,20 +961,3 @@ func cloneTranscript(t *schema.Transcript) *schema.Transcript { } return &schema.Transcript{Segments: segments} } - -type llmDiagnosticsWriterAdapter struct { - dir string -} - -func (a *llmDiagnosticsWriterAdapter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (validators.InteractionArtifacts, error) { - art, err := llm.NewDiagnosticsWriter(a.dir).WriteInteraction(stage, requestMetadata, requestPayload, responsePayload, errorPayload) - if err != nil { - return validators.InteractionArtifacts{}, err - } - return validators.InteractionArtifacts{ - RequestMetadataPath: art.RequestMetadataPath, - RequestPayloadPath: art.RequestPayloadPath, - ResponsePayloadPath: art.ResponsePayloadPath, - ErrorPayloadPath: art.ErrorPayloadPath, - }, nil -} diff --git a/internal/framework/runner/runner_test.go b/internal/framework/runner/runner_test.go index 7a4c1c8..90a7592 100644 --- a/internal/framework/runner/runner_test.go +++ b/internal/framework/runner/runner_test.go @@ -16,6 +16,7 @@ import ( "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" @@ -1163,10 +1164,10 @@ func TestRunnerLLMValidatorDiagnosticsWritten(t *testing.T) { }}, }}) out, err := r.Run(context.Background(), RunInput{ - Config: &cfg, - Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}}, - ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}}, - ValidationDiagnosticsDir: diagDir, + Config: &cfg, + Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}}, + ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}}, + Interactions: llm.NewDiagnosticsWriter(diagDir), }) if err != nil { t.Fatalf("Run error: %v", err) @@ -1217,7 +1218,7 @@ func (m proposalGenerationModule) Propose(ctx context.Context, req contracts.Pro PromptID: "modules.grammar.proposal", Variables: map[string]string{"TranscriptDescriptionBlock": "", "GlossaryJSON": "[]", "SectionJSON": "[]"}, LLMClient: req.LLMClient, - DiagnosticsDir: req.DiagnosticsDir, + Interactions: req.Interactions, }) if err != nil { return contracts.ProposalResult{}, err @@ -1263,10 +1264,10 @@ func TestRunnerProposalGenerationHelperFlowsThroughPipeline(t *testing.T) { }}) out, err := r.Run(context.Background(), RunInput{ - Config: &cfg, - Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}}, - ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ProposalClient: client, Settings: config.Stage{}}}, - ProposalDiagnosticsDir: diagDir, + Config: &cfg, + Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}}, + ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ProposalClient: client, Settings: config.Stage{}}}, + Interactions: llm.NewDiagnosticsWriter(diagDir), }) if err != nil { t.Fatalf("unexpected run error: %v", err) diff --git a/internal/framework/structuredoutput/diagnostics.go b/internal/framework/structuredoutput/diagnostics.go new file mode 100644 index 0000000..0b0bf1e --- /dev/null +++ b/internal/framework/structuredoutput/diagnostics.go @@ -0,0 +1,24 @@ +package structuredoutput + +// InteractionSink captures optional diagnostics. Calls can occur concurrently. +// It returns paths for the artifacts that were written even when others fail. +// A sink can collect write errors for presentation outside the processing result. +type InteractionSink interface { + WriteInteraction(Interaction) (InteractionArtifacts, error) +} + +type Interaction struct { + Name string + Context DiagnosticContext + RequestMetadata map[string]any + RequestPayload any + ResponsePayload any + ErrorPayload any +} + +type InteractionArtifacts struct { + RequestMetadataPath string `json:"request_metadata_path,omitempty"` + RequestPayloadPath string `json:"request_payload_path,omitempty"` + ResponsePayloadPath string `json:"response_payload_path,omitempty"` + ErrorPayloadPath string `json:"error_payload_path,omitempty"` +} diff --git a/internal/framework/validators/llm_validators.go b/internal/framework/validators/llm_validators.go index 68a1afe..56a9c82 100644 --- a/internal/framework/validators/llm_validators.go +++ b/internal/framework/validators/llm_validators.go @@ -123,10 +123,11 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, var response LLMValidationResponse var completion StructuredCompletionResponse promptMetadata := validatorPromptMetadata(v.validatorType) + coordinates := structuredoutput.DiagnosticContext{StageIndex: req.StageIndex, StageID: req.StageID, Role: "validation", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: req.SectionIndex, BatchIndex: &batch.BatchIndex, ValidatorName: v.name, ProposalIndex: req.RepairProposalIndex, RepairAttempt: req.RepairAttempt} completion, err = req.LLMClient.CompleteStructured(ctx, StructuredCompletionRequest{ PromptID: promptMetadata.PromptID, Variables: variables, Schema: responseschema.ValidatorDecisionSetKey, - Context: structuredoutput.DiagnosticContext{StageIndex: req.StageIndex, StageID: req.StageID, Role: "validation", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: req.SectionIndex, BatchIndex: &batch.BatchIndex, ValidatorName: v.name, ProposalIndex: req.RepairProposalIndex, RepairAttempt: req.RepairAttempt}, + Context: coordinates, }, &response) batchIndex := batch.BatchIndex operation := outcomes.Operation{ @@ -134,9 +135,9 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, Counts: completion.Execution, } - writeInteraction := func(reconciled *validationReconciliation, callErr error) InteractionArtifacts { - if req.DiagnosticsWriter == nil { - return InteractionArtifacts{} + writeInteraction := func(reconciled *validationReconciliation, callErr error) structuredoutput.InteractionArtifacts { + if req.Interactions == nil { + return structuredoutput.InteractionArtifacts{} } stageInstance := req.ModuleInstance if req.SectionIndex != nil { @@ -151,10 +152,8 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, if reconciled != nil { responsePayload["reconciliation"] = *reconciled } - var diagnosticErr error - artifacts, diagnosticErr := req.DiagnosticsWriter.WriteInteraction( - stage, - map[string]any{ + artifacts, _ := req.Interactions.WriteInteraction(structuredoutput.Interaction{ + Name: stage, Context: coordinates, RequestMetadata: map[string]any{ "stage_index": req.StageIndex, "stage_id": req.StageID, "module_key": req.ModuleKey, "validator_name": v.name, "validator_type": v.validatorType, @@ -164,17 +163,14 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, "prompt": completion.Prompt, "response_schema": responseSchema.DiagnosticsMap(), }, - completion.Prepared, - responsePayload, - errPayload(callErr), - ) - if diagnosticErr != nil && req.ReportDiagnosticError != nil { - req.ReportDiagnosticError(fmt.Errorf("validation interaction %s: %w", stage, diagnosticErr)) - } + RequestPayload: completion.Prepared, + ResponsePayload: responsePayload, + ErrorPayload: errPayload(callErr), + }) return artifacts } if err != nil { - writeInteraction(nil, err) + artifacts := writeInteraction(nil, err) var failure *structuredoutput.Failure if errors.As(err, &failure) { operation.FailureReason = failure.Reason @@ -185,7 +181,11 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, operation.ProposalIndices = batchProposalIndices(batch.Items) operations = append(operations, operation) if structuredoutput.IsRecoverable(err) && failure != nil && failure.Reason != "" { - llmDecisions = append(llmDecisions, rejectBatch(batch.Items, string(failure.Reason), "model validation was unavailable")...) + decisions := rejectBatch(batch.Items, string(failure.Reason), "model validation was unavailable") + for i := range decisions { + decisions[i].DiagnosticArtifactPath = artifacts.ResponsePayloadPath + } + llmDecisions = append(llmDecisions, decisions...) continue } return partialResult(), fmt.Errorf("LLM validator %q completion failed: %w", v.name, err) diff --git a/internal/framework/validators/llm_validators_test.go b/internal/framework/validators/llm_validators_test.go index f24c9ac..a98b6ec 100644 --- a/internal/framework/validators/llm_validators_test.go +++ b/internal/framework/validators/llm_validators_test.go @@ -30,13 +30,10 @@ type captureValidationDiagnosticsWriter struct { lastResponsePayload any } -func (w *captureValidationDiagnosticsWriter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error) { - _ = stage - _ = requestPayload - _ = errorPayload - w.lastRequestMetadata = requestMetadata - w.lastResponsePayload = responsePayload - return InteractionArtifacts{ResponsePayloadPath: w.path}, w.err +func (w *captureValidationDiagnosticsWriter) WriteInteraction(interaction structuredoutput.Interaction) (structuredoutput.InteractionArtifacts, error) { + w.lastRequestMetadata = interaction.RequestMetadata + w.lastResponsePayload = interaction.ResponsePayload + return structuredoutput.InteractionArtifacts{ResponsePayloadPath: w.path}, w.err } func (f *fakeStructuredLLMClient) CompleteStructured(ctx context.Context, req StructuredCompletionRequest, out any) (StructuredCompletionResponse, error) { @@ -237,7 +234,7 @@ func TestLLMBackedValidatorReconcilesDecisionIdentities(t *testing.T) { v, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview) req := makeReq([]proposals.EnrichedCorrectionProposal{mk(7, "gestures", "Jesters"), mk(11, "gestures", "actors"), mk(18, "gestures", "figures")}) req.LLMClient = client - req.DiagnosticsWriter = writer + req.Interactions = writer result, err := v.Validate(context.Background(), req) if err != nil || len(client.calls) != 1 || len(result.Decisions) != len(tc.want) || len(result.Operations) != 1 { t.Fatalf("result=%+v calls=%d err=%v", result, len(client.calls), err) @@ -343,7 +340,7 @@ func TestLLMBackedValidatorDiagnosticsIncludeSchemaMetadata(t *testing.T) { } req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")}) req.LLMClient = client - req.DiagnosticsWriter = writer + req.Interactions = writer _, err = v.Validate(context.Background(), req) if err != nil { t.Fatalf("validate error: %v", err) @@ -399,12 +396,11 @@ func TestValidationDiagnosticFailureDoesNotChangeDecision(t *testing.T) { cause := errors.New("diagnostic storage unavailable") req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")}) req.LLMClient = &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}}} - req.DiagnosticsWriter = &captureValidationDiagnosticsWriter{err: cause} - var reported error - req.ReportDiagnosticError = func(err error) { reported = err } + sink := &captureValidationDiagnosticsWriter{err: cause} + req.Interactions = sink validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview) result, err := validator.Validate(context.Background(), req) - if err != nil || !errors.Is(reported, cause) || len(result.Decisions) != 1 || !result.Decisions[0].Approved || result.Decisions[0].DiagnosticArtifactPath != "" || len(result.Operations) != 1 || !result.Operations[0].Succeeded || len(result.Operations[0].RecoveryReasons) != 0 { - t.Fatalf("debug failure changed outcome: %+v %v reported=%v", result, err, reported) + if err != nil || sink.lastRequestMetadata == nil || len(result.Decisions) != 1 || !result.Decisions[0].Approved || result.Decisions[0].DiagnosticArtifactPath != "" || len(result.Operations) != 1 || !result.Operations[0].Succeeded || len(result.Operations[0].RecoveryReasons) != 0 { + t.Fatalf("debug failure changed outcome: %+v %v", result, err) } } diff --git a/internal/framework/validators/models.go b/internal/framework/validators/models.go index f31653e..0612346 100644 --- a/internal/framework/validators/models.go +++ b/internal/framework/validators/models.go @@ -10,6 +10,7 @@ import ( "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput" ) const ( @@ -29,28 +30,26 @@ const ( // Request is the runtime input shared by deterministic validators. type Request struct { - StageIndex *int `json:"stage_index,omitempty"` - StageID *string `json:"stage_id,omitempty"` - // ReportDiagnosticError may be called concurrently; it must not affect processing. - ReportDiagnosticError func(error) `json:"-"` - IndependentEvidence bool `json:"-"` - RepairProposalIndex *int `json:"repair_proposal_index,omitempty"` - RepairAttempt *int `json:"repair_attempt,omitempty"` - SectionIndex *int `json:"section_index,omitempty"` - WorkingTranscript *schema.Transcript `json:"-"` - Section *SectionLookup `json:"-"` - Previews map[int]proposals.SegmentPreviewResult `json:"-"` - CandidateProposal []proposals.EnrichedCorrectionProposal `json:"candidate_proposals"` - ModuleKey string `json:"module_key"` - ModuleInstance string `json:"module_instance"` - ValidatorOrder *int `json:"validator_order,omitempty"` - ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"` - Glossary []glossary.Entry `json:"-"` - ProtectedTerms []glossary.Term `json:"-"` - GlossaryView *GlossaryView `json:"-"` - Settings config.Stage `json:"-"` - LLMClient StructuredLLMClient `json:"-"` - DiagnosticsWriter InteractionDiagnosticsWriter `json:"-"` + StageIndex *int `json:"stage_index,omitempty"` + StageID *string `json:"stage_id,omitempty"` + IndependentEvidence bool `json:"-"` + RepairProposalIndex *int `json:"repair_proposal_index,omitempty"` + RepairAttempt *int `json:"repair_attempt,omitempty"` + SectionIndex *int `json:"section_index,omitempty"` + WorkingTranscript *schema.Transcript `json:"-"` + Section *SectionLookup `json:"-"` + Previews map[int]proposals.SegmentPreviewResult `json:"-"` + CandidateProposal []proposals.EnrichedCorrectionProposal `json:"candidate_proposals"` + ModuleKey string `json:"module_key"` + ModuleInstance string `json:"module_instance"` + ValidatorOrder *int `json:"validator_order,omitempty"` + ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"` + Glossary []glossary.Entry `json:"-"` + ProtectedTerms []glossary.Term `json:"-"` + GlossaryView *GlossaryView `json:"-"` + Settings config.Stage `json:"-"` + LLMClient StructuredLLMClient `json:"-"` + Interactions structuredoutput.InteractionSink `json:"-"` } // Decision is one validator decision for one proposal index. @@ -81,17 +80,6 @@ type Result struct { Operations []outcomes.Operation `json:"operations,omitempty"` } -type InteractionDiagnosticsWriter interface { - WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error) -} - -type InteractionArtifacts struct { - RequestMetadataPath string `json:"request_metadata_path,omitempty"` - RequestPayloadPath string `json:"request_payload_path,omitempty"` - ResponsePayloadPath string `json:"response_payload_path,omitempty"` - ErrorPayloadPath string `json:"error_payload_path,omitempty"` -} - // Validator is the deterministic runtime validator interface. type Validator interface { Name() string diff --git a/internal/modules/glossary/module_test.go b/internal/modules/glossary/module_test.go index 3d509be..f1cf193 100644 --- a/internal/modules/glossary/module_test.go +++ b/internal/modules/glossary/module_test.go @@ -11,6 +11,7 @@ import ( domain "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/v3/internal/prompts" @@ -105,7 +106,7 @@ func TestGlossaryModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T) ExecutionContext: contracts.ExecutionContext{ WorkingTranscript: tinyTranscript(), Glossary: tinyGlossary(), - DiagnosticsDir: diagDir, + Interactions: llm.NewDiagnosticsWriter(diagDir), }, RunSpec: contracts.ModuleRunSpec{ ModuleKey: "glossary", diff --git a/internal/modules/grammar/module_test.go b/internal/modules/grammar/module_test.go index 9fcc65c..4f6f7be 100644 --- a/internal/modules/grammar/module_test.go +++ b/internal/modules/grammar/module_test.go @@ -11,6 +11,7 @@ import ( domain "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/v3/internal/prompts" @@ -114,7 +115,7 @@ func TestGrammarModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T) { ExecutionContext: contracts.ExecutionContext{ WorkingTranscript: tinyTranscript(), Glossary: tinyGlossary(), - DiagnosticsDir: diagDir, + Interactions: llm.NewDiagnosticsWriter(diagDir), }, RunSpec: contracts.ModuleRunSpec{ ModuleKey: "grammar", diff --git a/internal/modules/homophones/module_test.go b/internal/modules/homophones/module_test.go index 3a48701..8f74edc 100644 --- a/internal/modules/homophones/module_test.go +++ b/internal/modules/homophones/module_test.go @@ -11,6 +11,7 @@ import ( domain "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators" @@ -106,7 +107,7 @@ func TestHomophonesModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T ExecutionContext: contracts.ExecutionContext{ WorkingTranscript: tinyTranscript(), Glossary: tinyGlossary(), - DiagnosticsDir: diagDir, + Interactions: llm.NewDiagnosticsWriter(diagDir), }, RunSpec: contracts.ModuleRunSpec{ ModuleKey: "homophones", diff --git a/internal/modules/spoken_word/module_test.go b/internal/modules/spoken_word/module_test.go index f8d7c84..c2f0a32 100644 --- a/internal/modules/spoken_word/module_test.go +++ b/internal/modules/spoken_word/module_test.go @@ -11,6 +11,7 @@ import ( domain "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators" @@ -106,7 +107,7 @@ func TestSpokenWordModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T ExecutionContext: contracts.ExecutionContext{ WorkingTranscript: tinyTranscript(), Glossary: tinyGlossary(), - DiagnosticsDir: diagDir, + Interactions: llm.NewDiagnosticsWriter(diagDir), }, RunSpec: contracts.ModuleRunSpec{ ModuleKey: "spoken_word",