Unify optional interaction capture and warnings

This commit is contained in:
2026-11-16 03:43:29 +00:00
parent ffe0e3491e
commit cea7e44232
22 changed files with 450 additions and 367 deletions

View File

@@ -4,7 +4,6 @@ import (
"context"
"errors"
"fmt"
"path/filepath"
"sort"
"sync"
@@ -13,7 +12,6 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/promptcontext"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
@@ -94,14 +92,11 @@ type ValidatorRejectedChange struct {
// RunInput is the deterministic runner input.
type RunInput struct {
// ReportDiagnosticError may be called concurrently; it must not affect processing.
ReportDiagnosticError func(error) `json:"-"`
Config *config.Config
Transcript *schema.Transcript
ModuleSpecs []contracts.ModuleRunSpec
ProposalDiagnosticsDir string
ValidationDiagnosticsDir string
CaptureCandidates bool
Config *config.Config
Transcript *schema.Transcript
ModuleSpecs []contracts.ModuleRunSpec
Interactions structuredoutput.InteractionSink
CaptureCandidates bool
}
// RunOutput is the deterministic runner output.
@@ -172,17 +167,15 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
}
glossaryView := validators.PrepareGlossaryView(spec.Glossary, spec.ProtectedTerms)
pipelineResult, pipelineErr := runModulePipeline(ctx, collectSectionProposalsInput{
Module: module,
Spec: spec,
Policy: policy,
Sections: sections,
ProposalClient: spec.ProposalClient,
DiagnosticsDir: input.ProposalDiagnosticsDir,
ValidationClient: spec.ValidationClient,
ValidationDiagnosticsDir: input.ValidationDiagnosticsDir,
GlossaryVariables: &glossaryVariables,
GlossaryView: glossaryView,
ReportDiagnosticError: input.ReportDiagnosticError,
Module: module,
Spec: spec,
Policy: policy,
Sections: sections,
ProposalClient: spec.ProposalClient,
Interactions: input.Interactions,
ValidationClient: spec.ValidationClient,
GlossaryVariables: &glossaryVariables,
GlossaryView: glossaryView,
})
if input.CaptureCandidates {
candidates = append(candidates, pipelineResult.Candidates...)
@@ -335,18 +328,16 @@ func applyWithinSections(sections []chunking.Section, approved []proposals.Enric
}
type collectSectionProposalsInput struct {
// ReportDiagnosticError may be called concurrently; it must not affect processing.
ReportDiagnosticError func(error) `json:"-"`
Module contracts.TranscriptModule
Spec contracts.ModuleRunSpec
Policy proposals.ReplacementPolicy
Sections []chunking.Section
ProposalClient contracts.StructuredLLMClient
DiagnosticsDir string
ValidationClient contracts.StructuredLLMClient
ValidationDiagnosticsDir string
GlossaryVariables *promptcontext.GlossaryVariables
GlossaryView *validators.GlossaryView
Interactions structuredoutput.InteractionSink
Module contracts.TranscriptModule
Spec contracts.ModuleRunSpec
Policy proposals.ReplacementPolicy
Sections []chunking.Section
ProposalClient contracts.StructuredLLMClient
ValidationClient contracts.StructuredLLMClient
GlossaryVariables *promptcontext.GlossaryVariables
GlossaryView *validators.GlossaryView
}
type sectionProposalResult struct {
@@ -403,14 +394,13 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI
proposalResult, err := input.Module.Propose(runCtx, contracts.ProposalRequest{
ExecutionContext: contracts.ExecutionContext{
StageIndex: &input.Spec.Index, StageID: input.Spec.ID,
Settings: input.Spec.Settings,
WorkingTranscript: transcriptFromSection(section),
Glossary: input.Spec.Glossary,
ProtectedTerms: input.Spec.ProtectedTerms,
GlossaryVariables: input.GlossaryVariables,
Section: &meta,
DiagnosticsDir: input.DiagnosticsDir,
ReportDiagnosticError: input.ReportDiagnosticError,
Settings: input.Spec.Settings,
WorkingTranscript: transcriptFromSection(section),
Glossary: input.Spec.Glossary,
ProtectedTerms: input.Spec.ProtectedTerms,
GlossaryVariables: input.GlossaryVariables,
Section: &meta,
Interactions: input.Interactions,
},
RunSpec: contracts.ModuleRunSpec{
ModuleKey: input.Spec.ModuleKey,
@@ -548,17 +538,16 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
go func(sectionPos int, enriched []proposals.EnrichedCorrectionProposal, sectionMetadata contracts.SectionMetadata) {
defer vwg.Done()
validationInput := validateSectionCandidatesInput{
Spec: input.Spec,
Policy: input.Policy,
WorkingTranscript: transcriptFromSection(sectionResult.section),
SectionIndex: &sectionMetadata.Index,
Validators: validatorsOrdered,
SectionEnriched: enriched,
Exchange: sectionResult.exchange,
ValidationLLMClient: input.ValidationClient,
DiagnosticsDir: input.ValidationDiagnosticsDir,
GlossaryView: input.GlossaryView,
ReportDiagnosticError: input.ReportDiagnosticError,
Spec: input.Spec,
Policy: input.Policy,
WorkingTranscript: transcriptFromSection(sectionResult.section),
SectionIndex: &sectionMetadata.Index,
Validators: validatorsOrdered,
SectionEnriched: enriched,
Exchange: sectionResult.exchange,
ValidationLLMClient: input.ValidationClient,
GlossaryView: input.GlossaryView,
Interactions: input.Interactions,
}
if input.Spec.Settings.LLM.CorrectionRepairAttempts > 0 && len(enriched) > 0 {
validationInput.Section = validators.NewSectionLookup(validationInput.WorkingTranscript)
@@ -656,17 +645,15 @@ type validateSectionCandidatesInput struct {
RepairProposalIndex *int
RepairAttempt *int
Exchange *structuredoutput.Exchange
// ReportDiagnosticError may be called concurrently; it must not affect processing.
ReportDiagnosticError func(error) `json:"-"`
Spec contracts.ModuleRunSpec
Policy proposals.ReplacementPolicy
WorkingTranscript *schema.Transcript
SectionIndex *int
Validators []contracts.Validator
SectionEnriched []proposals.EnrichedCorrectionProposal
ValidationLLMClient contracts.StructuredLLMClient
DiagnosticsDir string
GlossaryView *validators.GlossaryView
Spec contracts.ModuleRunSpec
Policy proposals.ReplacementPolicy
WorkingTranscript *schema.Transcript
SectionIndex *int
Validators []contracts.Validator
SectionEnriched []proposals.EnrichedCorrectionProposal
ValidationLLMClient contracts.StructuredLLMClient
Interactions structuredoutput.InteractionSink
GlossaryView *validators.GlossaryView
}
type validateSectionCandidatesResult struct {
@@ -693,21 +680,11 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
evidence.Section = validators.NewSectionLookup(input.WorkingTranscript)
}
}
var interactionWriter *llmDiagnosticsWriterAdapter
for validatorOrder, validator := range input.Validators {
if err := ctx.Err(); err != nil {
return validateSectionCandidatesResult{decisions: decisions, rejected: rejected, operations: operations, filtered: filtered}, err
}
var diagnosticsWriter validators.InteractionDiagnosticsWriter
_, directLLMValidator := validator.(*validators.LLMBackedValidator)
if input.DiagnosticsDir != "" && (directLLMValidator || validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked) {
if interactionWriter == nil {
interactionWriter = &llmDiagnosticsWriterAdapter{dir: filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName)}
}
diagnosticsWriter = interactionWriter
}
submitted := eligible
if feedback {
submitted = feedbackCandidates(input, validator, noEffect, evidence)
@@ -721,15 +698,14 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
ModuleInstance: input.Spec.InstanceName,
SectionIndex: input.SectionIndex,
IndependentEvidence: feedback, RepairProposalIndex: input.RepairProposalIndex, RepairAttempt: input.RepairAttempt,
ValidatorOrder: &validatorOrder,
ReplacementPolicy: input.Policy,
Glossary: input.Spec.Glossary,
ProtectedTerms: input.Spec.ProtectedTerms,
GlossaryView: input.GlossaryView,
Settings: input.Spec.Settings,
LLMClient: input.ValidationLLMClient,
DiagnosticsWriter: diagnosticsWriter,
ReportDiagnosticError: input.ReportDiagnosticError,
ValidatorOrder: &validatorOrder,
ReplacementPolicy: input.Policy,
Glossary: input.Spec.Glossary,
ProtectedTerms: input.Spec.ProtectedTerms,
GlossaryView: input.GlossaryView,
Settings: input.Spec.Settings,
LLMClient: input.ValidationLLMClient,
Interactions: input.Interactions,
})
operations = append(operations, vResult.Operations...)
@@ -985,20 +961,3 @@ func cloneTranscript(t *schema.Transcript) *schema.Transcript {
}
return &schema.Transcript{Segments: segments}
}
type llmDiagnosticsWriterAdapter struct {
dir string
}
func (a *llmDiagnosticsWriterAdapter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (validators.InteractionArtifacts, error) {
art, err := llm.NewDiagnosticsWriter(a.dir).WriteInteraction(stage, requestMetadata, requestPayload, responsePayload, errorPayload)
if err != nil {
return validators.InteractionArtifacts{}, err
}
return validators.InteractionArtifacts{
RequestMetadataPath: art.RequestMetadataPath,
RequestPayloadPath: art.RequestPayloadPath,
ResponsePayloadPath: art.ResponsePayloadPath,
ErrorPayloadPath: art.ErrorPayloadPath,
}, nil
}