Unify optional interaction capture and warnings
This commit is contained in:
@@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"sync"
|
||||
|
||||
@@ -13,7 +12,6 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/promptcontext"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||
@@ -94,14 +92,11 @@ type ValidatorRejectedChange struct {
|
||||
|
||||
// RunInput is the deterministic runner input.
|
||||
type RunInput struct {
|
||||
// ReportDiagnosticError may be called concurrently; it must not affect processing.
|
||||
ReportDiagnosticError func(error) `json:"-"`
|
||||
Config *config.Config
|
||||
Transcript *schema.Transcript
|
||||
ModuleSpecs []contracts.ModuleRunSpec
|
||||
ProposalDiagnosticsDir string
|
||||
ValidationDiagnosticsDir string
|
||||
CaptureCandidates bool
|
||||
Config *config.Config
|
||||
Transcript *schema.Transcript
|
||||
ModuleSpecs []contracts.ModuleRunSpec
|
||||
Interactions structuredoutput.InteractionSink
|
||||
CaptureCandidates bool
|
||||
}
|
||||
|
||||
// RunOutput is the deterministic runner output.
|
||||
@@ -172,17 +167,15 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
|
||||
}
|
||||
glossaryView := validators.PrepareGlossaryView(spec.Glossary, spec.ProtectedTerms)
|
||||
pipelineResult, pipelineErr := runModulePipeline(ctx, collectSectionProposalsInput{
|
||||
Module: module,
|
||||
Spec: spec,
|
||||
Policy: policy,
|
||||
Sections: sections,
|
||||
ProposalClient: spec.ProposalClient,
|
||||
DiagnosticsDir: input.ProposalDiagnosticsDir,
|
||||
ValidationClient: spec.ValidationClient,
|
||||
ValidationDiagnosticsDir: input.ValidationDiagnosticsDir,
|
||||
GlossaryVariables: &glossaryVariables,
|
||||
GlossaryView: glossaryView,
|
||||
ReportDiagnosticError: input.ReportDiagnosticError,
|
||||
Module: module,
|
||||
Spec: spec,
|
||||
Policy: policy,
|
||||
Sections: sections,
|
||||
ProposalClient: spec.ProposalClient,
|
||||
Interactions: input.Interactions,
|
||||
ValidationClient: spec.ValidationClient,
|
||||
GlossaryVariables: &glossaryVariables,
|
||||
GlossaryView: glossaryView,
|
||||
})
|
||||
if input.CaptureCandidates {
|
||||
candidates = append(candidates, pipelineResult.Candidates...)
|
||||
@@ -335,18 +328,16 @@ func applyWithinSections(sections []chunking.Section, approved []proposals.Enric
|
||||
}
|
||||
|
||||
type collectSectionProposalsInput struct {
|
||||
// ReportDiagnosticError may be called concurrently; it must not affect processing.
|
||||
ReportDiagnosticError func(error) `json:"-"`
|
||||
Module contracts.TranscriptModule
|
||||
Spec contracts.ModuleRunSpec
|
||||
Policy proposals.ReplacementPolicy
|
||||
Sections []chunking.Section
|
||||
ProposalClient contracts.StructuredLLMClient
|
||||
DiagnosticsDir string
|
||||
ValidationClient contracts.StructuredLLMClient
|
||||
ValidationDiagnosticsDir string
|
||||
GlossaryVariables *promptcontext.GlossaryVariables
|
||||
GlossaryView *validators.GlossaryView
|
||||
Interactions structuredoutput.InteractionSink
|
||||
Module contracts.TranscriptModule
|
||||
Spec contracts.ModuleRunSpec
|
||||
Policy proposals.ReplacementPolicy
|
||||
Sections []chunking.Section
|
||||
ProposalClient contracts.StructuredLLMClient
|
||||
|
||||
ValidationClient contracts.StructuredLLMClient
|
||||
GlossaryVariables *promptcontext.GlossaryVariables
|
||||
GlossaryView *validators.GlossaryView
|
||||
}
|
||||
|
||||
type sectionProposalResult struct {
|
||||
@@ -403,14 +394,13 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI
|
||||
proposalResult, err := input.Module.Propose(runCtx, contracts.ProposalRequest{
|
||||
ExecutionContext: contracts.ExecutionContext{
|
||||
StageIndex: &input.Spec.Index, StageID: input.Spec.ID,
|
||||
Settings: input.Spec.Settings,
|
||||
WorkingTranscript: transcriptFromSection(section),
|
||||
Glossary: input.Spec.Glossary,
|
||||
ProtectedTerms: input.Spec.ProtectedTerms,
|
||||
GlossaryVariables: input.GlossaryVariables,
|
||||
Section: &meta,
|
||||
DiagnosticsDir: input.DiagnosticsDir,
|
||||
ReportDiagnosticError: input.ReportDiagnosticError,
|
||||
Settings: input.Spec.Settings,
|
||||
WorkingTranscript: transcriptFromSection(section),
|
||||
Glossary: input.Spec.Glossary,
|
||||
ProtectedTerms: input.Spec.ProtectedTerms,
|
||||
GlossaryVariables: input.GlossaryVariables,
|
||||
Section: &meta,
|
||||
Interactions: input.Interactions,
|
||||
},
|
||||
RunSpec: contracts.ModuleRunSpec{
|
||||
ModuleKey: input.Spec.ModuleKey,
|
||||
@@ -548,17 +538,16 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
||||
go func(sectionPos int, enriched []proposals.EnrichedCorrectionProposal, sectionMetadata contracts.SectionMetadata) {
|
||||
defer vwg.Done()
|
||||
validationInput := validateSectionCandidatesInput{
|
||||
Spec: input.Spec,
|
||||
Policy: input.Policy,
|
||||
WorkingTranscript: transcriptFromSection(sectionResult.section),
|
||||
SectionIndex: §ionMetadata.Index,
|
||||
Validators: validatorsOrdered,
|
||||
SectionEnriched: enriched,
|
||||
Exchange: sectionResult.exchange,
|
||||
ValidationLLMClient: input.ValidationClient,
|
||||
DiagnosticsDir: input.ValidationDiagnosticsDir,
|
||||
GlossaryView: input.GlossaryView,
|
||||
ReportDiagnosticError: input.ReportDiagnosticError,
|
||||
Spec: input.Spec,
|
||||
Policy: input.Policy,
|
||||
WorkingTranscript: transcriptFromSection(sectionResult.section),
|
||||
SectionIndex: §ionMetadata.Index,
|
||||
Validators: validatorsOrdered,
|
||||
SectionEnriched: enriched,
|
||||
Exchange: sectionResult.exchange,
|
||||
ValidationLLMClient: input.ValidationClient,
|
||||
GlossaryView: input.GlossaryView,
|
||||
Interactions: input.Interactions,
|
||||
}
|
||||
if input.Spec.Settings.LLM.CorrectionRepairAttempts > 0 && len(enriched) > 0 {
|
||||
validationInput.Section = validators.NewSectionLookup(validationInput.WorkingTranscript)
|
||||
@@ -656,17 +645,15 @@ type validateSectionCandidatesInput struct {
|
||||
RepairProposalIndex *int
|
||||
RepairAttempt *int
|
||||
Exchange *structuredoutput.Exchange
|
||||
// ReportDiagnosticError may be called concurrently; it must not affect processing.
|
||||
ReportDiagnosticError func(error) `json:"-"`
|
||||
Spec contracts.ModuleRunSpec
|
||||
Policy proposals.ReplacementPolicy
|
||||
WorkingTranscript *schema.Transcript
|
||||
SectionIndex *int
|
||||
Validators []contracts.Validator
|
||||
SectionEnriched []proposals.EnrichedCorrectionProposal
|
||||
ValidationLLMClient contracts.StructuredLLMClient
|
||||
DiagnosticsDir string
|
||||
GlossaryView *validators.GlossaryView
|
||||
Spec contracts.ModuleRunSpec
|
||||
Policy proposals.ReplacementPolicy
|
||||
WorkingTranscript *schema.Transcript
|
||||
SectionIndex *int
|
||||
Validators []contracts.Validator
|
||||
SectionEnriched []proposals.EnrichedCorrectionProposal
|
||||
ValidationLLMClient contracts.StructuredLLMClient
|
||||
Interactions structuredoutput.InteractionSink
|
||||
GlossaryView *validators.GlossaryView
|
||||
}
|
||||
|
||||
type validateSectionCandidatesResult struct {
|
||||
@@ -693,21 +680,11 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
||||
evidence.Section = validators.NewSectionLookup(input.WorkingTranscript)
|
||||
}
|
||||
}
|
||||
var interactionWriter *llmDiagnosticsWriterAdapter
|
||||
|
||||
for validatorOrder, validator := range input.Validators {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return validateSectionCandidatesResult{decisions: decisions, rejected: rejected, operations: operations, filtered: filtered}, err
|
||||
}
|
||||
var diagnosticsWriter validators.InteractionDiagnosticsWriter
|
||||
_, directLLMValidator := validator.(*validators.LLMBackedValidator)
|
||||
if input.DiagnosticsDir != "" && (directLLMValidator || validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked) {
|
||||
if interactionWriter == nil {
|
||||
interactionWriter = &llmDiagnosticsWriterAdapter{dir: filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName)}
|
||||
}
|
||||
diagnosticsWriter = interactionWriter
|
||||
}
|
||||
|
||||
submitted := eligible
|
||||
if feedback {
|
||||
submitted = feedbackCandidates(input, validator, noEffect, evidence)
|
||||
@@ -721,15 +698,14 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
||||
ModuleInstance: input.Spec.InstanceName,
|
||||
SectionIndex: input.SectionIndex,
|
||||
IndependentEvidence: feedback, RepairProposalIndex: input.RepairProposalIndex, RepairAttempt: input.RepairAttempt,
|
||||
ValidatorOrder: &validatorOrder,
|
||||
ReplacementPolicy: input.Policy,
|
||||
Glossary: input.Spec.Glossary,
|
||||
ProtectedTerms: input.Spec.ProtectedTerms,
|
||||
GlossaryView: input.GlossaryView,
|
||||
Settings: input.Spec.Settings,
|
||||
LLMClient: input.ValidationLLMClient,
|
||||
DiagnosticsWriter: diagnosticsWriter,
|
||||
ReportDiagnosticError: input.ReportDiagnosticError,
|
||||
ValidatorOrder: &validatorOrder,
|
||||
ReplacementPolicy: input.Policy,
|
||||
Glossary: input.Spec.Glossary,
|
||||
ProtectedTerms: input.Spec.ProtectedTerms,
|
||||
GlossaryView: input.GlossaryView,
|
||||
Settings: input.Spec.Settings,
|
||||
LLMClient: input.ValidationLLMClient,
|
||||
Interactions: input.Interactions,
|
||||
})
|
||||
operations = append(operations, vResult.Operations...)
|
||||
|
||||
@@ -985,20 +961,3 @@ func cloneTranscript(t *schema.Transcript) *schema.Transcript {
|
||||
}
|
||||
return &schema.Transcript{Segments: segments}
|
||||
}
|
||||
|
||||
type llmDiagnosticsWriterAdapter struct {
|
||||
dir string
|
||||
}
|
||||
|
||||
func (a *llmDiagnosticsWriterAdapter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (validators.InteractionArtifacts, error) {
|
||||
art, err := llm.NewDiagnosticsWriter(a.dir).WriteInteraction(stage, requestMetadata, requestPayload, responsePayload, errorPayload)
|
||||
if err != nil {
|
||||
return validators.InteractionArtifacts{}, err
|
||||
}
|
||||
return validators.InteractionArtifacts{
|
||||
RequestMetadataPath: art.RequestMetadataPath,
|
||||
RequestPayloadPath: art.RequestPayloadPath,
|
||||
ResponsePayloadPath: art.ResponsePayloadPath,
|
||||
ErrorPayloadPath: art.ErrorPayloadPath,
|
||||
}, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user