eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Share section lookups and replacement previews across validators

Browse Source
This commit is contained in:
Eric Rakestraw
2026-11-11 12:28:00 +00:00
parent 45b42a2689
commit 5d90478f70
7 changed files with 107 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
internal/framework/runner/feedback.go
View File

@@ -11,30 +11,13 @@ import (
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
)
func feedbackPreview(input validateSectionCandidatesInput, candidate proposals.EnrichedCorrectionProposal) proposals.SegmentPreviewResult {
if input.WorkingTranscript != nil {
for i := range input.WorkingTranscript.Segments {
segment := &input.WorkingTranscript.Segments[i]
if segment.ID == candidate.TargetSegmentID {
correction := candidate.CorrectionProposal
if math.IsNaN(correction.Confidence) || correction.Confidence < 0 || correction.Confidence > 1 {
correction.Confidence = 0.5
}
return proposals.PreviewProposalForSegment(segment, correction, input.Policy)
}
}
}
return proposals.PreviewProposalForSegment(nil, candidate.CorrectionProposal, input.Policy)
}
// feedbackCandidates submits only candidates for which the validator has usable evidence.
func feedbackCandidates(input validateSectionCandidatesInput, validator contracts.Validator, noEffect map[int]bool) []proposals.EnrichedCorrectionProposal {
func feedbackCandidates(input validateSectionCandidatesInput, validator contracts.Validator, noEffect map[int]bool, evidence contracts.ValidationRequest) []proposals.EnrichedCorrectionProposal {
var result []proposals.EnrichedCorrectionProposal
for _, candidate := range input.SectionEnriched {
if noEffect[candidate.ProposalIndex] {
continue
}
preview := feedbackPreview(input, candidate)
usableSpan := candidate.OriginalText != ""
target := candidate.TargetSegmentID > 0
confidence := !math.IsNaN(candidate.Confidence) && candidate.Confidence >= 0 && candidate.Confidence <= 1
@@ -51,13 +34,15 @@ func feedbackCandidates(input validateSectionCandidatesInput, validator contract
case "protected_terms":
applicable = usableSpan
if input.Spec.ModuleKey == modulecatalog.KeyGlossary || input.Spec.ModuleKey == modulecatalog.KeySpokenWord {
preview := evidence.Preview(candidate)
applicable = preview.Applicable || preview.SkipReason == proposals.SkipReasonEmptyResultingText
}
case "non_empty_resulting_segment":
preview := evidence.Preview(candidate)
applicable = preview.Applicable || preview.SkipReason == proposals.SkipReasonEmptyResultingText
default:
if validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked {
applicable = preview.Applicable
applicable = evidence.Preview(candidate).Applicable
} else {
applicable = true
}

34
internal/framework/runner/runner.go
View File

@@ -543,7 +543,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
vwg.Add(1)
go func(sectionPos int, enriched []proposals.EnrichedCorrectionProposal, sectionMetadata contracts.SectionMetadata) {
defer vwg.Done()
validated, err := validateSectionCandidates(runCtx, validateSectionCandidatesInput{
validationInput := validateSectionCandidatesInput{
Spec: input.Spec,
Policy: input.Policy,
WorkingTranscript: transcriptFromSection(sectionResult.section),
@@ -555,16 +555,19 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
DiagnosticsDir: input.ValidationDiagnosticsDir,
GlossaryView: input.GlossaryView,
ReportDiagnosticError: input.ReportDiagnosticError,
})
}
if input.Spec.Settings.LLM.CorrectionRepairAttempts > 0 && len(enriched) > 0 {
validationInput.Section = validators.NewSectionLookup(validationInput.WorkingTranscript)
}
validated, err := validateSectionCandidates(runCtx, validationInput)
var repairs []RepairOutcome
if err == nil && input.Spec.Settings.LLM.CorrectionRepairAttempts > 0 {
promptID := ""
if provider, ok := input.Module.(interface{ RepairPromptID() string }); ok {
promptID = provider.RepairPromptID()
}
repairInput := validateSectionCandidatesInput{Spec: input.Spec, Policy: input.Policy, WorkingTranscript: transcriptFromSection(sectionResult.section), SectionIndex: &sectionMetadata.Index, Validators: validatorsOrdered, SectionEnriched: enriched, Exchange: sectionResult.exchange, ValidationLLMClient: input.ValidationClient, DiagnosticsDir: input.ValidationDiagnosticsDir, GlossaryView: input.GlossaryView, ReportDiagnosticError: input.ReportDiagnosticError}
validated, repairs, err = repairCandidates(runCtx, repairInput, sectionResult.references, validated, promptID)
validated.operations = append(validated.operations, repairOperations(repairInput, repairs)...)
validated, repairs, err = repairCandidates(runCtx, validationInput, sectionResult.references, validated, promptID)
validated.operations = append(validated.operations, repairOperations(validationInput, repairs)...)
}
setErr(err)
validationResults <- sectionValidationResult{
@@ -646,6 +649,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
}
type validateSectionCandidatesInput struct {
Section *validators.SectionLookup
RepairProposalIndex *int
RepairAttempt *int
Exchange *structuredoutput.Exchange
@@ -677,8 +681,15 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
filtered := make([]outcomes.FilteredProposal, 0)
eligible := append([]proposals.EnrichedCorrectionProposal(nil), input.SectionEnriched...)
feedback := input.Spec.Settings.LLM.CorrectionRepairAttempts > 0
blocked := make(map[int]bool)
noEffect := make(map[int]bool)
var blocked, noEffect map[int]bool
evidence := contracts.ValidationRequest{WorkingTranscript: input.WorkingTranscript, Section: input.Section, ReplacementPolicy: input.Policy, IndependentEvidence: feedback}
if feedback {
blocked, noEffect = make(map[int]bool), make(map[int]bool)
evidence.Previews = make(map[int]proposals.SegmentPreviewResult)
if evidence.Section == nil && len(input.SectionEnriched) > 0 {
evidence.Section = validators.NewSectionLookup(input.WorkingTranscript)
}
}
var interactionWriter *llmDiagnosticsWriterAdapter
for validatorOrder, validator := range input.Validators {
@@ -701,11 +712,12 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
submitted := eligible
if feedback {
submitted = feedbackCandidates(input, validator, noEffect)
submitted = feedbackCandidates(input, validator, noEffect, evidence)
}
vResult, err := validator.Validate(ctx, contracts.ValidationRequest{
StageIndex: &input.Spec.Index, StageID: input.Spec.ID,
WorkingTranscript: input.WorkingTranscript,
WorkingTranscript: input.WorkingTranscript,
Section: evidence.Section, Previews: evidence.Previews,
CandidateProposal: submitted,
ModuleKey: input.Spec.ModuleKey,
ModuleInstance: input.Spec.InstanceName,
@@ -743,7 +755,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
for _, d := range vResult.Decisions {
if feedback && validator.Name() == "original_text_presence" && d.Approved {
preview := feedbackPreview(input, byIndex[d.ProposalIndex])
preview := evidence.Preview(byIndex[d.ProposalIndex])
if preview.SkipReason == proposals.SkipReasonAmbiguousOriginal {
d.Approved = false
d.Disposition = validators.DispositionRejected
@@ -751,7 +763,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
d.Message = "original_text matches multiple locations in the target segment"
}
}
if !d.Approved {
if feedback && !d.Approved {
blocked[d.ProposalIndex] = true
if d.ReasonCode == validators.ReasonNoEffect {
noEffect[d.ProposalIndex] = true