eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Share section lookups and replacement previews across validators

Browse Source
This commit is contained in:
Eric Rakestraw
2026-11-11 12:28:00 +00:00
parent 45b42a2689
commit 5d90478f70
7 changed files with 107 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
docs/roadmap/remediation.md
View File

@@ -2,7 +2,7 @@
**Updated:** 2026-11-11
**Status:** 3 of nine findings resolved. Findings refer to implementation revision
**Status:** 4 of nine findings resolved. Findings refer to implementation revision
`aa8073c`. See the [implementation plan](implementation.md).
## Review Result
@@ -139,6 +139,8 @@ outbound request without depending on provider caching behavior.
### R4 — Reuse Replacement Previews Across Feedback Checks (P3)
**Completed.** Each source section shares one target index across validation passes. Preview-dependent checks reuse one lazy preview per version. Targeted tests passed, and the offline benchmark showed lower allocation counts at both sizes.
**Affected plan:** Stage 5, items 1–2.
**Evidence:** [feedbackCandidates](../../internal/framework/runner/feedback.go#L30)

23
internal/framework/runner/feedback.go
View File

@@ -11,30 +11,13 @@ import (
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
)
func feedbackPreview(input validateSectionCandidatesInput, candidate proposals.EnrichedCorrectionProposal) proposals.SegmentPreviewResult {
if input.WorkingTranscript != nil {
for i := range input.WorkingTranscript.Segments {
segment := &input.WorkingTranscript.Segments[i]
if segment.ID == candidate.TargetSegmentID {
correction := candidate.CorrectionProposal
if math.IsNaN(correction.Confidence) || correction.Confidence < 0 || correction.Confidence > 1 {
correction.Confidence = 0.5
}
return proposals.PreviewProposalForSegment(segment, correction, input.Policy)
}
}
}
return proposals.PreviewProposalForSegment(nil, candidate.CorrectionProposal, input.Policy)
}
// feedbackCandidates submits only candidates for which the validator has usable evidence.
func feedbackCandidates(input validateSectionCandidatesInput, validator contracts.Validator, noEffect map[int]bool) []proposals.EnrichedCorrectionProposal {
func feedbackCandidates(input validateSectionCandidatesInput, validator contracts.Validator, noEffect map[int]bool, evidence contracts.ValidationRequest) []proposals.EnrichedCorrectionProposal {
var result []proposals.EnrichedCorrectionProposal
for _, candidate := range input.SectionEnriched {
if noEffect[candidate.ProposalIndex] {
continue
}
preview := feedbackPreview(input, candidate)
usableSpan := candidate.OriginalText != ""
target := candidate.TargetSegmentID > 0
confidence := !math.IsNaN(candidate.Confidence) && candidate.Confidence >= 0 && candidate.Confidence <= 1
@@ -51,13 +34,15 @@ func feedbackCandidates(input validateSectionCandidatesInput, validator contract
case "protected_terms":
applicable = usableSpan
if input.Spec.ModuleKey == modulecatalog.KeyGlossary || input.Spec.ModuleKey == modulecatalog.KeySpokenWord {
preview := evidence.Preview(candidate)
applicable = preview.Applicable || preview.SkipReason == proposals.SkipReasonEmptyResultingText
}
case "non_empty_resulting_segment":
preview := evidence.Preview(candidate)
applicable = preview.Applicable || preview.SkipReason == proposals.SkipReasonEmptyResultingText
default:
if validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked {
applicable = preview.Applicable
applicable = evidence.Preview(candidate).Applicable
} else {
applicable = true
}

34
internal/framework/runner/runner.go
View File

@@ -543,7 +543,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
vwg.Add(1)
go func(sectionPos int, enriched []proposals.EnrichedCorrectionProposal, sectionMetadata contracts.SectionMetadata) {
defer vwg.Done()
validated, err := validateSectionCandidates(runCtx, validateSectionCandidatesInput{
validationInput := validateSectionCandidatesInput{
Spec: input.Spec,
Policy: input.Policy,
WorkingTranscript: transcriptFromSection(sectionResult.section),
@@ -555,16 +555,19 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
DiagnosticsDir: input.ValidationDiagnosticsDir,
GlossaryView: input.GlossaryView,
ReportDiagnosticError: input.ReportDiagnosticError,
})
}
if input.Spec.Settings.LLM.CorrectionRepairAttempts > 0 && len(enriched) > 0 {
validationInput.Section = validators.NewSectionLookup(validationInput.WorkingTranscript)
}
validated, err := validateSectionCandidates(runCtx, validationInput)
var repairs []RepairOutcome
if err == nil && input.Spec.Settings.LLM.CorrectionRepairAttempts > 0 {
promptID := ""
if provider, ok := input.Module.(interface{ RepairPromptID() string }); ok {
promptID = provider.RepairPromptID()
}
repairInput := validateSectionCandidatesInput{Spec: input.Spec, Policy: input.Policy, WorkingTranscript: transcriptFromSection(sectionResult.section), SectionIndex: &sectionMetadata.Index, Validators: validatorsOrdered, SectionEnriched: enriched, Exchange: sectionResult.exchange, ValidationLLMClient: input.ValidationClient, DiagnosticsDir: input.ValidationDiagnosticsDir, GlossaryView: input.GlossaryView, ReportDiagnosticError: input.ReportDiagnosticError}
validated, repairs, err = repairCandidates(runCtx, repairInput, sectionResult.references, validated, promptID)
validated.operations = append(validated.operations, repairOperations(repairInput, repairs)...)
validated, repairs, err = repairCandidates(runCtx, validationInput, sectionResult.references, validated, promptID)
validated.operations = append(validated.operations, repairOperations(validationInput, repairs)...)
}
setErr(err)
validationResults <- sectionValidationResult{
@@ -646,6 +649,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
}
type validateSectionCandidatesInput struct {
Section *validators.SectionLookup
RepairProposalIndex *int
RepairAttempt *int
Exchange *structuredoutput.Exchange
@@ -677,8 +681,15 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
filtered := make([]outcomes.FilteredProposal, 0)
eligible := append([]proposals.EnrichedCorrectionProposal(nil), input.SectionEnriched...)
feedback := input.Spec.Settings.LLM.CorrectionRepairAttempts > 0
blocked := make(map[int]bool)
noEffect := make(map[int]bool)
var blocked, noEffect map[int]bool
evidence := contracts.ValidationRequest{WorkingTranscript: input.WorkingTranscript, Section: input.Section, ReplacementPolicy: input.Policy, IndependentEvidence: feedback}
if feedback {
blocked, noEffect = make(map[int]bool), make(map[int]bool)
evidence.Previews = make(map[int]proposals.SegmentPreviewResult)
if evidence.Section == nil && len(input.SectionEnriched) > 0 {
evidence.Section = validators.NewSectionLookup(input.WorkingTranscript)
}
}
var interactionWriter *llmDiagnosticsWriterAdapter
for validatorOrder, validator := range input.Validators {
@@ -701,11 +712,12 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
submitted := eligible
if feedback {
submitted = feedbackCandidates(input, validator, noEffect)
submitted = feedbackCandidates(input, validator, noEffect, evidence)
}
vResult, err := validator.Validate(ctx, contracts.ValidationRequest{
StageIndex: &input.Spec.Index, StageID: input.Spec.ID,
WorkingTranscript: input.WorkingTranscript,
WorkingTranscript: input.WorkingTranscript,
Section: evidence.Section, Previews: evidence.Previews,
CandidateProposal: submitted,
ModuleKey: input.Spec.ModuleKey,
ModuleInstance: input.Spec.InstanceName,
@@ -743,7 +755,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
for _, d := range vResult.Decisions {
if feedback && validator.Name() == "original_text_presence" && d.Approved {
preview := feedbackPreview(input, byIndex[d.ProposalIndex])
preview := evidence.Preview(byIndex[d.ProposalIndex])
if preview.SkipReason == proposals.SkipReasonAmbiguousOriginal {
d.Approved = false
d.Disposition = validators.DispositionRejected
@@ -751,7 +763,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
d.Message = "original_text matches multiple locations in the target segment"
}
}
if !d.Approved {
if feedback && !d.Approved {
blocked[d.ProposalIndex] = true
if d.ReasonCode == validators.ReasonNoEffect {
noEffect[d.ProposalIndex] = true

47
internal/framework/validators/deterministic.go
View File

@@ -3,11 +3,9 @@ package validators
import (
"context"
"fmt"
"math"
"strings"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
)
@@ -60,21 +58,16 @@ type OriginalTextPresenceValidator struct{}
func (v OriginalTextPresenceValidator) Name() string { return "original_text_presence" }
func (v OriginalTextPresenceValidator) Validate(_ context.Context, req Request) (Result, error) {
byID := make(map[int]string)
if req.WorkingTranscript != nil {
for _, seg := range req.WorkingTranscript.Segments {
byID[seg.ID] = seg.Text
}
}
section := req.sectionLookup()
decisions := make([]Decision, 0, len(req.CandidateProposal))
for _, c := range req.CandidateProposal {
text, ok := byID[c.TargetSegmentID]
position, ok := section.Position(c.TargetSegmentID)
if !ok {
decisions = append(decisions, rejection(c.ProposalIndex, ReasonMissingTargetSegment, "target segment was not found"))
continue
}
if !strings.Contains(text, c.OriginalText) {
if !strings.Contains(section.segments[position].Text, c.OriginalText) {
decisions = append(decisions, rejection(c.ProposalIndex, ReasonMissingOriginalText, "original_text was not found in current segment text"))
continue
}
@@ -91,21 +84,11 @@ type NonEmptyResultingSegmentValidator struct{}
func (v NonEmptyResultingSegmentValidator) Name() string { return "non_empty_resulting_segment" }
func (v NonEmptyResultingSegmentValidator) Validate(_ context.Context, req Request) (Result, error) {
segmentsByID := make(map[int]schema.Segment)
if req.WorkingTranscript != nil {
for _, seg := range req.WorkingTranscript.Segments {
segmentsByID[seg.ID] = seg
}
}
req.Section = req.sectionLookup()
decisions := make([]Decision, 0, len(req.CandidateProposal))
for _, c := range req.CandidateProposal {
segment, ok := segmentsByID[c.TargetSegmentID]
if !ok {
decisions = append(decisions, approval(c.ProposalIndex))
continue
}
preview := previewForEvidence(req, &segment, c.CorrectionProposal)
preview := req.Preview(c)
if preview.SkipReason == proposals.SkipReasonEmptyResultingText {
decisions = append(decisions, rejection(c.ProposalIndex, ReasonEmptyResultingText, "proposal would leave the segment empty"))
continue
@@ -149,13 +132,11 @@ func ValidateProtectedTerms(req Request) (Result, error) {
for _, c := range req.CandidateProposal {
reason := vocab.violationReason(c.OriginalText, c.CorrectedText)
repetitionCleanup := reason != "" && req.ModuleKey == modulecatalog.KeySpokenWord
if vocab.pattern != nil && (glossaryStage || repetitionCleanup) && req.WorkingTranscript != nil {
for i := range req.WorkingTranscript.Segments {
segment := &req.WorkingTranscript.Segments[i]
if segment.ID != c.TargetSegmentID {
continue
}
preview := previewForEvidence(req, segment, c.CorrectionProposal)
if vocab.pattern != nil && (glossaryStage || repetitionCleanup) {
req.Section = req.sectionLookup()
if position, ok := req.Section.Position(c.TargetSegmentID); ok {
segment := &req.Section.segments[position]
preview := req.Preview(c)
if preview.Applicable || (req.IndependentEvidence && preview.SkipReason == proposals.SkipReasonEmptyResultingText) {
if glossaryStage {
// A correction span may sit inside a longer exempt name.
@@ -164,7 +145,6 @@ func ValidateProtectedTerms(req Request) (Result, error) {
reason = ""
}
}
break
}
}
if reason != "" {
@@ -178,10 +158,3 @@ func ValidateProtectedTerms(req Request) (Result, error) {
}
return Result{ValidatorName: "protected_terms", Decisions: decisions}, nil
}
func previewForEvidence(req Request, segment *schema.Segment, correction proposals.CorrectionProposal) proposals.SegmentPreviewResult {
if req.IndependentEvidence && (math.IsNaN(correction.Confidence) || correction.Confidence < 0 || correction.Confidence > 1) {
correction.Confidence = 0.5
}
return proposals.PreviewProposalForSegment(segment, correction, req.ReplacementPolicy)
}

15
internal/framework/validators/llm_validators.go
View File

@@ -9,7 +9,6 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/stagename"
@@ -243,30 +242,24 @@ func errPayload(err error) any {
func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorType, req Request) (LLMValidationRequest, []Decision) {
items := make([]LLMValidationItem, 0, len(req.CandidateProposal))
immediate := make([]Decision, 0)
var segments []schema.Segment
positions := make(map[int]int)
req.Section = req.sectionLookup()
segments := req.Section.segments
var glossary glossaryContext
if req.GlossaryView != nil {
glossary = req.GlossaryView.context
} else {
glossary = prepareGlossaryContext(req.Glossary)
}
if req.WorkingTranscript != nil {
segments = req.WorkingTranscript.Segments
for i, seg := range segments {
positions[seg.ID] = i
}
}
for _, p := range req.CandidateProposal {
position, ok := positions[p.TargetSegmentID]
position, ok := req.Section.Position(p.TargetSegmentID)
if !ok {
immediate = append(immediate, rejection(p.ProposalIndex, ReasonMissingTargetSegment, "target segment was not found"))
continue
}
seg := segments[position]
preview := previewForEvidence(req, &seg, p.CorrectionProposal)
preview := req.Preview(p)
if !preview.Applicable {
immediate = append(immediate, rejection(p.ProposalIndex, string(preview.SkipReason), "proposal is not previewable for LLM validation"))
continue

2
internal/framework/validators/models.go
View File

@@ -41,6 +41,8 @@ type Request struct {
RepairAttempt *int `json:"repair_attempt,omitempty"`
SectionIndex *int `json:"section_index,omitempty"`
WorkingTranscript *schema.Transcript `json:"-"`
Section *SectionLookup `json:"-"`
Previews map[int]proposals.SegmentPreviewResult `json:"-"`
CandidateProposal []proposals.EnrichedCorrectionProposal `json:"candidate_proposals"`
ModuleKey string `json:"module_key"`
ModuleInstance string `json:"module_instance"`

61
internal/framework/validators/section.go Normal file
View File

@@ -0,0 +1,61 @@
package validators
import (
"math"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
)
// SectionLookup indexes the immutable source section shared by validation
// and revalidation. Application checks its own, changing transcript.
type SectionLookup struct {
segments []schema.Segment
positions map[int]int
}
func NewSectionLookup(transcript *schema.Transcript) *SectionLookup {
section := &SectionLookup{}
if transcript != nil {
section.segments = transcript.Segments
section.positions = make(map[int]int, len(section.segments))
for i, segment := range section.segments {
section.positions[segment.ID] = i
}
}
return section
}
func (s *SectionLookup) Position(id int) (int, bool) {
position, ok := s.positions[id]
return position, ok
}
func (r Request) sectionLookup() *SectionLookup {
if r.Section != nil {
return r.Section
}
return NewSectionLookup(r.WorkingTranscript)
}
// Preview reuses the replacement preview for this proposal index within one
// validation pass. A revision starts a new pass with its own preview map.
func (r Request) Preview(candidate proposals.EnrichedCorrectionProposal) proposals.SegmentPreviewResult {
if preview, ok := r.Previews[candidate.ProposalIndex]; ok {
return preview
}
section := r.sectionLookup()
var segment *schema.Segment
if position, ok := section.Position(candidate.TargetSegmentID); ok {
segment = &section.segments[position]
}
correction := candidate.CorrectionProposal
if r.IndependentEvidence && (math.IsNaN(correction.Confidence) || correction.Confidence < 0 || correction.Confidence > 1) {
correction.Confidence = 0.5
}
preview := proposals.PreviewProposalForSegment(segment, correction, r.ReplacementPolicy)
if r.Previews != nil {
r.Previews[candidate.ProposalIndex] = preview
}
return preview
}