diff --git a/docs/roadmap/remediation.md b/docs/roadmap/remediation.md index 9608696..d9db35b 100644 --- a/docs/roadmap/remediation.md +++ b/docs/roadmap/remediation.md @@ -2,7 +2,7 @@ **Updated:** 2026-11-11 -**Status:** 3 of nine findings resolved. Findings refer to implementation revision +**Status:** 4 of nine findings resolved. Findings refer to implementation revision `aa8073c`. See the [implementation plan](implementation.md). ## Review Result @@ -139,6 +139,8 @@ outbound request without depending on provider caching behavior. ### R4 — Reuse Replacement Previews Across Feedback Checks (P3) +**Completed.** Each source section shares one target index across validation passes. Preview-dependent checks reuse one lazy preview per version. Targeted tests passed, and the offline benchmark showed lower allocation counts at both sizes. + **Affected plan:** Stage 5, items 1–2. **Evidence:** [feedbackCandidates](../../internal/framework/runner/feedback.go#L30) diff --git a/internal/framework/runner/feedback.go b/internal/framework/runner/feedback.go index 3cc7895..4e794e6 100644 --- a/internal/framework/runner/feedback.go +++ b/internal/framework/runner/feedback.go @@ -11,30 +11,13 @@ import ( validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata" ) -func feedbackPreview(input validateSectionCandidatesInput, candidate proposals.EnrichedCorrectionProposal) proposals.SegmentPreviewResult { - if input.WorkingTranscript != nil { - for i := range input.WorkingTranscript.Segments { - segment := &input.WorkingTranscript.Segments[i] - if segment.ID == candidate.TargetSegmentID { - correction := candidate.CorrectionProposal - if math.IsNaN(correction.Confidence) || correction.Confidence < 0 || correction.Confidence > 1 { - correction.Confidence = 0.5 - } - return proposals.PreviewProposalForSegment(segment, correction, input.Policy) - } - } - } - return proposals.PreviewProposalForSegment(nil, candidate.CorrectionProposal, input.Policy) -} - // feedbackCandidates submits only candidates for which the validator has usable evidence. -func feedbackCandidates(input validateSectionCandidatesInput, validator contracts.Validator, noEffect map[int]bool) []proposals.EnrichedCorrectionProposal { +func feedbackCandidates(input validateSectionCandidatesInput, validator contracts.Validator, noEffect map[int]bool, evidence contracts.ValidationRequest) []proposals.EnrichedCorrectionProposal { var result []proposals.EnrichedCorrectionProposal for _, candidate := range input.SectionEnriched { if noEffect[candidate.ProposalIndex] { continue } - preview := feedbackPreview(input, candidate) usableSpan := candidate.OriginalText != "" target := candidate.TargetSegmentID > 0 confidence := !math.IsNaN(candidate.Confidence) && candidate.Confidence >= 0 && candidate.Confidence <= 1 @@ -51,13 +34,15 @@ func feedbackCandidates(input validateSectionCandidatesInput, validator contract case "protected_terms": applicable = usableSpan if input.Spec.ModuleKey == modulecatalog.KeyGlossary || input.Spec.ModuleKey == modulecatalog.KeySpokenWord { + preview := evidence.Preview(candidate) applicable = preview.Applicable || preview.SkipReason == proposals.SkipReasonEmptyResultingText } case "non_empty_resulting_segment": + preview := evidence.Preview(candidate) applicable = preview.Applicable || preview.SkipReason == proposals.SkipReasonEmptyResultingText default: if validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked { - applicable = preview.Applicable + applicable = evidence.Preview(candidate).Applicable } else { applicable = true } diff --git a/internal/framework/runner/runner.go b/internal/framework/runner/runner.go index c5dcd6f..c505797 100644 --- a/internal/framework/runner/runner.go +++ b/internal/framework/runner/runner.go @@ -543,7 +543,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) vwg.Add(1) go func(sectionPos int, enriched []proposals.EnrichedCorrectionProposal, sectionMetadata contracts.SectionMetadata) { defer vwg.Done() - validated, err := validateSectionCandidates(runCtx, validateSectionCandidatesInput{ + validationInput := validateSectionCandidatesInput{ Spec: input.Spec, Policy: input.Policy, WorkingTranscript: transcriptFromSection(sectionResult.section), @@ -555,16 +555,19 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) DiagnosticsDir: input.ValidationDiagnosticsDir, GlossaryView: input.GlossaryView, ReportDiagnosticError: input.ReportDiagnosticError, - }) + } + if input.Spec.Settings.LLM.CorrectionRepairAttempts > 0 && len(enriched) > 0 { + validationInput.Section = validators.NewSectionLookup(validationInput.WorkingTranscript) + } + validated, err := validateSectionCandidates(runCtx, validationInput) var repairs []RepairOutcome if err == nil && input.Spec.Settings.LLM.CorrectionRepairAttempts > 0 { promptID := "" if provider, ok := input.Module.(interface{ RepairPromptID() string }); ok { promptID = provider.RepairPromptID() } - repairInput := validateSectionCandidatesInput{Spec: input.Spec, Policy: input.Policy, WorkingTranscript: transcriptFromSection(sectionResult.section), SectionIndex: §ionMetadata.Index, Validators: validatorsOrdered, SectionEnriched: enriched, Exchange: sectionResult.exchange, ValidationLLMClient: input.ValidationClient, DiagnosticsDir: input.ValidationDiagnosticsDir, GlossaryView: input.GlossaryView, ReportDiagnosticError: input.ReportDiagnosticError} - validated, repairs, err = repairCandidates(runCtx, repairInput, sectionResult.references, validated, promptID) - validated.operations = append(validated.operations, repairOperations(repairInput, repairs)...) + validated, repairs, err = repairCandidates(runCtx, validationInput, sectionResult.references, validated, promptID) + validated.operations = append(validated.operations, repairOperations(validationInput, repairs)...) } setErr(err) validationResults <- sectionValidationResult{ @@ -646,6 +649,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) } type validateSectionCandidatesInput struct { + Section *validators.SectionLookup RepairProposalIndex *int RepairAttempt *int Exchange *structuredoutput.Exchange @@ -677,8 +681,15 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida filtered := make([]outcomes.FilteredProposal, 0) eligible := append([]proposals.EnrichedCorrectionProposal(nil), input.SectionEnriched...) feedback := input.Spec.Settings.LLM.CorrectionRepairAttempts > 0 - blocked := make(map[int]bool) - noEffect := make(map[int]bool) + var blocked, noEffect map[int]bool + evidence := contracts.ValidationRequest{WorkingTranscript: input.WorkingTranscript, Section: input.Section, ReplacementPolicy: input.Policy, IndependentEvidence: feedback} + if feedback { + blocked, noEffect = make(map[int]bool), make(map[int]bool) + evidence.Previews = make(map[int]proposals.SegmentPreviewResult) + if evidence.Section == nil && len(input.SectionEnriched) > 0 { + evidence.Section = validators.NewSectionLookup(input.WorkingTranscript) + } + } var interactionWriter *llmDiagnosticsWriterAdapter for validatorOrder, validator := range input.Validators { @@ -701,11 +712,12 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida submitted := eligible if feedback { - submitted = feedbackCandidates(input, validator, noEffect) + submitted = feedbackCandidates(input, validator, noEffect, evidence) } vResult, err := validator.Validate(ctx, contracts.ValidationRequest{ StageIndex: &input.Spec.Index, StageID: input.Spec.ID, - WorkingTranscript: input.WorkingTranscript, + WorkingTranscript: input.WorkingTranscript, + Section: evidence.Section, Previews: evidence.Previews, CandidateProposal: submitted, ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, @@ -743,7 +755,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida for _, d := range vResult.Decisions { if feedback && validator.Name() == "original_text_presence" && d.Approved { - preview := feedbackPreview(input, byIndex[d.ProposalIndex]) + preview := evidence.Preview(byIndex[d.ProposalIndex]) if preview.SkipReason == proposals.SkipReasonAmbiguousOriginal { d.Approved = false d.Disposition = validators.DispositionRejected @@ -751,7 +763,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida d.Message = "original_text matches multiple locations in the target segment" } } - if !d.Approved { + if feedback && !d.Approved { blocked[d.ProposalIndex] = true if d.ReasonCode == validators.ReasonNoEffect { noEffect[d.ProposalIndex] = true diff --git a/internal/framework/validators/deterministic.go b/internal/framework/validators/deterministic.go index 97addf3..3ca10f4 100644 --- a/internal/framework/validators/deterministic.go +++ b/internal/framework/validators/deterministic.go @@ -3,11 +3,9 @@ package validators import ( "context" "fmt" - "math" "strings" "gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog" - "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" ) @@ -60,21 +58,16 @@ type OriginalTextPresenceValidator struct{} func (v OriginalTextPresenceValidator) Name() string { return "original_text_presence" } func (v OriginalTextPresenceValidator) Validate(_ context.Context, req Request) (Result, error) { - byID := make(map[int]string) - if req.WorkingTranscript != nil { - for _, seg := range req.WorkingTranscript.Segments { - byID[seg.ID] = seg.Text - } - } + section := req.sectionLookup() decisions := make([]Decision, 0, len(req.CandidateProposal)) for _, c := range req.CandidateProposal { - text, ok := byID[c.TargetSegmentID] + position, ok := section.Position(c.TargetSegmentID) if !ok { decisions = append(decisions, rejection(c.ProposalIndex, ReasonMissingTargetSegment, "target segment was not found")) continue } - if !strings.Contains(text, c.OriginalText) { + if !strings.Contains(section.segments[position].Text, c.OriginalText) { decisions = append(decisions, rejection(c.ProposalIndex, ReasonMissingOriginalText, "original_text was not found in current segment text")) continue } @@ -91,21 +84,11 @@ type NonEmptyResultingSegmentValidator struct{} func (v NonEmptyResultingSegmentValidator) Name() string { return "non_empty_resulting_segment" } func (v NonEmptyResultingSegmentValidator) Validate(_ context.Context, req Request) (Result, error) { - segmentsByID := make(map[int]schema.Segment) - if req.WorkingTranscript != nil { - for _, seg := range req.WorkingTranscript.Segments { - segmentsByID[seg.ID] = seg - } - } + req.Section = req.sectionLookup() decisions := make([]Decision, 0, len(req.CandidateProposal)) for _, c := range req.CandidateProposal { - segment, ok := segmentsByID[c.TargetSegmentID] - if !ok { - decisions = append(decisions, approval(c.ProposalIndex)) - continue - } - preview := previewForEvidence(req, &segment, c.CorrectionProposal) + preview := req.Preview(c) if preview.SkipReason == proposals.SkipReasonEmptyResultingText { decisions = append(decisions, rejection(c.ProposalIndex, ReasonEmptyResultingText, "proposal would leave the segment empty")) continue @@ -149,13 +132,11 @@ func ValidateProtectedTerms(req Request) (Result, error) { for _, c := range req.CandidateProposal { reason := vocab.violationReason(c.OriginalText, c.CorrectedText) repetitionCleanup := reason != "" && req.ModuleKey == modulecatalog.KeySpokenWord - if vocab.pattern != nil && (glossaryStage || repetitionCleanup) && req.WorkingTranscript != nil { - for i := range req.WorkingTranscript.Segments { - segment := &req.WorkingTranscript.Segments[i] - if segment.ID != c.TargetSegmentID { - continue - } - preview := previewForEvidence(req, segment, c.CorrectionProposal) + if vocab.pattern != nil && (glossaryStage || repetitionCleanup) { + req.Section = req.sectionLookup() + if position, ok := req.Section.Position(c.TargetSegmentID); ok { + segment := &req.Section.segments[position] + preview := req.Preview(c) if preview.Applicable || (req.IndependentEvidence && preview.SkipReason == proposals.SkipReasonEmptyResultingText) { if glossaryStage { // A correction span may sit inside a longer exempt name. @@ -164,7 +145,6 @@ func ValidateProtectedTerms(req Request) (Result, error) { reason = "" } } - break } } if reason != "" { @@ -178,10 +158,3 @@ func ValidateProtectedTerms(req Request) (Result, error) { } return Result{ValidatorName: "protected_terms", Decisions: decisions}, nil } - -func previewForEvidence(req Request, segment *schema.Segment, correction proposals.CorrectionProposal) proposals.SegmentPreviewResult { - if req.IndependentEvidence && (math.IsNaN(correction.Confidence) || correction.Confidence < 0 || correction.Confidence > 1) { - correction.Confidence = 0.5 - } - return proposals.PreviewProposalForSegment(segment, correction, req.ReplacementPolicy) -} diff --git a/internal/framework/validators/llm_validators.go b/internal/framework/validators/llm_validators.go index bb039ca..20722dc 100644 --- a/internal/framework/validators/llm_validators.go +++ b/internal/framework/validators/llm_validators.go @@ -9,7 +9,6 @@ import ( "gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking" "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" - "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/stagename" @@ -243,30 +242,24 @@ func errPayload(err error) any { func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorType, req Request) (LLMValidationRequest, []Decision) { items := make([]LLMValidationItem, 0, len(req.CandidateProposal)) immediate := make([]Decision, 0) - var segments []schema.Segment - positions := make(map[int]int) + req.Section = req.sectionLookup() + segments := req.Section.segments var glossary glossaryContext if req.GlossaryView != nil { glossary = req.GlossaryView.context } else { glossary = prepareGlossaryContext(req.Glossary) } - if req.WorkingTranscript != nil { - segments = req.WorkingTranscript.Segments - for i, seg := range segments { - positions[seg.ID] = i - } - } for _, p := range req.CandidateProposal { - position, ok := positions[p.TargetSegmentID] + position, ok := req.Section.Position(p.TargetSegmentID) if !ok { immediate = append(immediate, rejection(p.ProposalIndex, ReasonMissingTargetSegment, "target segment was not found")) continue } seg := segments[position] - preview := previewForEvidence(req, &seg, p.CorrectionProposal) + preview := req.Preview(p) if !preview.Applicable { immediate = append(immediate, rejection(p.ProposalIndex, string(preview.SkipReason), "proposal is not previewable for LLM validation")) continue diff --git a/internal/framework/validators/models.go b/internal/framework/validators/models.go index 2022208..7e837c0 100644 --- a/internal/framework/validators/models.go +++ b/internal/framework/validators/models.go @@ -41,6 +41,8 @@ type Request struct { RepairAttempt *int `json:"repair_attempt,omitempty"` SectionIndex *int `json:"section_index,omitempty"` WorkingTranscript *schema.Transcript `json:"-"` + Section *SectionLookup `json:"-"` + Previews map[int]proposals.SegmentPreviewResult `json:"-"` CandidateProposal []proposals.EnrichedCorrectionProposal `json:"candidate_proposals"` ModuleKey string `json:"module_key"` ModuleInstance string `json:"module_instance"` diff --git a/internal/framework/validators/section.go b/internal/framework/validators/section.go new file mode 100644 index 0000000..bd7ed45 --- /dev/null +++ b/internal/framework/validators/section.go @@ -0,0 +1,61 @@ +package validators + +import ( + "math" + + "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" +) + +// SectionLookup indexes the immutable source section shared by validation +// and revalidation. Application checks its own, changing transcript. +type SectionLookup struct { + segments []schema.Segment + positions map[int]int +} + +func NewSectionLookup(transcript *schema.Transcript) *SectionLookup { + section := &SectionLookup{} + if transcript != nil { + section.segments = transcript.Segments + section.positions = make(map[int]int, len(section.segments)) + for i, segment := range section.segments { + section.positions[segment.ID] = i + } + } + return section +} + +func (s *SectionLookup) Position(id int) (int, bool) { + position, ok := s.positions[id] + return position, ok +} + +func (r Request) sectionLookup() *SectionLookup { + if r.Section != nil { + return r.Section + } + return NewSectionLookup(r.WorkingTranscript) +} + +// Preview reuses the replacement preview for this proposal index within one +// validation pass. A revision starts a new pass with its own preview map. +func (r Request) Preview(candidate proposals.EnrichedCorrectionProposal) proposals.SegmentPreviewResult { + if preview, ok := r.Previews[candidate.ProposalIndex]; ok { + return preview + } + section := r.sectionLookup() + var segment *schema.Segment + if position, ok := section.Position(candidate.TargetSegmentID); ok { + segment = §ion.segments[position] + } + correction := candidate.CorrectionProposal + if r.IndependentEvidence && (math.IsNaN(correction.Confidence) || correction.Confidence < 0 || correction.Confidence > 1) { + correction.Confidence = 0.5 + } + preview := proposals.PreviewProposalForSegment(segment, correction, r.ReplacementPolicy) + if r.Previews != nil { + r.Previews[candidate.ProposalIndex] = preview + } + return preview +}