From cc0a420722dcc7b479285e81d950d62218c17a59 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Tue, 17 Nov 2026 04:34:31 +0000 Subject: [PATCH] Reuse validation facts when attributing skipped edits --- docs/internal/pipeline.md | 4 +- docs/roadmap/remediation.md | 7 ++- internal/framework/proposals/proposal.go | 11 ++++ internal/framework/runner/repair.go | 3 +- internal/framework/runner/repair_test.go | 43 +++++++++++++++ internal/framework/runner/runner.go | 68 ++++++++++++++++-------- internal/framework/runner/runner_test.go | 18 +++++-- internal/framework/runner/work_totals.go | 29 +++++++--- internal/framework/validators/models.go | 41 +++++++------- internal/framework/validators/section.go | 3 ++ 10 files changed, 172 insertions(+), 55 deletions(-) diff --git a/docs/internal/pipeline.md b/docs/internal/pipeline.md index 8f89335..ec14ab2 100644 --- a/docs/internal/pipeline.md +++ b/docs/internal/pipeline.md @@ -33,7 +33,9 @@ formats; [configuration](../config.md) owns settings; the without affecting independent approvals. 7. Only fully approved proposals reach application. Corrections apply in proposal-index order within their owning section. Existing replacement - policies and apply-time safety checks resolve stale conflicts. + policies and apply-time safety checks resolve stale conflicts. Skip + attribution uses available validation previews of the original text to + distinguish stale edits from defects in a proposal or repair response. 8. CLI encodes the final transcript and publishes active files through the output session. `framework/processreport` builds one snapshot for YAML, Markdown, and CLI totals. Debug evidence is captured separately when enabled. diff --git a/docs/roadmap/remediation.md b/docs/roadmap/remediation.md index 5b86136..07b771d 100644 --- a/docs/roadmap/remediation.md +++ b/docs/roadmap/remediation.md @@ -2,7 +2,7 @@ **Reviewed revision:** `e8c0708` -**Status:** Findings and remediation progress from review of the 20 +**Status:** All fourteen findings addressed after review of the 20 [implementation stages](implementation.md) against the [target roadmap](reliability-and-maintainability.md), including follow-up audits of component interactions and algorithmic costs. @@ -472,6 +472,11 @@ both initial and repair filtering, and an interrupted run with available origin. ### R14 — Reuse Original Applicability When Attributing Application Skips (P3) +**Complete.** Application still checks current text. Skip attribution +reuses known original applicability, and repair filtering indexes approved +operations once per affected unit. Tests distinguish stale conflicts from +original defects and verify attribution across multiple repairs. + **Affected plan:** Stages 8 and 17, item 3. **Evidence:** [Application](../../internal/framework/proposals/apply.go#L59) diff --git a/internal/framework/proposals/proposal.go b/internal/framework/proposals/proposal.go index 91c8537..7dd7b9f 100644 --- a/internal/framework/proposals/proposal.go +++ b/internal/framework/proposals/proposal.go @@ -26,8 +26,19 @@ type ProposalMetadata struct { type EnrichedCorrectionProposal struct { CorrectionProposal ProposalMetadata + OriginalApplicability Applicability `json:"-"` } +// Applicability records whether validation previewed a proposal against the +// original section and could apply it there. +type Applicability uint8 + +const ( + ApplicabilityUnknown Applicability = iota + Applicable + Inapplicable +) + // Validate checks only basic structural integrity of the proposal. // Semantic checks (staleness, span presence, replacement behavior) are handled // by proposal preview/application and validator execution. diff --git a/internal/framework/runner/repair.go b/internal/framework/runner/repair.go index f70a3fe..b1259b6 100644 --- a/internal/framework/runner/repair.go +++ b/internal/framework/runner/repair.go @@ -283,7 +283,8 @@ func runRepairBranch(ctx context.Context, branchInput repairBranchInput) (repair } if approved { branch.Outcome = "approved" - result.approved = ¤t + revision := validated.approved[0] + result.approved = &revision return nil, nil } if actionable(original.ProposalIndex, validated.rejected) { diff --git a/internal/framework/runner/repair_test.go b/internal/framework/runner/repair_test.go index a204363..f5e125b 100644 --- a/internal/framework/runner/repair_test.go +++ b/internal/framework/runner/repair_test.go @@ -260,6 +260,49 @@ func TestRunnerAppliesRevisionsInOriginalOrder(t *testing.T) { } } +func TestRunnerAttributesApplicationFilteringToEachRepair(t *testing.T) { + client := &repairClient{t: t, outputs: []string{ + `{"proposal_ref":1,"decision":{"action":"revise","correction":{"id":1,"original_text":"missing","corrected_text":"new"}}}`, + `{"proposal_ref":2,"decision":{"action":"revise","correction":{"id":1,"original_text":"absent","corrected_text":"new"}}}`, + }} + module := repairModule{outcomeModule{ + fakeModule: fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, + validators: []validators.Validator{rejectUntilRepair()}}, + proposeResult: func(contracts.ProposalRequest) (contracts.ProposalResult, error) { + return contracts.ProposalResult{ + Proposals: []proposals.CorrectionProposal{ + {TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, + {TargetSegmentID: 1, OriginalText: "bird", CorrectedText: "owl"}, + }, + SegmentReferences: []int{1, 1}, + Exchange: &structuredoutput.Exchange{}, + Operations: []outcomes.Operation{{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true}}, + }, nil + }, + }} + r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}}) + out, err := r.Run(context.Background(), RunInput{ + Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat and bird"}}}, + ModuleSpecs: []contracts.ModuleRunSpec{{ + ModuleKey: "grammar", InstanceName: "grammar", RepairCoordinator: client, + Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}, + }}, + }) + if err != nil { + t.Fatal(err) + } + operations := out.WorkUnits[0].Operations + if out.Totals.Corrections != 0 || out.Totals.FilteringResponses != 0 || out.Totals.Recoveries != 2 || len(operations) != 2 { + t.Fatalf("unexpected filtering totals or reports: %+v, %+v", out.Totals, operations) + } + for i, operation := range operations { + if operation.Reference.Kind != outcomes.OperationCorrectionRepair || len(operation.Filtered) != 1 || + operation.Filtered[0].ProposalIndex != i || operation.Filtered[0].Reason != outcomes.ReasonCode(proposals.SkipReasonMissingOriginalText) { + t.Fatalf("filtering reported against the wrong repair: %+v", operation) + } + } +} + func TestRepairRevalidatesEarlierApprovalAndRetainsUnrelatedApprovalOnFailure(t *testing.T) { candidates := []proposals.EnrichedCorrectionProposal{ {CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}, diff --git a/internal/framework/runner/runner.go b/internal/framework/runner/runner.go index 9b08e61..ef4cf08 100644 --- a/internal/framework/runner/runner.go +++ b/internal/framework/runner/runner.go @@ -762,13 +762,19 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida feedback := input.Spec.Settings.LLM.CorrectionRepairAttempts > 0 var blocked, noEffect map[int]bool evidence := validators.Request{WorkingTranscript: input.WorkingTranscript, Section: input.Section, ReplacementPolicy: input.Policy, IndependentEvidence: feedback} - if feedback { - blocked, noEffect = make(map[int]bool), make(map[int]bool) - evidence.Previews = make(map[int]proposals.SegmentPreviewResult) - if evidence.Section == nil && len(input.SectionEnriched) > 0 { + if len(input.SectionEnriched) > 0 && len(input.Validators) > 0 { + if feedback { + evidence.Previews = make(map[int]proposals.SegmentPreviewResult) + } else { + evidence.PreviewApplicability = make(map[int]bool) + } + if evidence.Section == nil { evidence.Section = validators.NewSectionLookup(input.WorkingTranscript) } } + if feedback { + blocked, noEffect = make(map[int]bool), make(map[int]bool) + } for validatorOrder, validator := range input.Validators { if err := ctx.Err(); err != nil { @@ -782,7 +788,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida vResult, err := validator.Validate(ctx, validators.Request{ StageIndex: &input.Spec.Index, StageID: input.Spec.ID, WorkingTranscript: input.WorkingTranscript, - Section: evidence.Section, Previews: evidence.Previews, + Section: evidence.Section, Previews: evidence.Previews, PreviewApplicability: evidence.PreviewApplicability, CandidateProposal: submitted, ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, @@ -899,6 +905,19 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida } } } + for i := range eligible { + index := eligible[i].ProposalIndex + applicable, checked := evidence.PreviewApplicability[index] + if preview, ok := evidence.Previews[index]; ok { + applicable, checked = preview.Applicable, true + } + if checked { + eligible[i].OriginalApplicability = proposals.Inapplicable + if applicable { + eligible[i].OriginalApplicability = proposals.Applicable + } + } + } return validateSectionCandidatesResult{ approved: eligible, decisions: decisions, @@ -947,14 +966,19 @@ func findApplyFiltering(sections []chunking.Section, approved []proposals.Enrich if len(skipped) == 0 { return nil } - byIndex := make(map[int]proposals.EnrichedCorrectionProposal, len(approved)) + byIndex := make(map[int]proposals.EnrichedCorrectionProposal) + for _, proposal := range approved { + if proposal.OriginalApplicability != proposals.Applicable { + byIndex[proposal.ProposalIndex] = proposal + } + } + if len(byIndex) == 0 { + return nil + } sectionPosition := make(map[int]int, len(sections)) for pos, section := range sections { sectionPosition[section.Index] = pos } - for _, proposal := range approved { - byIndex[proposal.ProposalIndex] = proposal - } lookups := make(map[int]*validators.SectionLookup) filtered := make(map[int][]outcomes.FilteredProposal) for _, skip := range skipped { @@ -967,18 +991,20 @@ func findApplyFiltering(sections []chunking.Section, approved []proposals.Enrich if !ok { continue } - lookup := lookups[pos] - if lookup == nil { - lookup = validators.NewSectionLookup(transcriptFromSection(sections[pos])) - lookups[pos] = lookup - } - var segment *schema.Segment - if i, found := lookup.Position(proposal.TargetSegmentID); found { - segment = §ions[pos].Segments[i] - } - // A previously applicable proposal can become stale during application. - if proposals.PreviewProposalForSegment(segment, proposal.CorrectionProposal, policy).Applicable { - continue + if proposal.OriginalApplicability == proposals.ApplicabilityUnknown { + lookup := lookups[pos] + if lookup == nil { + lookup = validators.NewSectionLookup(&schema.Transcript{Segments: sections[pos].Segments}) + lookups[pos] = lookup + } + var segment *schema.Segment + if i, found := lookup.Position(proposal.TargetSegmentID); found { + segment = §ions[pos].Segments[i] + } + // A previously applicable proposal can become stale during application. + if proposals.PreviewProposalForSegment(segment, proposal.CorrectionProposal, policy).Applicable { + continue + } } filtered[pos] = append(filtered[pos], outcomes.FilteredProposal{ProposalIndex: skip.ProposalIndex, Reason: reason, Count: 1}) } diff --git a/internal/framework/runner/runner_test.go b/internal/framework/runner/runner_test.go index 23f5179..11099f2 100644 --- a/internal/framework/runner/runner_test.go +++ b/internal/framework/runner/runner_test.go @@ -733,9 +733,15 @@ func TestRunnerApplicationSkipAfterValidatorApprovalReported(t *testing.T) { return validators.Result{ValidatorName: "allow", Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: true, ReasonCode: validators.ReasonApproved, Message: "ok"}}}, nil }} r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ - "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{allow}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { - return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "word", CorrectedText: "term"}}, nil - }}, + "m": outcomeModule{ + fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{allow}}, + proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) { + return contracts.ProposalResult{ + Proposals: []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "word", CorrectedText: "term"}}, + Operations: []outcomes.Operation{{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true}}, + }, nil + }, + }, }}) out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}}) if err != nil { @@ -747,6 +753,12 @@ func TestRunnerApplicationSkipAfterValidatorApprovalReported(t *testing.T) { if len(out.ModuleResults[0].ValidatorRejected) != 0 { t.Fatalf("expected no validator rejection") } + operations := out.WorkUnits[0].Operations + if out.Totals.Corrections != 0 || out.Totals.FilteringResponses != 1 || out.Totals.Recoveries != 1 || + len(operations) != 1 || len(operations[0].Filtered) != 1 || + operations[0].Filtered[0].Reason != outcomes.ReasonCode(proposals.SkipReasonAmbiguousOriginal) { + t.Fatalf("original defect lost during application: totals=%+v operations=%+v", out.Totals, operations) + } } func TestRunnerApplyConflictAfterTwoInitiallyValidDeletions(t *testing.T) { diff --git a/internal/framework/runner/work_totals.go b/internal/framework/runner/work_totals.go index 68a23b9..6473134 100644 --- a/internal/framework/runner/work_totals.go +++ b/internal/framework/runner/work_totals.go @@ -63,20 +63,18 @@ func (p *modulePipelineResult) rememberProposalRecovery(unit *WorkUnitOutcome) { func (p *modulePipelineResult) recordApplyFiltering(filters map[int][]outcomes.FilteredProposal, reports bool) { for pos, items := range filters { original := items[:0] + var repairOperations map[int]*outcomes.Operation for _, item := range items { if _, repaired := p.revised[item.ProposalIndex]; !repaired { original = append(original, item) continue } if reports { - operations := p.WorkUnits[pos].Operations - for i := range operations { - op := &operations[i] - if op.Reference.Kind == outcomes.OperationCorrectionRepair && op.Reference.ProposalIndex != nil && - *op.Reference.ProposalIndex == item.ProposalIndex && slices.Contains(op.RecoveryReasons, outcomes.RecoveryCorrectionRevised) { - mergeOperationFiltering(op, []outcomes.FilteredProposal{item}) - break - } + if repairOperations == nil { + repairOperations = approvedRepairOperations(p.WorkUnits[pos].Operations) + } + if op := repairOperations[item.ProposalIndex]; op != nil { + mergeOperationFiltering(op, []outcomes.FilteredProposal{item}) } } } @@ -93,3 +91,18 @@ func (p *modulePipelineResult) recordApplyFiltering(filters map[int][]outcomes.F } } } + +func approvedRepairOperations(operations []outcomes.Operation) map[int]*outcomes.Operation { + byProposal := make(map[int]*outcomes.Operation) + for i := range operations { + op := &operations[i] + if op.Reference.Kind == outcomes.OperationCorrectionRepair && op.Reference.ProposalIndex != nil && + slices.Contains(op.RecoveryReasons, outcomes.RecoveryCorrectionRevised) { + index := *op.Reference.ProposalIndex + if byProposal[index] == nil { + byProposal[index] = op + } + } + } + return byProposal +} diff --git a/internal/framework/validators/models.go b/internal/framework/validators/models.go index 93b0681..9c36a76 100644 --- a/internal/framework/validators/models.go +++ b/internal/framework/validators/models.go @@ -30,26 +30,27 @@ const ( // Request is the runtime input shared by deterministic validators. type Request struct { - StageIndex *int `json:"stage_index,omitempty"` - StageID *string `json:"stage_id,omitempty"` - IndependentEvidence bool `json:"-"` - RepairProposalIndex *int `json:"repair_proposal_index,omitempty"` - RepairAttempt *int `json:"repair_attempt,omitempty"` - SectionIndex *int `json:"section_index,omitempty"` - WorkingTranscript *schema.Transcript `json:"-"` - Section *SectionLookup `json:"-"` - Previews map[int]proposals.SegmentPreviewResult `json:"-"` - CandidateProposal []proposals.EnrichedCorrectionProposal `json:"candidate_proposals"` - ModuleKey string `json:"module_key"` - ModuleInstance string `json:"module_instance"` - ValidatorOrder *int `json:"validator_order,omitempty"` - ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"` - Glossary []glossary.Entry `json:"-"` - ProtectedTerms []glossary.Term `json:"-"` - GlossaryView *GlossaryView `json:"-"` - Settings config.Stage `json:"-"` - LLMClient StructuredLLMClient `json:"-"` - Interactions structuredoutput.InteractionSink `json:"-"` + StageIndex *int `json:"stage_index,omitempty"` + StageID *string `json:"stage_id,omitempty"` + IndependentEvidence bool `json:"-"` + RepairProposalIndex *int `json:"repair_proposal_index,omitempty"` + RepairAttempt *int `json:"repair_attempt,omitempty"` + SectionIndex *int `json:"section_index,omitempty"` + WorkingTranscript *schema.Transcript `json:"-"` + Section *SectionLookup `json:"-"` + Previews map[int]proposals.SegmentPreviewResult `json:"-"` + PreviewApplicability map[int]bool `json:"-"` + CandidateProposal []proposals.EnrichedCorrectionProposal `json:"candidate_proposals"` + ModuleKey string `json:"module_key"` + ModuleInstance string `json:"module_instance"` + ValidatorOrder *int `json:"validator_order,omitempty"` + ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"` + Glossary []glossary.Entry `json:"-"` + ProtectedTerms []glossary.Term `json:"-"` + GlossaryView *GlossaryView `json:"-"` + Settings config.Stage `json:"-"` + LLMClient StructuredLLMClient `json:"-"` + Interactions structuredoutput.InteractionSink `json:"-"` } // Decision is one validator decision for one proposal index. diff --git a/internal/framework/validators/section.go b/internal/framework/validators/section.go index 34e453d..7de0367 100644 --- a/internal/framework/validators/section.go +++ b/internal/framework/validators/section.go @@ -52,6 +52,9 @@ func (r Request) Preview(candidate proposals.EnrichedCorrectionProposal) proposa if r.Previews != nil { r.Previews[candidate.ProposalIndex] = preview } + if r.PreviewApplicability != nil { + r.PreviewApplicability[candidate.ProposalIndex] = preview.Applicable + } return preview }