Attribute filtered revisions to their repair attempts
This commit is contained in:
@@ -34,11 +34,12 @@ type repairWireRequest struct {
|
||||
} `json:"response_format"`
|
||||
}
|
||||
type correctionProvider struct {
|
||||
mu sync.Mutex
|
||||
requests []repairWireRequest
|
||||
attempts map[int]int
|
||||
entered chan struct{}
|
||||
wait bool
|
||||
mu sync.Mutex
|
||||
requests []repairWireRequest
|
||||
attempts map[int]int
|
||||
entered chan struct{}
|
||||
wait bool
|
||||
invalidFirstRevision bool
|
||||
}
|
||||
|
||||
func (p *correctionProvider) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -92,6 +93,7 @@ func (p *correctionProvider) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
p.mu.Lock()
|
||||
p.attempts[ref]++
|
||||
attempt := p.attempts[ref]
|
||||
invalid := p.invalidFirstRevision && attempt == 1
|
||||
p.mu.Unlock()
|
||||
if ref == 3 {
|
||||
http.Error(w, `{"error":{"message":"context too long"}}`, http.StatusBadRequest)
|
||||
@@ -101,12 +103,50 @@ func (p *correctionProvider) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if attempt > 1 {
|
||||
confidence = 0.99
|
||||
}
|
||||
content = fmt.Sprintf(`{"proposal_ref":2,"action":"revise","correction":{"id":2,"original_text":"second","corrected_text":"Second","confidence":%v}}`, confidence)
|
||||
id := 2
|
||||
if invalid {
|
||||
id, confidence = 99, 0.99
|
||||
}
|
||||
content = fmt.Sprintf(`{"proposal_ref":2,"action":"revise","correction":{"id":%d,"original_text":"second","corrected_text":"Second","confidence":%v}}`, id, confidence)
|
||||
}
|
||||
raw, _ := json.Marshal(content)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
fmt.Fprintf(w, `{"model":"fixture","choices":[{"message":{"content":%s}}]}`, raw)
|
||||
}
|
||||
|
||||
func TestSubprocessReportsDefectOnOriginatingRepair(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
budget int
|
||||
outcome string
|
||||
recoveries int
|
||||
}{{1, "exhausted", 1}, {2, "applied", 2}} {
|
||||
t.Run(tc.outcome, func(t *testing.T) {
|
||||
f, provider := setupCorrectionFixture(t, tc.budget, false, false)
|
||||
provider.mu.Lock()
|
||||
provider.invalidFirstRevision = true
|
||||
provider.mu.Unlock()
|
||||
if code, _, stderr := f.run(t, true); code != 0 {
|
||||
t.Fatalf("code=%d stderr=%s", code, stderr)
|
||||
}
|
||||
doc := f.recoveries(t)
|
||||
if *doc.Summary.ProposalResponses != 1 || *doc.Summary.ProposalResponsesWithFiltering != 0 || len(doc.Recoveries) != tc.recoveries {
|
||||
t.Fatalf("unexpected recoveries: %+v", doc)
|
||||
}
|
||||
first := doc.Recoveries[0]
|
||||
want := []reporting.FilteredProposal{{ProposalIndex: 1, Reason: "invalid_target_segment_id", Count: 1}}
|
||||
if first.Operation != "correction_repair" || !reflect.DeepEqual(first.FilteredProposals, want) || !reflect.DeepEqual(first.Reasons, []string{"proposals_filtered"}) {
|
||||
t.Fatalf("first repair recovery: %+v", first)
|
||||
}
|
||||
if tc.recoveries == 2 && !reflect.DeepEqual(doc.Recoveries[1].Reasons, []string{"correction_revised"}) {
|
||||
t.Fatalf("successful repair recovery: %+v", doc.Recoveries[1])
|
||||
}
|
||||
report, err := os.ReadFile(filepath.Join(f.dir, "report.md"))
|
||||
if err != nil || !strings.Contains(string(report), fmt.Sprintf("| 1 | 1 | 1 | %d | %s |", tc.budget, tc.outcome)) {
|
||||
t.Fatalf("repair outcome: %s %v", report, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
func setupCorrectionFixture(t *testing.T, attempts int, replacement, wait bool) (processFixture, *correctionProvider) {
|
||||
t.Helper()
|
||||
f := newProcessFixture(t, "normal", "[grammar]")
|
||||
|
||||
@@ -65,7 +65,9 @@ Each recovery identifies its work unit, optional `stage_id`, and successful oper
|
||||
`service_recovered`, `correction_revised`, and `proposals_filtered` together.
|
||||
`correction_revised` means a repair yielded a fully validated revision; its
|
||||
application outcome is recorded in the Markdown report. Filtering covers
|
||||
defective response proposals; low confidence, protected-term rejection,
|
||||
defective proposals and revisions under their originating operation. Only
|
||||
original proposal operations count toward `proposal_responses_with_filtering`.
|
||||
Low confidence, protected-term rejection,
|
||||
ordinary editorial rejection, and an apply-time conflict caused by an earlier
|
||||
accepted correction are not filtering recoveries. Failed operations are errors,
|
||||
not recoveries. The initial chunk identity remains stable even if earlier
|
||||
|
||||
@@ -2,9 +2,8 @@
|
||||
|
||||
**Updated:** 2026-11-11
|
||||
|
||||
**Status:** Nine open findings against implementation revision `aa8073c`. The
|
||||
algorithmic audit expanded R4 and added R5–R9. See the
|
||||
[implementation plan](implementation.md).
|
||||
**Status:** 1 of nine findings resolved. Findings refer to implementation revision
|
||||
`aa8073c`. See the [implementation plan](implementation.md).
|
||||
|
||||
## Review Result
|
||||
|
||||
@@ -34,6 +33,8 @@ replaced by a new local context-size restriction.
|
||||
|
||||
### R1 — Preserve Defect Attribution for Revised Proposals (P2)
|
||||
|
||||
**Completed.** Revision defects now appear on the originating repair recovery. Subprocess tests cover exhaustion, later approval, and ordinary rejection.
|
||||
|
||||
**Affected plan:** Stage 6, item 5; Stage 7, items 1–2.
|
||||
|
||||
**Evidence:** In [repairCandidates](../../internal/framework/runner/repair.go),
|
||||
|
||||
@@ -28,6 +28,7 @@ type RepairAttempt struct {
|
||||
Succeeded bool
|
||||
Approved bool
|
||||
Operations []outcomes.Operation
|
||||
Filtered []outcomes.FilteredProposal
|
||||
}
|
||||
|
||||
// RepairOutcome keeps the proposal identity through revision and application.
|
||||
@@ -180,6 +181,11 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
||||
record.Revision = &revisionSnapshot
|
||||
record.Decisions = validated.decisions
|
||||
record.Rejected = validated.rejected
|
||||
if len(validated.filtered) > 0 {
|
||||
// A repair response contains one proposal. Attribute its first
|
||||
// defect once, as for proposals in the original response.
|
||||
record.Filtered = validated.filtered[:1]
|
||||
}
|
||||
for i := range validated.operations {
|
||||
proposalIndex, attemptNumber := original.ProposalIndex, attempt+1
|
||||
validated.operations[i].Reference.ProposalIndex = &proposalIndex
|
||||
@@ -239,6 +245,10 @@ func repairOperations(input validateSectionCandidatesInput, branches []RepairOut
|
||||
operation := outcomes.Operation{Reference: outcomes.Reference{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: section, Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &index, RepairAttempt: &number}, Counts: attempt.Counts, Succeeded: attempt.Succeeded, FailureReason: attempt.FailureReason, ProposalIndices: []int{index}}
|
||||
if attempt.Succeeded {
|
||||
operation.RecoveryReasons = outcomes.ExecutionRecoveries(attempt.Counts)
|
||||
if len(attempt.Filtered) > 0 {
|
||||
operation.Filtered = attempt.Filtered
|
||||
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryProposalsFiltered)
|
||||
}
|
||||
if attempt.Approved {
|
||||
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryCorrectionRevised)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user