Reuse validation facts when attributing skipped edits
This commit is contained in:
@@ -26,8 +26,19 @@ type ProposalMetadata struct {
|
||||
type EnrichedCorrectionProposal struct {
|
||||
CorrectionProposal
|
||||
ProposalMetadata
|
||||
OriginalApplicability Applicability `json:"-"`
|
||||
}
|
||||
|
||||
// Applicability records whether validation previewed a proposal against the
|
||||
// original section and could apply it there.
|
||||
type Applicability uint8
|
||||
|
||||
const (
|
||||
ApplicabilityUnknown Applicability = iota
|
||||
Applicable
|
||||
Inapplicable
|
||||
)
|
||||
|
||||
// Validate checks only basic structural integrity of the proposal.
|
||||
// Semantic checks (staleness, span presence, replacement behavior) are handled
|
||||
// by proposal preview/application and validator execution.
|
||||
|
||||
@@ -283,7 +283,8 @@ func runRepairBranch(ctx context.Context, branchInput repairBranchInput) (repair
|
||||
}
|
||||
if approved {
|
||||
branch.Outcome = "approved"
|
||||
result.approved = ¤t
|
||||
revision := validated.approved[0]
|
||||
result.approved = &revision
|
||||
return nil, nil
|
||||
}
|
||||
if actionable(original.ProposalIndex, validated.rejected) {
|
||||
|
||||
@@ -260,6 +260,49 @@ func TestRunnerAppliesRevisionsInOriginalOrder(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerAttributesApplicationFilteringToEachRepair(t *testing.T) {
|
||||
client := &repairClient{t: t, outputs: []string{
|
||||
`{"proposal_ref":1,"decision":{"action":"revise","correction":{"id":1,"original_text":"missing","corrected_text":"new"}}}`,
|
||||
`{"proposal_ref":2,"decision":{"action":"revise","correction":{"id":1,"original_text":"absent","corrected_text":"new"}}}`,
|
||||
}}
|
||||
module := repairModule{outcomeModule{
|
||||
fakeModule: fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique,
|
||||
validators: []validators.Validator{rejectUntilRepair()}},
|
||||
proposeResult: func(contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
return contracts.ProposalResult{
|
||||
Proposals: []proposals.CorrectionProposal{
|
||||
{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"},
|
||||
{TargetSegmentID: 1, OriginalText: "bird", CorrectedText: "owl"},
|
||||
},
|
||||
SegmentReferences: []int{1, 1},
|
||||
Exchange: &structuredoutput.Exchange{},
|
||||
Operations: []outcomes.Operation{{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true}},
|
||||
}, nil
|
||||
},
|
||||
}}
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}})
|
||||
out, err := r.Run(context.Background(), RunInput{
|
||||
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat and bird"}}},
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{{
|
||||
ModuleKey: "grammar", InstanceName: "grammar", RepairCoordinator: client,
|
||||
Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}},
|
||||
}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
operations := out.WorkUnits[0].Operations
|
||||
if out.Totals.Corrections != 0 || out.Totals.FilteringResponses != 0 || out.Totals.Recoveries != 2 || len(operations) != 2 {
|
||||
t.Fatalf("unexpected filtering totals or reports: %+v, %+v", out.Totals, operations)
|
||||
}
|
||||
for i, operation := range operations {
|
||||
if operation.Reference.Kind != outcomes.OperationCorrectionRepair || len(operation.Filtered) != 1 ||
|
||||
operation.Filtered[0].ProposalIndex != i || operation.Filtered[0].Reason != outcomes.ReasonCode(proposals.SkipReasonMissingOriginalText) {
|
||||
t.Fatalf("filtering reported against the wrong repair: %+v", operation)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepairRevalidatesEarlierApprovalAndRetainsUnrelatedApprovalOnFailure(t *testing.T) {
|
||||
candidates := []proposals.EnrichedCorrectionProposal{
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}},
|
||||
|
||||
@@ -762,13 +762,19 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
||||
feedback := input.Spec.Settings.LLM.CorrectionRepairAttempts > 0
|
||||
var blocked, noEffect map[int]bool
|
||||
evidence := validators.Request{WorkingTranscript: input.WorkingTranscript, Section: input.Section, ReplacementPolicy: input.Policy, IndependentEvidence: feedback}
|
||||
if feedback {
|
||||
blocked, noEffect = make(map[int]bool), make(map[int]bool)
|
||||
evidence.Previews = make(map[int]proposals.SegmentPreviewResult)
|
||||
if evidence.Section == nil && len(input.SectionEnriched) > 0 {
|
||||
if len(input.SectionEnriched) > 0 && len(input.Validators) > 0 {
|
||||
if feedback {
|
||||
evidence.Previews = make(map[int]proposals.SegmentPreviewResult)
|
||||
} else {
|
||||
evidence.PreviewApplicability = make(map[int]bool)
|
||||
}
|
||||
if evidence.Section == nil {
|
||||
evidence.Section = validators.NewSectionLookup(input.WorkingTranscript)
|
||||
}
|
||||
}
|
||||
if feedback {
|
||||
blocked, noEffect = make(map[int]bool), make(map[int]bool)
|
||||
}
|
||||
|
||||
for validatorOrder, validator := range input.Validators {
|
||||
if err := ctx.Err(); err != nil {
|
||||
@@ -782,7 +788,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
||||
vResult, err := validator.Validate(ctx, validators.Request{
|
||||
StageIndex: &input.Spec.Index, StageID: input.Spec.ID,
|
||||
WorkingTranscript: input.WorkingTranscript,
|
||||
Section: evidence.Section, Previews: evidence.Previews,
|
||||
Section: evidence.Section, Previews: evidence.Previews, PreviewApplicability: evidence.PreviewApplicability,
|
||||
CandidateProposal: submitted,
|
||||
ModuleKey: input.Spec.ModuleKey,
|
||||
ModuleInstance: input.Spec.InstanceName,
|
||||
@@ -899,6 +905,19 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
||||
}
|
||||
}
|
||||
}
|
||||
for i := range eligible {
|
||||
index := eligible[i].ProposalIndex
|
||||
applicable, checked := evidence.PreviewApplicability[index]
|
||||
if preview, ok := evidence.Previews[index]; ok {
|
||||
applicable, checked = preview.Applicable, true
|
||||
}
|
||||
if checked {
|
||||
eligible[i].OriginalApplicability = proposals.Inapplicable
|
||||
if applicable {
|
||||
eligible[i].OriginalApplicability = proposals.Applicable
|
||||
}
|
||||
}
|
||||
}
|
||||
return validateSectionCandidatesResult{
|
||||
approved: eligible,
|
||||
decisions: decisions,
|
||||
@@ -947,14 +966,19 @@ func findApplyFiltering(sections []chunking.Section, approved []proposals.Enrich
|
||||
if len(skipped) == 0 {
|
||||
return nil
|
||||
}
|
||||
byIndex := make(map[int]proposals.EnrichedCorrectionProposal, len(approved))
|
||||
byIndex := make(map[int]proposals.EnrichedCorrectionProposal)
|
||||
for _, proposal := range approved {
|
||||
if proposal.OriginalApplicability != proposals.Applicable {
|
||||
byIndex[proposal.ProposalIndex] = proposal
|
||||
}
|
||||
}
|
||||
if len(byIndex) == 0 {
|
||||
return nil
|
||||
}
|
||||
sectionPosition := make(map[int]int, len(sections))
|
||||
for pos, section := range sections {
|
||||
sectionPosition[section.Index] = pos
|
||||
}
|
||||
for _, proposal := range approved {
|
||||
byIndex[proposal.ProposalIndex] = proposal
|
||||
}
|
||||
lookups := make(map[int]*validators.SectionLookup)
|
||||
filtered := make(map[int][]outcomes.FilteredProposal)
|
||||
for _, skip := range skipped {
|
||||
@@ -967,18 +991,20 @@ func findApplyFiltering(sections []chunking.Section, approved []proposals.Enrich
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
lookup := lookups[pos]
|
||||
if lookup == nil {
|
||||
lookup = validators.NewSectionLookup(transcriptFromSection(sections[pos]))
|
||||
lookups[pos] = lookup
|
||||
}
|
||||
var segment *schema.Segment
|
||||
if i, found := lookup.Position(proposal.TargetSegmentID); found {
|
||||
segment = §ions[pos].Segments[i]
|
||||
}
|
||||
// A previously applicable proposal can become stale during application.
|
||||
if proposals.PreviewProposalForSegment(segment, proposal.CorrectionProposal, policy).Applicable {
|
||||
continue
|
||||
if proposal.OriginalApplicability == proposals.ApplicabilityUnknown {
|
||||
lookup := lookups[pos]
|
||||
if lookup == nil {
|
||||
lookup = validators.NewSectionLookup(&schema.Transcript{Segments: sections[pos].Segments})
|
||||
lookups[pos] = lookup
|
||||
}
|
||||
var segment *schema.Segment
|
||||
if i, found := lookup.Position(proposal.TargetSegmentID); found {
|
||||
segment = §ions[pos].Segments[i]
|
||||
}
|
||||
// A previously applicable proposal can become stale during application.
|
||||
if proposals.PreviewProposalForSegment(segment, proposal.CorrectionProposal, policy).Applicable {
|
||||
continue
|
||||
}
|
||||
}
|
||||
filtered[pos] = append(filtered[pos], outcomes.FilteredProposal{ProposalIndex: skip.ProposalIndex, Reason: reason, Count: 1})
|
||||
}
|
||||
|
||||
@@ -733,9 +733,15 @@ func TestRunnerApplicationSkipAfterValidatorApprovalReported(t *testing.T) {
|
||||
return validators.Result{ValidatorName: "allow", Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: true, ReasonCode: validators.ReasonApproved, Message: "ok"}}}, nil
|
||||
}}
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{allow}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "word", CorrectedText: "term"}}, nil
|
||||
}},
|
||||
"m": outcomeModule{
|
||||
fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{allow}},
|
||||
proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
return contracts.ProposalResult{
|
||||
Proposals: []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "word", CorrectedText: "term"}},
|
||||
Operations: []outcomes.Operation{{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true}},
|
||||
}, nil
|
||||
},
|
||||
},
|
||||
}})
|
||||
out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}})
|
||||
if err != nil {
|
||||
@@ -747,6 +753,12 @@ func TestRunnerApplicationSkipAfterValidatorApprovalReported(t *testing.T) {
|
||||
if len(out.ModuleResults[0].ValidatorRejected) != 0 {
|
||||
t.Fatalf("expected no validator rejection")
|
||||
}
|
||||
operations := out.WorkUnits[0].Operations
|
||||
if out.Totals.Corrections != 0 || out.Totals.FilteringResponses != 1 || out.Totals.Recoveries != 1 ||
|
||||
len(operations) != 1 || len(operations[0].Filtered) != 1 ||
|
||||
operations[0].Filtered[0].Reason != outcomes.ReasonCode(proposals.SkipReasonAmbiguousOriginal) {
|
||||
t.Fatalf("original defect lost during application: totals=%+v operations=%+v", out.Totals, operations)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerApplyConflictAfterTwoInitiallyValidDeletions(t *testing.T) {
|
||||
|
||||
@@ -63,20 +63,18 @@ func (p *modulePipelineResult) rememberProposalRecovery(unit *WorkUnitOutcome) {
|
||||
func (p *modulePipelineResult) recordApplyFiltering(filters map[int][]outcomes.FilteredProposal, reports bool) {
|
||||
for pos, items := range filters {
|
||||
original := items[:0]
|
||||
var repairOperations map[int]*outcomes.Operation
|
||||
for _, item := range items {
|
||||
if _, repaired := p.revised[item.ProposalIndex]; !repaired {
|
||||
original = append(original, item)
|
||||
continue
|
||||
}
|
||||
if reports {
|
||||
operations := p.WorkUnits[pos].Operations
|
||||
for i := range operations {
|
||||
op := &operations[i]
|
||||
if op.Reference.Kind == outcomes.OperationCorrectionRepair && op.Reference.ProposalIndex != nil &&
|
||||
*op.Reference.ProposalIndex == item.ProposalIndex && slices.Contains(op.RecoveryReasons, outcomes.RecoveryCorrectionRevised) {
|
||||
mergeOperationFiltering(op, []outcomes.FilteredProposal{item})
|
||||
break
|
||||
}
|
||||
if repairOperations == nil {
|
||||
repairOperations = approvedRepairOperations(p.WorkUnits[pos].Operations)
|
||||
}
|
||||
if op := repairOperations[item.ProposalIndex]; op != nil {
|
||||
mergeOperationFiltering(op, []outcomes.FilteredProposal{item})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -93,3 +91,18 @@ func (p *modulePipelineResult) recordApplyFiltering(filters map[int][]outcomes.F
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func approvedRepairOperations(operations []outcomes.Operation) map[int]*outcomes.Operation {
|
||||
byProposal := make(map[int]*outcomes.Operation)
|
||||
for i := range operations {
|
||||
op := &operations[i]
|
||||
if op.Reference.Kind == outcomes.OperationCorrectionRepair && op.Reference.ProposalIndex != nil &&
|
||||
slices.Contains(op.RecoveryReasons, outcomes.RecoveryCorrectionRevised) {
|
||||
index := *op.Reference.ProposalIndex
|
||||
if byProposal[index] == nil {
|
||||
byProposal[index] = op
|
||||
}
|
||||
}
|
||||
}
|
||||
return byProposal
|
||||
}
|
||||
|
||||
@@ -30,26 +30,27 @@ const (
|
||||
|
||||
// Request is the runtime input shared by deterministic validators.
|
||||
type Request struct {
|
||||
StageIndex *int `json:"stage_index,omitempty"`
|
||||
StageID *string `json:"stage_id,omitempty"`
|
||||
IndependentEvidence bool `json:"-"`
|
||||
RepairProposalIndex *int `json:"repair_proposal_index,omitempty"`
|
||||
RepairAttempt *int `json:"repair_attempt,omitempty"`
|
||||
SectionIndex *int `json:"section_index,omitempty"`
|
||||
WorkingTranscript *schema.Transcript `json:"-"`
|
||||
Section *SectionLookup `json:"-"`
|
||||
Previews map[int]proposals.SegmentPreviewResult `json:"-"`
|
||||
CandidateProposal []proposals.EnrichedCorrectionProposal `json:"candidate_proposals"`
|
||||
ModuleKey string `json:"module_key"`
|
||||
ModuleInstance string `json:"module_instance"`
|
||||
ValidatorOrder *int `json:"validator_order,omitempty"`
|
||||
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
|
||||
Glossary []glossary.Entry `json:"-"`
|
||||
ProtectedTerms []glossary.Term `json:"-"`
|
||||
GlossaryView *GlossaryView `json:"-"`
|
||||
Settings config.Stage `json:"-"`
|
||||
LLMClient StructuredLLMClient `json:"-"`
|
||||
Interactions structuredoutput.InteractionSink `json:"-"`
|
||||
StageIndex *int `json:"stage_index,omitempty"`
|
||||
StageID *string `json:"stage_id,omitempty"`
|
||||
IndependentEvidence bool `json:"-"`
|
||||
RepairProposalIndex *int `json:"repair_proposal_index,omitempty"`
|
||||
RepairAttempt *int `json:"repair_attempt,omitempty"`
|
||||
SectionIndex *int `json:"section_index,omitempty"`
|
||||
WorkingTranscript *schema.Transcript `json:"-"`
|
||||
Section *SectionLookup `json:"-"`
|
||||
Previews map[int]proposals.SegmentPreviewResult `json:"-"`
|
||||
PreviewApplicability map[int]bool `json:"-"`
|
||||
CandidateProposal []proposals.EnrichedCorrectionProposal `json:"candidate_proposals"`
|
||||
ModuleKey string `json:"module_key"`
|
||||
ModuleInstance string `json:"module_instance"`
|
||||
ValidatorOrder *int `json:"validator_order,omitempty"`
|
||||
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
|
||||
Glossary []glossary.Entry `json:"-"`
|
||||
ProtectedTerms []glossary.Term `json:"-"`
|
||||
GlossaryView *GlossaryView `json:"-"`
|
||||
Settings config.Stage `json:"-"`
|
||||
LLMClient StructuredLLMClient `json:"-"`
|
||||
Interactions structuredoutput.InteractionSink `json:"-"`
|
||||
}
|
||||
|
||||
// Decision is one validator decision for one proposal index.
|
||||
|
||||
@@ -52,6 +52,9 @@ func (r Request) Preview(candidate proposals.EnrichedCorrectionProposal) proposa
|
||||
if r.Previews != nil {
|
||||
r.Previews[candidate.ProposalIndex] = preview
|
||||
}
|
||||
if r.PreviewApplicability != nil {
|
||||
r.PreviewApplicability[candidate.ProposalIndex] = preview.Applicable
|
||||
}
|
||||
return preview
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user