Improve prompt roles and correction repair context

This commit is contained in:
2026-11-11 17:42:30 +00:00
parent cf1fbe5748
commit f42573f8b0
23 changed files with 256 additions and 40 deletions

View File

@@ -57,6 +57,9 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
if input.Spec.Settings.LLM.CorrectionRepairAttempts <= 0 {
return initial, nil, nil
}
if input.Section == nil {
input.Section = validators.NewSectionLookup(input.WorkingTranscript)
}
initialFindings := make(map[int][]ValidatorRejectedChange)
for _, decision := range initial.rejected {
initialFindings[decision.ProposalIndex] = append(initialFindings[decision.ProposalIndex], decision)
@@ -100,9 +103,6 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
if position < len(references) {
localRef = references[position]
} else {
if input.Section == nil {
input.Section = validators.NewSectionLookup(input.WorkingTranscript)
}
if pos, ok := input.Section.Position(original.TargetSegmentID); ok {
localRef = pos + 1
}
@@ -119,7 +119,20 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
findings := safeFindings(rejected)
replay := &structuredoutput.Replay{Exchange: input.Exchange}
for attempt := 0; attempt < input.Spec.Settings.LLM.CorrectionRepairAttempts; attempt++ {
payload, err := (proposal_generation.RepairRequest{ProposalRef: position + 1, Correction: proposal_generation.StructuredCorrectionProposal{SegmentReference: localRef, OriginalText: current.OriginalText, CorrectedText: current.CorrectedText}, Findings: findings}).JSON()
request := proposal_generation.RepairRequest{
ProposalRef: position + 1,
Correction: proposal_generation.StructuredCorrectionProposal{SegmentReference: localRef, OriginalText: current.OriginalText, CorrectedText: current.CorrectedText},
Findings: findings,
}
if pos, ok := input.Section.Position(current.TargetSegmentID); ok {
segment := &input.WorkingTranscript.Segments[pos]
request.OriginalSegmentText = &segment.Text
preview := proposals.PreviewProposalForSegment(segment, current.CorrectionProposal, input.Policy)
if preview.Applicable {
request.CorrectedSegmentText = &preview.CorrectedSegmentText
}
}
payload, err := request.JSON()
if err != nil {
branches = append(branches, branch)
return finish(err)

View File

@@ -67,11 +67,98 @@ func TestRepairBranchesRetargetAndPreserveOtherApprovals(t *testing.T) {
if err := json.Unmarshal([]byte(client.calls[0].Variables["RepairRequestJSON"]), &payload); err != nil {
t.Fatal(err)
}
if payload.ProposalRef != 2 || payload.Correction.SegmentReference != 9 {
if payload.ProposalRef != 2 || payload.Correction.SegmentReference != 9 || payload.OriginalSegmentText != nil || payload.CorrectedSegmentText != nil {
t.Fatalf("request: %+v", payload)
}
}
func TestRepairRequestCarriesLiteralPreview(t *testing.T) {
for _, tc := range []struct {
name, segment, original, replacement, result string
policy proposals.ReplacementPolicy
preview bool
}{
{"deletion", "uh hello", "uh ", "", "hello", proposals.ReplacementPolicyRequireUnique, true},
{"all occurrences", "gestures and gestures", "gestures", "Jesters", "Jesters and Jesters", proposals.ReplacementPolicyReplaceAll, true},
{"ambiguous span", "gestures and gestures", "gestures", "Jesters", "", proposals.ReplacementPolicyRequireUnique, false},
{"missing span", "hello", "absent", "world", "", proposals.ReplacementPolicyRequireUnique, false},
{"blank result", "uh", "uh", "", "", proposals.ReplacementPolicyRequireUnique, false},
} {
t.Run(tc.name, func(t *testing.T) {
client := &repairClient{outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`}}
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 81, OriginalText: tc.original, CorrectedText: tc.replacement}}
input := validateSectionCandidatesInput{
Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}},
Policy: tc.policy,
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 81, Text: tc.segment}}},
Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.NonEmptyResultingSegmentValidator{}, rejectUntilRepair()},
SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate},
Exchange: &structuredoutput.Exchange{PromptID: "proposal"},
}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
}
_, branches, err := repairCandidates(context.Background(), input, []int{1}, initial, "repair")
if err != nil || len(client.calls) != 1 || len(branches) != 1 || branches[0].Outcome != "withdrawn" {
t.Fatalf("repair: branches=%+v calls=%d err=%v", branches, len(client.calls), err)
}
var payload proposal_generation.RepairRequest
if err := json.Unmarshal([]byte(client.calls[0].Variables["RepairRequestJSON"]), &payload); err != nil {
t.Fatal(err)
}
if payload.OriginalSegmentText == nil || *payload.OriginalSegmentText != tc.segment || len(payload.Findings) == 0 {
t.Fatalf("missing source or findings: %+v", payload)
}
if tc.preview {
if payload.CorrectedSegmentText == nil || *payload.CorrectedSegmentText != tc.result {
t.Fatalf("incorrect preview: %+v", payload)
}
} else if payload.CorrectedSegmentText != nil {
t.Fatalf("inapplicable proposal has a preview: %+v", payload)
}
if input.WorkingTranscript.Segments[0].Text != tc.segment {
t.Fatal("repair preview changed the transcript")
}
})
}
}
func TestRepairRequestRefreshesPreviewAfterRetarget(t *testing.T) {
client := &repairClient{outputs: []string{
`{"proposal_ref":1,"action":"revise","correction":{"id":2,"original_text":"uh ","corrected_text":""}}`,
`{"proposal_ref":1,"action":"withdraw","correction":null}`,
}}
reject := fakeValidator{name: "editorial_review", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
return validators.Result{Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Disposition: validators.DispositionRejected, ReasonCode: "llm_rejected", Message: "The context does not support this edit."}}}, nil
}}
input := validateSectionCandidatesInput{
Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 2}}},
Policy: proposals.ReplacementPolicyRequireUnique,
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 81, Text: "cat"}, {ID: 104, Text: "uh hello"}}},
Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, reject},
SectionEnriched: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 81, OriginalText: "cat", CorrectedText: "dog"}}},
Exchange: &structuredoutput.Exchange{PromptID: "proposal"},
}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
}
_, branches, err := repairCandidates(context.Background(), input, []int{1}, initial, "repair")
if err != nil || len(client.calls) != 2 || len(branches) != 1 || branches[0].Outcome != "withdrawn" {
t.Fatalf("repair: branches=%+v calls=%d err=%v", branches, len(client.calls), err)
}
for i, want := range []struct{ before, after string }{{"cat", "dog"}, {"uh hello", "hello"}} {
var payload proposal_generation.RepairRequest
if err := json.Unmarshal([]byte(client.calls[i].Variables["RepairRequestJSON"]), &payload); err != nil {
t.Fatal(err)
}
if payload.ProposalRef != 1 || payload.Correction.SegmentReference != i+1 || payload.OriginalSegmentText == nil || *payload.OriginalSegmentText != want.before || payload.CorrectedSegmentText == nil || *payload.CorrectedSegmentText != want.after {
t.Fatalf("attempt %d did not describe its current proposal: %+v", i+1, payload)
}
}
}
func TestRepairBranchTerminalOutcomes(t *testing.T) {
for _, tc := range []struct {
name string