Improve prompt roles and correction repair context

This commit is contained in:
2026-11-11 17:42:30 +00:00
parent cf1fbe5748
commit f42573f8b0
23 changed files with 256 additions and 40 deletions

View File

@@ -12,9 +12,11 @@ type RepairFinding struct {
// RepairRequest identifies the proposal version and its ordered findings.
type RepairRequest struct {
ProposalRef int `json:"proposal_ref"`
Correction StructuredCorrectionProposal `json:"correction"`
Findings []RepairFinding `json:"findings"`
ProposalRef int `json:"proposal_ref"`
Correction StructuredCorrectionProposal `json:"correction"`
Findings []RepairFinding `json:"findings"`
OriginalSegmentText *string `json:"original_segment_text,omitempty"`
CorrectedSegmentText *string `json:"corrected_segment_text,omitempty"`
}
func (r RepairRequest) JSON() (string, error) {

View File

@@ -57,6 +57,9 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
if input.Spec.Settings.LLM.CorrectionRepairAttempts <= 0 {
return initial, nil, nil
}
if input.Section == nil {
input.Section = validators.NewSectionLookup(input.WorkingTranscript)
}
initialFindings := make(map[int][]ValidatorRejectedChange)
for _, decision := range initial.rejected {
initialFindings[decision.ProposalIndex] = append(initialFindings[decision.ProposalIndex], decision)
@@ -100,9 +103,6 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
if position < len(references) {
localRef = references[position]
} else {
if input.Section == nil {
input.Section = validators.NewSectionLookup(input.WorkingTranscript)
}
if pos, ok := input.Section.Position(original.TargetSegmentID); ok {
localRef = pos + 1
}
@@ -119,7 +119,20 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
findings := safeFindings(rejected)
replay := &structuredoutput.Replay{Exchange: input.Exchange}
for attempt := 0; attempt < input.Spec.Settings.LLM.CorrectionRepairAttempts; attempt++ {
payload, err := (proposal_generation.RepairRequest{ProposalRef: position + 1, Correction: proposal_generation.StructuredCorrectionProposal{SegmentReference: localRef, OriginalText: current.OriginalText, CorrectedText: current.CorrectedText}, Findings: findings}).JSON()
request := proposal_generation.RepairRequest{
ProposalRef: position + 1,
Correction: proposal_generation.StructuredCorrectionProposal{SegmentReference: localRef, OriginalText: current.OriginalText, CorrectedText: current.CorrectedText},
Findings: findings,
}
if pos, ok := input.Section.Position(current.TargetSegmentID); ok {
segment := &input.WorkingTranscript.Segments[pos]
request.OriginalSegmentText = &segment.Text
preview := proposals.PreviewProposalForSegment(segment, current.CorrectionProposal, input.Policy)
if preview.Applicable {
request.CorrectedSegmentText = &preview.CorrectedSegmentText
}
}
payload, err := request.JSON()
if err != nil {
branches = append(branches, branch)
return finish(err)

View File

@@ -67,11 +67,98 @@ func TestRepairBranchesRetargetAndPreserveOtherApprovals(t *testing.T) {
if err := json.Unmarshal([]byte(client.calls[0].Variables["RepairRequestJSON"]), &payload); err != nil {
t.Fatal(err)
}
if payload.ProposalRef != 2 || payload.Correction.SegmentReference != 9 {
if payload.ProposalRef != 2 || payload.Correction.SegmentReference != 9 || payload.OriginalSegmentText != nil || payload.CorrectedSegmentText != nil {
t.Fatalf("request: %+v", payload)
}
}
func TestRepairRequestCarriesLiteralPreview(t *testing.T) {
for _, tc := range []struct {
name, segment, original, replacement, result string
policy proposals.ReplacementPolicy
preview bool
}{
{"deletion", "uh hello", "uh ", "", "hello", proposals.ReplacementPolicyRequireUnique, true},
{"all occurrences", "gestures and gestures", "gestures", "Jesters", "Jesters and Jesters", proposals.ReplacementPolicyReplaceAll, true},
{"ambiguous span", "gestures and gestures", "gestures", "Jesters", "", proposals.ReplacementPolicyRequireUnique, false},
{"missing span", "hello", "absent", "world", "", proposals.ReplacementPolicyRequireUnique, false},
{"blank result", "uh", "uh", "", "", proposals.ReplacementPolicyRequireUnique, false},
} {
t.Run(tc.name, func(t *testing.T) {
client := &repairClient{outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`}}
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 81, OriginalText: tc.original, CorrectedText: tc.replacement}}
input := validateSectionCandidatesInput{
Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}},
Policy: tc.policy,
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 81, Text: tc.segment}}},
Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.NonEmptyResultingSegmentValidator{}, rejectUntilRepair()},
SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate},
Exchange: &structuredoutput.Exchange{PromptID: "proposal"},
}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
}
_, branches, err := repairCandidates(context.Background(), input, []int{1}, initial, "repair")
if err != nil || len(client.calls) != 1 || len(branches) != 1 || branches[0].Outcome != "withdrawn" {
t.Fatalf("repair: branches=%+v calls=%d err=%v", branches, len(client.calls), err)
}
var payload proposal_generation.RepairRequest
if err := json.Unmarshal([]byte(client.calls[0].Variables["RepairRequestJSON"]), &payload); err != nil {
t.Fatal(err)
}
if payload.OriginalSegmentText == nil || *payload.OriginalSegmentText != tc.segment || len(payload.Findings) == 0 {
t.Fatalf("missing source or findings: %+v", payload)
}
if tc.preview {
if payload.CorrectedSegmentText == nil || *payload.CorrectedSegmentText != tc.result {
t.Fatalf("incorrect preview: %+v", payload)
}
} else if payload.CorrectedSegmentText != nil {
t.Fatalf("inapplicable proposal has a preview: %+v", payload)
}
if input.WorkingTranscript.Segments[0].Text != tc.segment {
t.Fatal("repair preview changed the transcript")
}
})
}
}
func TestRepairRequestRefreshesPreviewAfterRetarget(t *testing.T) {
client := &repairClient{outputs: []string{
`{"proposal_ref":1,"action":"revise","correction":{"id":2,"original_text":"uh ","corrected_text":""}}`,
`{"proposal_ref":1,"action":"withdraw","correction":null}`,
}}
reject := fakeValidator{name: "editorial_review", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
return validators.Result{Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Disposition: validators.DispositionRejected, ReasonCode: "llm_rejected", Message: "The context does not support this edit."}}}, nil
}}
input := validateSectionCandidatesInput{
Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 2}}},
Policy: proposals.ReplacementPolicyRequireUnique,
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 81, Text: "cat"}, {ID: 104, Text: "uh hello"}}},
Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, reject},
SectionEnriched: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 81, OriginalText: "cat", CorrectedText: "dog"}}},
Exchange: &structuredoutput.Exchange{PromptID: "proposal"},
}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
}
_, branches, err := repairCandidates(context.Background(), input, []int{1}, initial, "repair")
if err != nil || len(client.calls) != 2 || len(branches) != 1 || branches[0].Outcome != "withdrawn" {
t.Fatalf("repair: branches=%+v calls=%d err=%v", branches, len(client.calls), err)
}
for i, want := range []struct{ before, after string }{{"cat", "dog"}, {"uh hello", "hello"}} {
var payload proposal_generation.RepairRequest
if err := json.Unmarshal([]byte(client.calls[i].Variables["RepairRequestJSON"]), &payload); err != nil {
t.Fatal(err)
}
if payload.ProposalRef != 1 || payload.Correction.SegmentReference != i+1 || payload.OriginalSegmentText == nil || *payload.OriginalSegmentText != want.before || payload.CorrectedSegmentText == nil || *payload.CorrectedSegmentText != want.after {
t.Fatalf("attempt %d did not describe its current proposal: %+v", i+1, payload)
}
}
}
func TestRepairBranchTerminalOutcomes(t *testing.T) {
for _, tc := range []struct {
name string

View File

@@ -19,6 +19,7 @@ func TestComposedValidationBudgetMatchesSerializedPayload(t *testing.T) {
for i := range items {
items[i] = LLMValidationItem{
CorrectionIndex: 100 + i*7,
Speaker: "Guest \"A\" — session host",
OriginalText: "one\u00a0two!",
CorrectedText: "one two",
OriginalSegmentText: "she said \"one\u00a0two!\"\n",

View File

@@ -14,9 +14,9 @@ import (
func TestValidationContextCarriesAdjacentEvidenceAndRelevantDefinitions(t *testing.T) {
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(7, "gestures", "Jesters")})
req.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{
{ID: 1, Text: "The performers arrived."},
{ID: 2, Text: "Those gestures are back."},
{ID: 3, Text: "Lyra recognized the troupe."},
{ID: 1, Speaker: "Host", Text: "The performers arrived."},
{ID: 2, Speaker: "Guest", Text: "Those gestures are back."},
{ID: 3, Speaker: "Host", Text: "Lyra recognized the troupe."},
{ID: 4, Text: "Elsewhere."},
}}
req.CandidateProposal[0].TargetSegmentID = 2
@@ -44,14 +44,15 @@ func TestValidationContextCarriesAdjacentEvidenceAndRelevantDefinitions(t *testi
if strings.Contains(variables["PayloadJSON"], `"segment_id":`) || rendered[0].CorrectionIndex != 1 || payload.Items[0].CorrectionIndex != 7 {
t.Fatalf("leaked internal identity or mutated request: %s", variables["PayloadJSON"])
}
if len(rendered[0].NeighboringSegments) != 2 || rendered[0].GlossaryEntries[0].Description != "A traveling troupe" {
if rendered[0].Speaker != "Guest" || len(rendered[0].NeighboringSegments) != 2 || rendered[0].NeighboringSegments[0].Speaker != "Host" || rendered[0].GlossaryEntries[0].Description != "A traveling troupe" {
t.Fatalf("rendered context: %+v", rendered)
}
// The context-free endpoint must not reach beyond the current chunk.
req.WorkingTranscript.Segments = req.WorkingTranscript.Segments[1:2]
req.WorkingTranscript.Segments[0].Speaker = ""
req.Glossary = nil
isolated, _ := BuildLLMValidationRequest("editorial_review", LLMValidatorTypeEditorialReview, req)
if len(isolated.Items[0].NeighboringSegments) != 0 || len(isolated.Items[0].GlossaryEntries) != 0 {
if isolated.Items[0].Speaker != "" || len(isolated.Items[0].NeighboringSegments) != 0 || len(isolated.Items[0].GlossaryEntries) != 0 {
t.Fatalf("unexpected external context: %+v", isolated)
}
}
@@ -63,12 +64,18 @@ func TestValidationContextFitsBudgetWithoutDiscardingCandidate(t *testing.T) {
if err != nil {
t.Fatal(err)
}
base.Speaker = "Guest speaker with a long identifying label"
withSpeaker, err := estimateBatchTokens(estimator, []LLMValidationItem{base})
if err != nil || withSpeaker <= required {
t.Fatalf("speaker missing from token accounting: before=%d after=%d err=%v", required, withSpeaker, err)
}
required = withSpeaker
enriched := base
enriched.NeighboringSegments = []NeighboringSegment{{Position: "following", Text: strings.Repeat("long context ", 300)}}
enriched.GlossaryEntries = []glossary.Entry{{Name: "cat", Description: strings.Repeat("long definition ", 300)}}
fitted := fitValidationContext(enriched, required, estimator)
tokens, err := estimateBatchTokens(estimator, []LLMValidationItem{fitted})
if err != nil || tokens > required || fitted.OriginalSegmentText != base.OriginalSegmentText || fitted.CorrectedSegmentText != base.CorrectedSegmentText {
if err != nil || tokens > required || fitted.Speaker != base.Speaker || fitted.OriginalSegmentText != base.OriginalSegmentText || fitted.CorrectedSegmentText != base.CorrectedSegmentText {
t.Fatalf("fitted: %+v tokens=%d err=%v", fitted, tokens, err)
}
full, _ := estimateBatchTokens(estimator, []LLMValidationItem{enriched})

View File

@@ -22,6 +22,7 @@ type StructuredLLMClient = structuredoutput.Client
type LLMValidationItem struct {
CorrectionIndex int `json:"correction_index"`
SegmentID int `json:"-"`
Speaker string `json:"speaker"`
OriginalText string `json:"original_text"`
CorrectedText string `json:"corrected_text"`
OriginalSegmentText string `json:"original_segment_text"`

View File

@@ -269,6 +269,7 @@ func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorT
items = append(items, LLMValidationItem{
CorrectionIndex: p.ProposalIndex,
SegmentID: p.TargetSegmentID,
Speaker: seg.Speaker,
OriginalText: p.OriginalText,
CorrectedText: p.CorrectedText,
OriginalSegmentText: seg.Text,