Improve prompt roles and correction repair context
This commit is contained in:
@@ -57,6 +57,9 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
||||
if input.Spec.Settings.LLM.CorrectionRepairAttempts <= 0 {
|
||||
return initial, nil, nil
|
||||
}
|
||||
if input.Section == nil {
|
||||
input.Section = validators.NewSectionLookup(input.WorkingTranscript)
|
||||
}
|
||||
initialFindings := make(map[int][]ValidatorRejectedChange)
|
||||
for _, decision := range initial.rejected {
|
||||
initialFindings[decision.ProposalIndex] = append(initialFindings[decision.ProposalIndex], decision)
|
||||
@@ -100,9 +103,6 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
||||
if position < len(references) {
|
||||
localRef = references[position]
|
||||
} else {
|
||||
if input.Section == nil {
|
||||
input.Section = validators.NewSectionLookup(input.WorkingTranscript)
|
||||
}
|
||||
if pos, ok := input.Section.Position(original.TargetSegmentID); ok {
|
||||
localRef = pos + 1
|
||||
}
|
||||
@@ -119,7 +119,20 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
||||
findings := safeFindings(rejected)
|
||||
replay := &structuredoutput.Replay{Exchange: input.Exchange}
|
||||
for attempt := 0; attempt < input.Spec.Settings.LLM.CorrectionRepairAttempts; attempt++ {
|
||||
payload, err := (proposal_generation.RepairRequest{ProposalRef: position + 1, Correction: proposal_generation.StructuredCorrectionProposal{SegmentReference: localRef, OriginalText: current.OriginalText, CorrectedText: current.CorrectedText}, Findings: findings}).JSON()
|
||||
request := proposal_generation.RepairRequest{
|
||||
ProposalRef: position + 1,
|
||||
Correction: proposal_generation.StructuredCorrectionProposal{SegmentReference: localRef, OriginalText: current.OriginalText, CorrectedText: current.CorrectedText},
|
||||
Findings: findings,
|
||||
}
|
||||
if pos, ok := input.Section.Position(current.TargetSegmentID); ok {
|
||||
segment := &input.WorkingTranscript.Segments[pos]
|
||||
request.OriginalSegmentText = &segment.Text
|
||||
preview := proposals.PreviewProposalForSegment(segment, current.CorrectionProposal, input.Policy)
|
||||
if preview.Applicable {
|
||||
request.CorrectedSegmentText = &preview.CorrectedSegmentText
|
||||
}
|
||||
}
|
||||
payload, err := request.JSON()
|
||||
if err != nil {
|
||||
branches = append(branches, branch)
|
||||
return finish(err)
|
||||
|
||||
@@ -67,11 +67,98 @@ func TestRepairBranchesRetargetAndPreserveOtherApprovals(t *testing.T) {
|
||||
if err := json.Unmarshal([]byte(client.calls[0].Variables["RepairRequestJSON"]), &payload); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if payload.ProposalRef != 2 || payload.Correction.SegmentReference != 9 {
|
||||
if payload.ProposalRef != 2 || payload.Correction.SegmentReference != 9 || payload.OriginalSegmentText != nil || payload.CorrectedSegmentText != nil {
|
||||
t.Fatalf("request: %+v", payload)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepairRequestCarriesLiteralPreview(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, segment, original, replacement, result string
|
||||
policy proposals.ReplacementPolicy
|
||||
preview bool
|
||||
}{
|
||||
{"deletion", "uh hello", "uh ", "", "hello", proposals.ReplacementPolicyRequireUnique, true},
|
||||
{"all occurrences", "gestures and gestures", "gestures", "Jesters", "Jesters and Jesters", proposals.ReplacementPolicyReplaceAll, true},
|
||||
{"ambiguous span", "gestures and gestures", "gestures", "Jesters", "", proposals.ReplacementPolicyRequireUnique, false},
|
||||
{"missing span", "hello", "absent", "world", "", proposals.ReplacementPolicyRequireUnique, false},
|
||||
{"blank result", "uh", "uh", "", "", proposals.ReplacementPolicyRequireUnique, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
client := &repairClient{outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`}}
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 81, OriginalText: tc.original, CorrectedText: tc.replacement}}
|
||||
input := validateSectionCandidatesInput{
|
||||
Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}},
|
||||
Policy: tc.policy,
|
||||
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 81, Text: tc.segment}}},
|
||||
Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.NonEmptyResultingSegmentValidator{}, rejectUntilRepair()},
|
||||
SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate},
|
||||
Exchange: &structuredoutput.Exchange{PromptID: "proposal"},
|
||||
}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, branches, err := repairCandidates(context.Background(), input, []int{1}, initial, "repair")
|
||||
if err != nil || len(client.calls) != 1 || len(branches) != 1 || branches[0].Outcome != "withdrawn" {
|
||||
t.Fatalf("repair: branches=%+v calls=%d err=%v", branches, len(client.calls), err)
|
||||
}
|
||||
var payload proposal_generation.RepairRequest
|
||||
if err := json.Unmarshal([]byte(client.calls[0].Variables["RepairRequestJSON"]), &payload); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if payload.OriginalSegmentText == nil || *payload.OriginalSegmentText != tc.segment || len(payload.Findings) == 0 {
|
||||
t.Fatalf("missing source or findings: %+v", payload)
|
||||
}
|
||||
if tc.preview {
|
||||
if payload.CorrectedSegmentText == nil || *payload.CorrectedSegmentText != tc.result {
|
||||
t.Fatalf("incorrect preview: %+v", payload)
|
||||
}
|
||||
} else if payload.CorrectedSegmentText != nil {
|
||||
t.Fatalf("inapplicable proposal has a preview: %+v", payload)
|
||||
}
|
||||
if input.WorkingTranscript.Segments[0].Text != tc.segment {
|
||||
t.Fatal("repair preview changed the transcript")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepairRequestRefreshesPreviewAfterRetarget(t *testing.T) {
|
||||
client := &repairClient{outputs: []string{
|
||||
`{"proposal_ref":1,"action":"revise","correction":{"id":2,"original_text":"uh ","corrected_text":""}}`,
|
||||
`{"proposal_ref":1,"action":"withdraw","correction":null}`,
|
||||
}}
|
||||
reject := fakeValidator{name: "editorial_review", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
return validators.Result{Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Disposition: validators.DispositionRejected, ReasonCode: "llm_rejected", Message: "The context does not support this edit."}}}, nil
|
||||
}}
|
||||
input := validateSectionCandidatesInput{
|
||||
Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 2}}},
|
||||
Policy: proposals.ReplacementPolicyRequireUnique,
|
||||
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 81, Text: "cat"}, {ID: 104, Text: "uh hello"}}},
|
||||
Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, reject},
|
||||
SectionEnriched: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 81, OriginalText: "cat", CorrectedText: "dog"}}},
|
||||
Exchange: &structuredoutput.Exchange{PromptID: "proposal"},
|
||||
}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, branches, err := repairCandidates(context.Background(), input, []int{1}, initial, "repair")
|
||||
if err != nil || len(client.calls) != 2 || len(branches) != 1 || branches[0].Outcome != "withdrawn" {
|
||||
t.Fatalf("repair: branches=%+v calls=%d err=%v", branches, len(client.calls), err)
|
||||
}
|
||||
for i, want := range []struct{ before, after string }{{"cat", "dog"}, {"uh hello", "hello"}} {
|
||||
var payload proposal_generation.RepairRequest
|
||||
if err := json.Unmarshal([]byte(client.calls[i].Variables["RepairRequestJSON"]), &payload); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if payload.ProposalRef != 1 || payload.Correction.SegmentReference != i+1 || payload.OriginalSegmentText == nil || *payload.OriginalSegmentText != want.before || payload.CorrectedSegmentText == nil || *payload.CorrectedSegmentText != want.after {
|
||||
t.Fatalf("attempt %d did not describe its current proposal: %+v", i+1, payload)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepairBranchTerminalOutcomes(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
|
||||
Reference in New Issue
Block a user