Improve prompt roles and correction repair context
This commit is contained in:
@@ -12,9 +12,11 @@ type RepairFinding struct {
|
||||
|
||||
// RepairRequest identifies the proposal version and its ordered findings.
|
||||
type RepairRequest struct {
|
||||
ProposalRef int `json:"proposal_ref"`
|
||||
Correction StructuredCorrectionProposal `json:"correction"`
|
||||
Findings []RepairFinding `json:"findings"`
|
||||
ProposalRef int `json:"proposal_ref"`
|
||||
Correction StructuredCorrectionProposal `json:"correction"`
|
||||
Findings []RepairFinding `json:"findings"`
|
||||
OriginalSegmentText *string `json:"original_segment_text,omitempty"`
|
||||
CorrectedSegmentText *string `json:"corrected_segment_text,omitempty"`
|
||||
}
|
||||
|
||||
func (r RepairRequest) JSON() (string, error) {
|
||||
|
||||
@@ -57,6 +57,9 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
||||
if input.Spec.Settings.LLM.CorrectionRepairAttempts <= 0 {
|
||||
return initial, nil, nil
|
||||
}
|
||||
if input.Section == nil {
|
||||
input.Section = validators.NewSectionLookup(input.WorkingTranscript)
|
||||
}
|
||||
initialFindings := make(map[int][]ValidatorRejectedChange)
|
||||
for _, decision := range initial.rejected {
|
||||
initialFindings[decision.ProposalIndex] = append(initialFindings[decision.ProposalIndex], decision)
|
||||
@@ -100,9 +103,6 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
||||
if position < len(references) {
|
||||
localRef = references[position]
|
||||
} else {
|
||||
if input.Section == nil {
|
||||
input.Section = validators.NewSectionLookup(input.WorkingTranscript)
|
||||
}
|
||||
if pos, ok := input.Section.Position(original.TargetSegmentID); ok {
|
||||
localRef = pos + 1
|
||||
}
|
||||
@@ -119,7 +119,20 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
||||
findings := safeFindings(rejected)
|
||||
replay := &structuredoutput.Replay{Exchange: input.Exchange}
|
||||
for attempt := 0; attempt < input.Spec.Settings.LLM.CorrectionRepairAttempts; attempt++ {
|
||||
payload, err := (proposal_generation.RepairRequest{ProposalRef: position + 1, Correction: proposal_generation.StructuredCorrectionProposal{SegmentReference: localRef, OriginalText: current.OriginalText, CorrectedText: current.CorrectedText}, Findings: findings}).JSON()
|
||||
request := proposal_generation.RepairRequest{
|
||||
ProposalRef: position + 1,
|
||||
Correction: proposal_generation.StructuredCorrectionProposal{SegmentReference: localRef, OriginalText: current.OriginalText, CorrectedText: current.CorrectedText},
|
||||
Findings: findings,
|
||||
}
|
||||
if pos, ok := input.Section.Position(current.TargetSegmentID); ok {
|
||||
segment := &input.WorkingTranscript.Segments[pos]
|
||||
request.OriginalSegmentText = &segment.Text
|
||||
preview := proposals.PreviewProposalForSegment(segment, current.CorrectionProposal, input.Policy)
|
||||
if preview.Applicable {
|
||||
request.CorrectedSegmentText = &preview.CorrectedSegmentText
|
||||
}
|
||||
}
|
||||
payload, err := request.JSON()
|
||||
if err != nil {
|
||||
branches = append(branches, branch)
|
||||
return finish(err)
|
||||
|
||||
@@ -67,11 +67,98 @@ func TestRepairBranchesRetargetAndPreserveOtherApprovals(t *testing.T) {
|
||||
if err := json.Unmarshal([]byte(client.calls[0].Variables["RepairRequestJSON"]), &payload); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if payload.ProposalRef != 2 || payload.Correction.SegmentReference != 9 {
|
||||
if payload.ProposalRef != 2 || payload.Correction.SegmentReference != 9 || payload.OriginalSegmentText != nil || payload.CorrectedSegmentText != nil {
|
||||
t.Fatalf("request: %+v", payload)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepairRequestCarriesLiteralPreview(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, segment, original, replacement, result string
|
||||
policy proposals.ReplacementPolicy
|
||||
preview bool
|
||||
}{
|
||||
{"deletion", "uh hello", "uh ", "", "hello", proposals.ReplacementPolicyRequireUnique, true},
|
||||
{"all occurrences", "gestures and gestures", "gestures", "Jesters", "Jesters and Jesters", proposals.ReplacementPolicyReplaceAll, true},
|
||||
{"ambiguous span", "gestures and gestures", "gestures", "Jesters", "", proposals.ReplacementPolicyRequireUnique, false},
|
||||
{"missing span", "hello", "absent", "world", "", proposals.ReplacementPolicyRequireUnique, false},
|
||||
{"blank result", "uh", "uh", "", "", proposals.ReplacementPolicyRequireUnique, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
client := &repairClient{outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`}}
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 81, OriginalText: tc.original, CorrectedText: tc.replacement}}
|
||||
input := validateSectionCandidatesInput{
|
||||
Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}},
|
||||
Policy: tc.policy,
|
||||
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 81, Text: tc.segment}}},
|
||||
Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.NonEmptyResultingSegmentValidator{}, rejectUntilRepair()},
|
||||
SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate},
|
||||
Exchange: &structuredoutput.Exchange{PromptID: "proposal"},
|
||||
}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, branches, err := repairCandidates(context.Background(), input, []int{1}, initial, "repair")
|
||||
if err != nil || len(client.calls) != 1 || len(branches) != 1 || branches[0].Outcome != "withdrawn" {
|
||||
t.Fatalf("repair: branches=%+v calls=%d err=%v", branches, len(client.calls), err)
|
||||
}
|
||||
var payload proposal_generation.RepairRequest
|
||||
if err := json.Unmarshal([]byte(client.calls[0].Variables["RepairRequestJSON"]), &payload); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if payload.OriginalSegmentText == nil || *payload.OriginalSegmentText != tc.segment || len(payload.Findings) == 0 {
|
||||
t.Fatalf("missing source or findings: %+v", payload)
|
||||
}
|
||||
if tc.preview {
|
||||
if payload.CorrectedSegmentText == nil || *payload.CorrectedSegmentText != tc.result {
|
||||
t.Fatalf("incorrect preview: %+v", payload)
|
||||
}
|
||||
} else if payload.CorrectedSegmentText != nil {
|
||||
t.Fatalf("inapplicable proposal has a preview: %+v", payload)
|
||||
}
|
||||
if input.WorkingTranscript.Segments[0].Text != tc.segment {
|
||||
t.Fatal("repair preview changed the transcript")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepairRequestRefreshesPreviewAfterRetarget(t *testing.T) {
|
||||
client := &repairClient{outputs: []string{
|
||||
`{"proposal_ref":1,"action":"revise","correction":{"id":2,"original_text":"uh ","corrected_text":""}}`,
|
||||
`{"proposal_ref":1,"action":"withdraw","correction":null}`,
|
||||
}}
|
||||
reject := fakeValidator{name: "editorial_review", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
return validators.Result{Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Disposition: validators.DispositionRejected, ReasonCode: "llm_rejected", Message: "The context does not support this edit."}}}, nil
|
||||
}}
|
||||
input := validateSectionCandidatesInput{
|
||||
Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 2}}},
|
||||
Policy: proposals.ReplacementPolicyRequireUnique,
|
||||
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 81, Text: "cat"}, {ID: 104, Text: "uh hello"}}},
|
||||
Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, reject},
|
||||
SectionEnriched: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 81, OriginalText: "cat", CorrectedText: "dog"}}},
|
||||
Exchange: &structuredoutput.Exchange{PromptID: "proposal"},
|
||||
}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, branches, err := repairCandidates(context.Background(), input, []int{1}, initial, "repair")
|
||||
if err != nil || len(client.calls) != 2 || len(branches) != 1 || branches[0].Outcome != "withdrawn" {
|
||||
t.Fatalf("repair: branches=%+v calls=%d err=%v", branches, len(client.calls), err)
|
||||
}
|
||||
for i, want := range []struct{ before, after string }{{"cat", "dog"}, {"uh hello", "hello"}} {
|
||||
var payload proposal_generation.RepairRequest
|
||||
if err := json.Unmarshal([]byte(client.calls[i].Variables["RepairRequestJSON"]), &payload); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if payload.ProposalRef != 1 || payload.Correction.SegmentReference != i+1 || payload.OriginalSegmentText == nil || *payload.OriginalSegmentText != want.before || payload.CorrectedSegmentText == nil || *payload.CorrectedSegmentText != want.after {
|
||||
t.Fatalf("attempt %d did not describe its current proposal: %+v", i+1, payload)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepairBranchTerminalOutcomes(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
|
||||
@@ -19,6 +19,7 @@ func TestComposedValidationBudgetMatchesSerializedPayload(t *testing.T) {
|
||||
for i := range items {
|
||||
items[i] = LLMValidationItem{
|
||||
CorrectionIndex: 100 + i*7,
|
||||
Speaker: "Guest \"A\" — session host",
|
||||
OriginalText: "one\u00a0two!",
|
||||
CorrectedText: "one two",
|
||||
OriginalSegmentText: "she said \"one\u00a0two!\"\n",
|
||||
|
||||
@@ -14,9 +14,9 @@ import (
|
||||
func TestValidationContextCarriesAdjacentEvidenceAndRelevantDefinitions(t *testing.T) {
|
||||
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(7, "gestures", "Jesters")})
|
||||
req.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{
|
||||
{ID: 1, Text: "The performers arrived."},
|
||||
{ID: 2, Text: "Those gestures are back."},
|
||||
{ID: 3, Text: "Lyra recognized the troupe."},
|
||||
{ID: 1, Speaker: "Host", Text: "The performers arrived."},
|
||||
{ID: 2, Speaker: "Guest", Text: "Those gestures are back."},
|
||||
{ID: 3, Speaker: "Host", Text: "Lyra recognized the troupe."},
|
||||
{ID: 4, Text: "Elsewhere."},
|
||||
}}
|
||||
req.CandidateProposal[0].TargetSegmentID = 2
|
||||
@@ -44,14 +44,15 @@ func TestValidationContextCarriesAdjacentEvidenceAndRelevantDefinitions(t *testi
|
||||
if strings.Contains(variables["PayloadJSON"], `"segment_id":`) || rendered[0].CorrectionIndex != 1 || payload.Items[0].CorrectionIndex != 7 {
|
||||
t.Fatalf("leaked internal identity or mutated request: %s", variables["PayloadJSON"])
|
||||
}
|
||||
if len(rendered[0].NeighboringSegments) != 2 || rendered[0].GlossaryEntries[0].Description != "A traveling troupe" {
|
||||
if rendered[0].Speaker != "Guest" || len(rendered[0].NeighboringSegments) != 2 || rendered[0].NeighboringSegments[0].Speaker != "Host" || rendered[0].GlossaryEntries[0].Description != "A traveling troupe" {
|
||||
t.Fatalf("rendered context: %+v", rendered)
|
||||
}
|
||||
// The context-free endpoint must not reach beyond the current chunk.
|
||||
req.WorkingTranscript.Segments = req.WorkingTranscript.Segments[1:2]
|
||||
req.WorkingTranscript.Segments[0].Speaker = ""
|
||||
req.Glossary = nil
|
||||
isolated, _ := BuildLLMValidationRequest("editorial_review", LLMValidatorTypeEditorialReview, req)
|
||||
if len(isolated.Items[0].NeighboringSegments) != 0 || len(isolated.Items[0].GlossaryEntries) != 0 {
|
||||
if isolated.Items[0].Speaker != "" || len(isolated.Items[0].NeighboringSegments) != 0 || len(isolated.Items[0].GlossaryEntries) != 0 {
|
||||
t.Fatalf("unexpected external context: %+v", isolated)
|
||||
}
|
||||
}
|
||||
@@ -63,12 +64,18 @@ func TestValidationContextFitsBudgetWithoutDiscardingCandidate(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
base.Speaker = "Guest speaker with a long identifying label"
|
||||
withSpeaker, err := estimateBatchTokens(estimator, []LLMValidationItem{base})
|
||||
if err != nil || withSpeaker <= required {
|
||||
t.Fatalf("speaker missing from token accounting: before=%d after=%d err=%v", required, withSpeaker, err)
|
||||
}
|
||||
required = withSpeaker
|
||||
enriched := base
|
||||
enriched.NeighboringSegments = []NeighboringSegment{{Position: "following", Text: strings.Repeat("long context ", 300)}}
|
||||
enriched.GlossaryEntries = []glossary.Entry{{Name: "cat", Description: strings.Repeat("long definition ", 300)}}
|
||||
fitted := fitValidationContext(enriched, required, estimator)
|
||||
tokens, err := estimateBatchTokens(estimator, []LLMValidationItem{fitted})
|
||||
if err != nil || tokens > required || fitted.OriginalSegmentText != base.OriginalSegmentText || fitted.CorrectedSegmentText != base.CorrectedSegmentText {
|
||||
if err != nil || tokens > required || fitted.Speaker != base.Speaker || fitted.OriginalSegmentText != base.OriginalSegmentText || fitted.CorrectedSegmentText != base.CorrectedSegmentText {
|
||||
t.Fatalf("fitted: %+v tokens=%d err=%v", fitted, tokens, err)
|
||||
}
|
||||
full, _ := estimateBatchTokens(estimator, []LLMValidationItem{enriched})
|
||||
|
||||
@@ -22,6 +22,7 @@ type StructuredLLMClient = structuredoutput.Client
|
||||
type LLMValidationItem struct {
|
||||
CorrectionIndex int `json:"correction_index"`
|
||||
SegmentID int `json:"-"`
|
||||
Speaker string `json:"speaker"`
|
||||
OriginalText string `json:"original_text"`
|
||||
CorrectedText string `json:"corrected_text"`
|
||||
OriginalSegmentText string `json:"original_segment_text"`
|
||||
|
||||
@@ -269,6 +269,7 @@ func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorT
|
||||
items = append(items, LLMValidationItem{
|
||||
CorrectionIndex: p.ProposalIndex,
|
||||
SegmentID: p.TargetSegmentID,
|
||||
Speaker: seg.Speaker,
|
||||
OriginalText: p.OriginalText,
|
||||
CorrectedText: p.CorrectedText,
|
||||
OriginalSegmentText: seg.Text,
|
||||
|
||||
Reference in New Issue
Block a user