Clarify prompt boundaries and enforce repair decisions

This commit is contained in:
2026-11-15 02:41:32 +00:00
parent e183708021
commit c4ccd7897e
13 changed files with 111 additions and 67 deletions

View File

@@ -41,7 +41,7 @@ func TestRoleClientContinuesIndependentRepairConversations(t *testing.T) {
}
var messages [][]wireMessage
var models []string
outputs := []string{`{"corrections":[]}`, `{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":"a","corrected_text":"b"}}`, `{"proposal_ref":1,"action":"withdraw","correction":null}`, `{"proposal_ref":1,"action":"withdraw","correction":null}`}
outputs := []string{`{"corrections":[]}`, `{"proposal_ref":1,"decision":{"action":"revise","correction":{"id":1,"original_text":"a","corrected_text":"b"}}}`, `{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`, `{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`}
transport := transportFunc(func(r *http.Request) (*http.Response, error) {
var req struct {
Model string `json:"model"`
@@ -126,7 +126,7 @@ func TestRepairContinuationUsesReplacementAndCountsStructuralRepair(t *testing.T
if err != nil {
t.Fatal(err)
}
outputs := []string{`{"corrections":[]}`, `{"bad":true}`, `{"proposal_ref":1,"action":"withdraw","correction":null}`}
outputs := []string{`{"corrections":[]}`, `{"proposal_ref":1,"decision":{"action":"revise","correction":null}}`, `{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`}
var requests [][]structuredoutput.Message
transport := transportFunc(func(r *http.Request) (*http.Response, error) {
var req struct {

View File

@@ -19,8 +19,8 @@ func TestCoordinatorFinalTurnFailuresAndAccounting(t *testing.T) {
again bool
reason outcomes.ReasonCode
}{
{name: "last turn assessed", reply: reply{200, `{"proposal_ref":1,"action":"withdraw","correction":null}`}},
{name: "exhaustion", reply: reply{200, `{"proposal_ref":1,"action":"withdraw","correction":null}`}, again: true},
{name: "last turn assessed", reply: reply{200, `{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`}},
{name: "exhaustion", reply: reply{200, `{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`}, again: true},
{name: "invalid structure", reply: reply{200, `{"wrong":true}`}, reason: outcomes.ReasonInvalidOutputExhausted},
{name: "provider error", reply: reply{400, ""}, reason: outcomes.ReasonProviderFailed},
} {
@@ -75,7 +75,7 @@ func TestCoordinatorFinalTurnFailuresAndAccounting(t *testing.T) {
}
func TestCoordinatorCanceledDeliveryRetainsEvidence(t *testing.T) {
runtime, _, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}, {200, `{"proposal_ref":1,"action":"withdraw","correction":null}`}}, 0, 0, nil)
runtime, _, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}, {200, `{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`}}, 0, 0, nil)
collector := NewObservationCollector()
proposal, _ := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, collector.Hooks())
req := adapterRequest(responseschema.CorrectionSetKey)
@@ -113,7 +113,7 @@ func TestCoordinatorCanceledDeliveryRetainsEvidence(t *testing.T) {
}
func TestCoordinatorReleasesAdmissionBeforeNestedValidation(t *testing.T) {
runtime, count, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}, {200, `{"proposal_ref":1,"action":"withdraw","correction":null}`}, {200, `{"validations":[]}`}}, 0, 0, nil)
runtime, count, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}, {200, `{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`}, {200, `{"validations":[]}`}}, 0, 0, nil)
collector := NewObservationCollector()
proposal, _ := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, collector.Hooks())
repair, _ := runtime.NewRoleClient("repair", "base", runtime.Config.LLM, collector.Hooks())

View File

@@ -9,9 +9,14 @@ import (
// RepairResponse is one model response for an original proposal reference.
type RepairResponse struct {
ProposalRef int `json:"proposal_ref"`
Action string `json:"action"`
Correction *StructuredCorrectionProposal `json:"correction"`
ProposalRef int `json:"proposal_ref"`
Decision RepairDecision `json:"decision"`
}
// RepairDecision selects revision or withdrawal for the referenced proposal.
type RepairDecision struct {
Action string `json:"action"`
Correction *StructuredCorrectionProposal `json:"correction"`
}
// DecodeRepairResponse checks correspondence after structural schema validation.
@@ -21,7 +26,7 @@ func DecodeRepairResponse(content []byte, expectedRef int) (RepairResponse, erro
if err := json.Unmarshal(content, &response); err != nil {
return RepairResponse{}, invalidRepairResponse(err)
}
if response.ProposalRef != expectedRef || (response.Action != "revise" || response.Correction == nil) && (response.Action != "withdraw" || response.Correction != nil) {
if response.ProposalRef != expectedRef || (response.Decision.Action != "revise" || response.Decision.Correction == nil) && (response.Decision.Action != "withdraw" || response.Decision.Correction != nil) {
return response, invalidRepairResponse(nil)
}
return response, nil

View File

@@ -13,11 +13,11 @@ func TestDecodeRepairResponse(t *testing.T) {
name, content string
valid bool
}{
{"revision with deletion", `{"proposal_ref":2,"action":"revise","correction":{"id":1,"original_text":" ","corrected_text":""}}`, true},
{"withdrawal", `{"proposal_ref":2,"action":"withdraw","correction":null}`, true},
{"wrong reference", `{"proposal_ref":1,"action":"withdraw","correction":null}`, false},
{"missing revision", `{"proposal_ref":2,"action":"revise","correction":null}`, false},
{"withdrawal with revision", `{"proposal_ref":2,"action":"withdraw","correction":{"id":1}}`, false},
{"revision with deletion", `{"proposal_ref":2,"decision":{"action":"revise","correction":{"id":1,"original_text":" ","corrected_text":""}}}`, true},
{"withdrawal", `{"proposal_ref":2,"decision":{"action":"withdraw","correction":null}}`, true},
{"wrong reference", `{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`, false},
{"missing revision", `{"proposal_ref":2,"decision":{"action":"revise","correction":null}}`, false},
{"withdrawal with revision", `{"proposal_ref":2,"decision":{"action":"withdraw","correction":{"id":1}}}`, false},
} {
t.Run(tc.name, func(t *testing.T) {
response, err := DecodeRepairResponse([]byte(tc.content), 2)
@@ -25,7 +25,7 @@ func TestDecodeRepairResponse(t *testing.T) {
t.Fatal(err)
}
if !tc.valid {
if response.ProposalRef == 0 || response.Action == "" {
if response.ProposalRef == 0 || response.Decision.Action == "" {
t.Fatalf("missing decoded evidence: %+v", response)
}
var failure *structuredoutput.Failure

View File

@@ -3,47 +3,71 @@
"additionalProperties": false,
"required": [
"proposal_ref",
"action",
"correction"
"decision"
],
"properties": {
"proposal_ref": {
"type": "integer",
"minimum": 1
},
"action": {
"type": "string",
"enum": [
"revise",
"withdraw"
]
},
"correction": {
"decision": {
"anyOf": [
{
"type": "object",
"additionalProperties": false,
"required": [
"id",
"original_text",
"corrected_text"
"action",
"correction"
],
"properties": {
"id": {
"type": "integer",
"minimum": 1
},
"original_text": {
"action": {
"type": "string",
"minLength": 1
"enum": [
"revise"
]
},
"corrected_text": {
"type": "string"
"correction": {
"type": "object",
"additionalProperties": false,
"required": [
"id",
"original_text",
"corrected_text"
],
"properties": {
"id": {
"type": "integer",
"minimum": 1
},
"original_text": {
"type": "string",
"minLength": 1
},
"corrected_text": {
"type": "string"
}
}
}
}
},
{
"type": "null"
"type": "object",
"additionalProperties": false,
"required": [
"action",
"correction"
],
"properties": {
"action": {
"type": "string",
"enum": [
"withdraw"
]
},
"correction": {
"type": "null"
}
}
}
]
}

View File

@@ -149,13 +149,16 @@ func TestCorrectionRepairSchema(t *testing.T) {
name, content string
valid bool
}{
{"delete whitespace", `{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":" ","corrected_text":""}}`, true},
{"withdraw", `{"proposal_ref":1,"action":"withdraw","correction":null}`, true},
{"missing correction", `{"proposal_ref":1,"action":"withdraw"}`, false},
{"extra field", `{"proposal_ref":1,"action":"withdraw","correction":null,"extra":1}`, false},
{"zero reference", `{"proposal_ref":0,"action":"withdraw","correction":null}`, false},
{"unknown action", `{"proposal_ref":1,"action":"other","correction":null}`, false},
{"empty original", `{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":"","corrected_text":"x"}}`, false},
{"delete whitespace", `{"proposal_ref":1,"decision":{"action":"revise","correction":{"id":1,"original_text":" ","corrected_text":""}}}`, true},
{"withdraw", `{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`, true},
{"revision without correction", `{"proposal_ref":1,"decision":{"action":"revise","correction":null}}`, false},
{"withdrawal with correction", `{"proposal_ref":1,"decision":{"action":"withdraw","correction":{"id":1,"original_text":"cat","corrected_text":"dog"}}}`, false},
{"flat response", `{"proposal_ref":1,"action":"withdraw","correction":null}`, false},
{"missing correction", `{"proposal_ref":1,"decision":{"action":"withdraw"}}`, false},
{"extra field", `{"proposal_ref":1,"extra":1,"decision":{"action":"withdraw","correction":null}}`, false},
{"zero reference", `{"proposal_ref":0,"decision":{"action":"withdraw","correction":null}}`, false},
{"unknown action", `{"proposal_ref":1,"decision":{"action":"other","correction":null}}`, false},
{"empty original", `{"proposal_ref":1,"decision":{"action":"revise","correction":{"id":1,"original_text":"","corrected_text":"x"}}}`, false},
} {
t.Run(tc.name, func(t *testing.T) {
var value any

View File

@@ -208,12 +208,12 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
}
operation.Succeeded = true
operation.RecoveryReasons = outcomes.ExecutionRecoveries(operation.Counts)
if response.Action == "withdraw" {
if response.Decision.Action == "withdraw" {
finishAttempt(nil, nil, validateSectionCandidatesResult{})
branch.Outcome = "withdrawn"
return nil, nil
}
revision := response.Correction
revision := response.Decision.Correction
localRef = revision.SegmentReference
target := 0
if input.WorkingTranscript != nil && localRef > 0 && localRef <= len(input.WorkingTranscript.Segments) {

View File

@@ -50,7 +50,7 @@ func (c *repairClient) CompleteStructured(_ context.Context, req structuredoutpu
func (c *repairClient) ConfiguredSecrets(context.Context) []string { return c.secrets }
func TestRepairBranchesRetargetAndPreserveOtherApprovals(t *testing.T) {
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":2,"action":"revise","correction":{"id":2,"original_text":"uh ","corrected_text":""}}`}}
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":2,"decision":{"action":"revise","correction":{"id":2,"original_text":"uh ","corrected_text":""}}}`}}
candidates := []proposals.EnrichedCorrectionProposal{
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 4}},
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 0, OriginalText: "x", CorrectedText: "y"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 5}},
@@ -89,7 +89,7 @@ func TestRepairRequestCarriesLiteralPreview(t *testing.T) {
{"blank result", "uh", "uh", "", "", proposals.ReplacementPolicyRequireUnique, false},
} {
t.Run(tc.name, func(t *testing.T) {
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`}}
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`}}
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 81, OriginalText: tc.original, CorrectedText: tc.replacement}}
input := validateSectionCandidatesInput{
Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}},
@@ -130,8 +130,8 @@ func TestRepairRequestCarriesLiteralPreview(t *testing.T) {
func TestRepairRequestRefreshesPreviewAfterRetarget(t *testing.T) {
client := &repairClient{t: t, outputs: []string{
`{"proposal_ref":1,"action":"revise","correction":{"id":2,"original_text":"uh ","corrected_text":""}}`,
`{"proposal_ref":1,"action":"withdraw","correction":null}`,
`{"proposal_ref":1,"decision":{"action":"revise","correction":{"id":2,"original_text":"uh ","corrected_text":""}}}`,
`{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`,
}}
reject := fakeValidator{name: "editorial_review", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
return validators.Result{Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Disposition: validators.DispositionRejected, ReasonCode: "llm_rejected", Message: "The context does not support this edit."}}}, nil
@@ -172,10 +172,10 @@ func TestRepairBranchTerminalOutcomes(t *testing.T) {
outcome string
calls int
}{
{"withdraw", []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`}, nil, 1, "withdrawn", 1},
{"exhausted", []string{`{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":"absent","corrected_text":"dog"}}`, `{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":"absent","corrected_text":"dog"}}`}, nil, 2, "exhausted", 2},
{"no-effect revision", []string{`{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":"cat","corrected_text":"cat"}}`}, nil, 2, "rejected", 1},
{"wrong reference", []string{`{"proposal_ref":2,"action":"withdraw","correction":null}`}, nil, 1, "unavailable", 1},
{"withdraw", []string{`{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`}, nil, 1, "withdrawn", 1},
{"exhausted", []string{`{"proposal_ref":1,"decision":{"action":"revise","correction":{"id":1,"original_text":"absent","corrected_text":"dog"}}}`, `{"proposal_ref":1,"decision":{"action":"revise","correction":{"id":1,"original_text":"absent","corrected_text":"dog"}}}`}, nil, 2, "exhausted", 2},
{"no-effect revision", []string{`{"proposal_ref":1,"decision":{"action":"revise","correction":{"id":1,"original_text":"cat","corrected_text":"cat"}}}`}, nil, 2, "rejected", 1},
{"wrong reference", []string{`{"proposal_ref":2,"decision":{"action":"withdraw","correction":null}}`}, nil, 1, "unavailable", 1},
{"provider failure", nil, &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonProviderFailed, Message: "provider failed"}, 1, "unavailable", 1},
} {
t.Run(tc.name, func(t *testing.T) {
@@ -206,8 +206,8 @@ func (m repairModule) RepairPromptID() string { return "repair" }
func TestRunnerAppliesRevisionsInOriginalOrder(t *testing.T) {
candidates := []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, {TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "fox"}, {TargetSegmentID: 1, OriginalText: "bird", CorrectedText: "owl"}}
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":2,"action":"revise","correction":{"id":1,"original_text":"cat","corrected_text":"fox"}}`,
`{"proposal_ref":3,"action":"revise","correction":{"id":1,"original_text":"bird","corrected_text":"owl"}}`,
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":2,"decision":{"action":"revise","correction":{"id":1,"original_text":"cat","corrected_text":"fox"}}}`,
`{"proposal_ref":3,"decision":{"action":"revise","correction":{"id":1,"original_text":"bird","corrected_text":"owl"}}}`,
}}
module := repairModule{outcomeModule{fakeModule: fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, rejectUntilRepair(0)}}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
return contracts.ProposalResult{Proposals: candidates, SegmentReferences: []int{1, 1, 1}, Exchange: &structuredoutput.Exchange{}}, nil
@@ -250,7 +250,7 @@ func TestRepairRevalidatesEarlierApprovalAndRetainsUnrelatedApprovalOnFailure(t
}
return validators.Result{Decisions: decisions}, nil
}}
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":2,"action":"revise","correction":{"id":1,"original_text":"bird","corrected_text":"owl"}}`}}
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":2,"decision":{"action":"revise","correction":{"id":1,"original_text":"bird","corrected_text":"owl"}}}`}}
if tc.failure {
client.err = &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonProviderFailed, Message: "failed"}
}
@@ -289,7 +289,7 @@ func TestRepairInterruptionRetainsAttempt(t *testing.T) {
}
func TestRepairArtifactsAreUniqueAndRedacted(t *testing.T) {
client := &repairClient{t: t, secrets: []string{"synthetic-secret"}, outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`, `{"proposal_ref":2,"action":"withdraw","correction":null}`}}
client := &repairClient{t: t, secrets: []string{"synthetic-secret"}, outputs: []string{`{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`, `{"proposal_ref":2,"decision":{"action":"withdraw","correction":null}}`}}
candidates := []proposals.EnrichedCorrectionProposal{
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}},
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "fox"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}},
@@ -322,7 +322,7 @@ func TestRepairArtifactsAreUniqueAndRedacted(t *testing.T) {
}
func TestRepairArtifactFailureDoesNotChangeOutcome(t *testing.T) {
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`}}
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`}}
path := filepath.Join(t.TempDir(), "file")
if err := os.WriteFile(path, []byte("x"), 0600); err != nil {
t.Fatal(err)
@@ -356,7 +356,7 @@ func TestNoEffectAndUnavailableOnlyDoNotStartRepair(t *testing.T) {
}
func TestRepairFeedbackRedactsValidationCredentials(t *testing.T) {
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`}}
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`}}
validation := &repairClient{t: t, secrets: []string{"validation-secret"}}
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, ValidationLLMClient: validation, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}, DiagnosticsDir: t.TempDir()}