Give each correction repair branch its own outcome
This commit is contained in:
@@ -49,7 +49,7 @@ func actionable(index int, rejected []ValidatorRejectedChange) bool {
|
||||
|
||||
// repairCandidates branches each rejected proposal from the same section exchange.
|
||||
func repairCandidates(ctx context.Context, input validateSectionCandidatesInput, references []int, initial validateSectionCandidatesResult, promptID string) (validateSectionCandidatesResult, []RepairOutcome, error) {
|
||||
if input.Spec.Settings.LLM.CorrectionRepairAttempts <= 0 {
|
||||
if input.Spec.Settings.LLM.CorrectionRepairAttempts <= 0 || len(initial.rejected) == 0 {
|
||||
return initial, nil, nil
|
||||
}
|
||||
if input.Section == nil {
|
||||
@@ -61,19 +61,12 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
||||
}
|
||||
superseded := make(map[int]bool)
|
||||
var branches []RepairOutcome
|
||||
var repairOps []outcomes.Operation
|
||||
debug := input.Interactions != nil
|
||||
section := 0
|
||||
if input.SectionIndex != nil {
|
||||
section = *input.SectionIndex
|
||||
}
|
||||
finish := func(err error) (validateSectionCandidatesResult, []RepairOutcome, error) {
|
||||
if len(superseded) > 0 {
|
||||
initial.filtered = slices.DeleteFunc(initial.filtered, func(item outcomes.FilteredProposal) bool {
|
||||
return superseded[item.ProposalIndex]
|
||||
})
|
||||
}
|
||||
initial.operations = append(initial.operations, repairOps...)
|
||||
return initial, branches, err
|
||||
}
|
||||
|
||||
@@ -82,178 +75,211 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
||||
if !actionable(original.ProposalIndex, rejected) {
|
||||
continue
|
||||
}
|
||||
branch := RepairOutcome{ProposalIndex: original.ProposalIndex, Outcome: "interrupted"}
|
||||
if debug {
|
||||
branch.Original, branch.Final = original, original
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
branches = append(branches, branch)
|
||||
return finish(err)
|
||||
}
|
||||
if input.Spec.RepairClient == nil || input.Exchange == nil || promptID == "" {
|
||||
branches = append(branches, branch)
|
||||
return finish(fmt.Errorf("correction repair setup is unavailable"))
|
||||
}
|
||||
localRef := 0
|
||||
if position < len(references) {
|
||||
localRef = references[position]
|
||||
} else {
|
||||
if pos, ok := input.Section.Position(original.TargetSegmentID); ok {
|
||||
localRef = pos + 1
|
||||
}
|
||||
} else if pos, ok := input.Section.Position(original.TargetSegmentID); ok {
|
||||
localRef = pos + 1
|
||||
}
|
||||
current := original
|
||||
findings := repairFindings(original.ProposalIndex, rejected)
|
||||
coordinator, ok := input.Spec.RepairClient.(structuredoutput.Coordinator)
|
||||
if !ok {
|
||||
return finish(fmt.Errorf("correction repair coordinator is unavailable"))
|
||||
}
|
||||
buildRequest := func() (*structuredoutput.Request, error) {
|
||||
request := proposal_generation.RepairRequest{
|
||||
ProposalRef: position + 1,
|
||||
Correction: proposal_generation.StructuredCorrectionProposal{SegmentReference: localRef, OriginalText: current.OriginalText, CorrectedText: current.CorrectedText},
|
||||
Findings: findings,
|
||||
}
|
||||
if pos, ok := input.Section.Position(current.TargetSegmentID); ok {
|
||||
segment := &input.WorkingTranscript.Segments[pos]
|
||||
request.OriginalSegmentText = &segment.Text
|
||||
preview := proposals.PreviewProposalForSegment(segment, current.CorrectionProposal, input.Policy)
|
||||
if preview.Applicable {
|
||||
request.CorrectedSegmentText = &preview.CorrectedSegmentText
|
||||
}
|
||||
}
|
||||
payload, err := request.JSON()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
proposalIndex, attemptNumber := original.ProposalIndex, branch.AttemptCount+1
|
||||
return &structuredoutput.Request{PromptID: promptID, Variables: map[string]string{"RepairRequestJSON": payload}, Schema: responseschema.CorrectionRepairKey, Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex, ProposalIndex: &proposalIndex, RepairAttempt: &attemptNumber}}, nil
|
||||
}
|
||||
exhausted, coordinationErr := coordinator.Coordinate(ctx, input.Exchange, input.Spec.Settings.LLM.CorrectionRepairAttempts, func(ctx context.Context, turn structuredoutput.Turn) (*structuredoutput.Request, error) {
|
||||
if turn.Response == nil {
|
||||
return buildRequest()
|
||||
}
|
||||
completion, callErr := *turn.Response, turn.Err
|
||||
proposalIndex, attemptNumber := original.ProposalIndex, branch.AttemptCount+1
|
||||
var response proposal_generation.RepairResponse
|
||||
|
||||
operation := outcomes.Operation{
|
||||
Reference: outcomes.Reference{
|
||||
StageIndex: &input.Spec.Index, StageID: input.Spec.ID, ModuleKey: input.Spec.ModuleKey,
|
||||
ModuleInstance: input.Spec.InstanceName, SectionIndex: section, Kind: outcomes.OperationCorrectionRepair,
|
||||
ProposalIndex: &proposalIndex, RepairAttempt: &attemptNumber,
|
||||
},
|
||||
Counts: completion.Execution, ProposalIndices: []int{proposalIndex},
|
||||
}
|
||||
finishAttempt := func(err error, revision *proposals.EnrichedCorrectionProposal, validated validateSectionCandidatesResult) {
|
||||
branch.AttemptCount++
|
||||
repairOps = append(repairOps, operation)
|
||||
if !debug {
|
||||
return
|
||||
}
|
||||
record := RepairAttempt{Decisions: validated.decisions, FailureReason: operation.FailureReason}
|
||||
if revision != nil {
|
||||
snapshot := *revision
|
||||
record.Revision = &snapshot
|
||||
}
|
||||
name := fmt.Sprintf("%s:section-%04d:proposal-%04d:attempt-%04d:repair", input.Spec.InstanceName, section, proposalIndex, attemptNumber)
|
||||
var failure any
|
||||
if err != nil {
|
||||
failure = map[string]string{"error": err.Error()}
|
||||
}
|
||||
record.Artifacts, _ = input.Interactions.WriteInteraction(structuredoutput.Interaction{
|
||||
Name: name,
|
||||
Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex, ProposalIndex: &proposalIndex, RepairAttempt: &attemptNumber},
|
||||
RequestMetadata: map[string]any{"stage_index": input.Spec.Index, "stage_id": input.Spec.ID, "proposal_index": proposalIndex, "repair_attempt": attemptNumber, "prompt_id": promptID, "prompt": completion.Prompt, "feedback": findings},
|
||||
RequestPayload: completion.Prepared,
|
||||
ResponsePayload: map[string]any{"decoded": response, "final_content": completion.FinalContent, "revision": record.Revision, "decisions": record.Decisions, "rejections": validated.rejected},
|
||||
ErrorPayload: failure,
|
||||
})
|
||||
branch.Attempts = append(branch.Attempts, record)
|
||||
}
|
||||
if callErr != nil {
|
||||
structuredoutput.RecordFailure(&operation, callErr)
|
||||
finishAttempt(callErr, nil, validateSectionCandidatesResult{})
|
||||
if operation.FailureClass == outcomes.FailureRecoverable {
|
||||
branch.Outcome = "unavailable"
|
||||
return nil, nil
|
||||
}
|
||||
return nil, callErr
|
||||
}
|
||||
var err error
|
||||
response, err = proposal_generation.DecodeRepairResponse(completion.Content, position+1)
|
||||
if err != nil {
|
||||
structuredoutput.RecordFailure(&operation, err)
|
||||
finishAttempt(err, nil, validateSectionCandidatesResult{})
|
||||
branch.Outcome = "unavailable"
|
||||
return nil, nil
|
||||
}
|
||||
operation.Succeeded = true
|
||||
operation.RecoveryReasons = outcomes.ExecutionRecoveries(operation.Counts)
|
||||
if response.Decision.Action == "withdraw" {
|
||||
finishAttempt(nil, nil, validateSectionCandidatesResult{})
|
||||
branch.Outcome = "withdrawn"
|
||||
return nil, nil
|
||||
}
|
||||
revision := response.Decision.Correction
|
||||
localRef = revision.SegmentReference
|
||||
current = original
|
||||
current.CorrectionProposal = proposals.CorrectionProposal{TargetSegmentID: proposals.SegmentIDAtReference(input.WorkingTranscript, localRef), OriginalText: revision.OriginalText, CorrectedText: revision.CorrectedText}
|
||||
if debug {
|
||||
branch.Final = current
|
||||
}
|
||||
next := input
|
||||
next.RepairProposalIndex, next.RepairAttempt = &proposalIndex, &attemptNumber
|
||||
next.SectionEnriched = []proposals.EnrichedCorrectionProposal{current}
|
||||
validated, validationErr := validateSectionCandidates(ctx, next)
|
||||
if len(validated.filtered) > 0 {
|
||||
// Each repair response contains only one proposal.
|
||||
operation.Filtered = validated.filtered[:1]
|
||||
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryProposalsFiltered)
|
||||
}
|
||||
for i := range validated.operations {
|
||||
proposalIndex, attemptNumber := original.ProposalIndex, attemptNumber
|
||||
validated.operations[i].Reference.ProposalIndex = &proposalIndex
|
||||
validated.operations[i].Reference.RepairAttempt = &attemptNumber
|
||||
}
|
||||
approved := validationErr == nil && len(validated.approved) == 1
|
||||
if approved {
|
||||
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryCorrectionRevised)
|
||||
}
|
||||
finishAttempt(validationErr, ¤t, validated)
|
||||
initial.operations = append(initial.operations, validated.operations...)
|
||||
if validationErr != nil {
|
||||
return nil, validationErr
|
||||
}
|
||||
if approved {
|
||||
branch.Outcome = "approved"
|
||||
initial.approved = append(initial.approved, current)
|
||||
superseded[original.ProposalIndex] = true
|
||||
return nil, nil
|
||||
}
|
||||
if actionable(original.ProposalIndex, validated.rejected) {
|
||||
findings = repairFindings(original.ProposalIndex, validated.rejected)
|
||||
return buildRequest()
|
||||
}
|
||||
branch.Outcome = "unavailable"
|
||||
for _, d := range validated.rejected {
|
||||
if d.Disposition != validators.DispositionUnavailable {
|
||||
branch.Outcome = "rejected"
|
||||
return nil, nil
|
||||
}
|
||||
}
|
||||
return nil, nil
|
||||
result, err := runRepairBranch(ctx, repairBranchInput{
|
||||
validation: input,
|
||||
original: original,
|
||||
proposalRef: position + 1,
|
||||
segmentRef: localRef,
|
||||
rejected: rejected,
|
||||
promptID: promptID,
|
||||
})
|
||||
if exhausted {
|
||||
branch.Outcome = "exhausted"
|
||||
branches = append(branches, result.outcome)
|
||||
initial.operations = append(initial.operations, result.operations...)
|
||||
if result.approved != nil {
|
||||
initial.approved = append(initial.approved, *result.approved)
|
||||
superseded[original.ProposalIndex] = true
|
||||
}
|
||||
if coordinationErr != nil {
|
||||
branches = append(branches, branch)
|
||||
return finish(coordinationErr)
|
||||
if err != nil {
|
||||
return finish(err)
|
||||
}
|
||||
|
||||
branches = append(branches, branch)
|
||||
}
|
||||
return finish(nil)
|
||||
}
|
||||
|
||||
type repairBranchInput struct {
|
||||
validation validateSectionCandidatesInput
|
||||
original proposals.EnrichedCorrectionProposal
|
||||
proposalRef int
|
||||
segmentRef int
|
||||
rejected []ValidatorRejectedChange
|
||||
promptID string
|
||||
}
|
||||
|
||||
type repairBranchResult struct {
|
||||
outcome RepairOutcome
|
||||
operations []outcomes.Operation
|
||||
approved *proposals.EnrichedCorrectionProposal
|
||||
}
|
||||
|
||||
func runRepairBranch(ctx context.Context, branchInput repairBranchInput) (repairBranchResult, error) {
|
||||
input, original := branchInput.validation, branchInput.original
|
||||
result := repairBranchResult{outcome: RepairOutcome{ProposalIndex: original.ProposalIndex, Outcome: "interrupted"}}
|
||||
branch := &result.outcome
|
||||
debug := input.Interactions != nil
|
||||
if debug {
|
||||
branch.Original, branch.Final = original, original
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return result, err
|
||||
}
|
||||
if input.Spec.RepairCoordinator == nil || input.Exchange == nil || branchInput.promptID == "" {
|
||||
return result, &structuredoutput.Failure{Class: outcomes.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "correction repair exchange or setup is unavailable"}
|
||||
}
|
||||
section := 0
|
||||
if input.SectionIndex != nil {
|
||||
section = *input.SectionIndex
|
||||
}
|
||||
localRef := branchInput.segmentRef
|
||||
proposalRef := branchInput.proposalRef
|
||||
promptID := branchInput.promptID
|
||||
current := original
|
||||
findings := repairFindings(original.ProposalIndex, branchInput.rejected)
|
||||
buildRequest := func() (*structuredoutput.Request, error) {
|
||||
request := proposal_generation.RepairRequest{
|
||||
ProposalRef: proposalRef,
|
||||
Correction: proposal_generation.StructuredCorrectionProposal{SegmentReference: localRef, OriginalText: current.OriginalText, CorrectedText: current.CorrectedText},
|
||||
Findings: findings,
|
||||
}
|
||||
if pos, ok := input.Section.Position(current.TargetSegmentID); ok {
|
||||
segment := &input.WorkingTranscript.Segments[pos]
|
||||
request.OriginalSegmentText = &segment.Text
|
||||
preview := proposals.PreviewProposalForSegment(segment, current.CorrectionProposal, input.Policy)
|
||||
if preview.Applicable {
|
||||
request.CorrectedSegmentText = &preview.CorrectedSegmentText
|
||||
}
|
||||
}
|
||||
payload, err := request.JSON()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
proposalIndex, attemptNumber := original.ProposalIndex, branch.AttemptCount+1
|
||||
return &structuredoutput.Request{PromptID: promptID, Variables: map[string]string{"RepairRequestJSON": payload}, Schema: responseschema.CorrectionRepairKey, Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex, ProposalIndex: &proposalIndex, RepairAttempt: &attemptNumber}}, nil
|
||||
}
|
||||
exhausted, coordinationErr := input.Spec.RepairCoordinator.Coordinate(ctx, input.Exchange, input.Spec.Settings.LLM.CorrectionRepairAttempts, func(ctx context.Context, turn structuredoutput.Turn) (*structuredoutput.Request, error) {
|
||||
if turn.Response == nil {
|
||||
return buildRequest()
|
||||
}
|
||||
completion, callErr := turn.Response, turn.Err
|
||||
proposalIndex, attemptNumber := original.ProposalIndex, branch.AttemptCount+1
|
||||
var response proposal_generation.RepairResponse
|
||||
|
||||
operation := outcomes.Operation{
|
||||
Reference: outcomes.Reference{
|
||||
StageIndex: &input.Spec.Index, StageID: input.Spec.ID, ModuleKey: input.Spec.ModuleKey,
|
||||
ModuleInstance: input.Spec.InstanceName, SectionIndex: section, Kind: outcomes.OperationCorrectionRepair,
|
||||
ProposalIndex: &proposalIndex, RepairAttempt: &attemptNumber,
|
||||
},
|
||||
Counts: completion.Execution, ProposalIndices: []int{proposalIndex},
|
||||
}
|
||||
finishAttempt := func(err error, revision *proposals.EnrichedCorrectionProposal, validated validateSectionCandidatesResult) {
|
||||
branch.AttemptCount++
|
||||
result.operations = append(result.operations, operation)
|
||||
if !debug {
|
||||
return
|
||||
}
|
||||
record := RepairAttempt{Decisions: validated.decisions, FailureReason: operation.FailureReason}
|
||||
if revision != nil {
|
||||
snapshot := *revision
|
||||
record.Revision = &snapshot
|
||||
}
|
||||
name := fmt.Sprintf("%s:section-%04d:proposal-%04d:attempt-%04d:repair", input.Spec.InstanceName, section, proposalIndex, attemptNumber)
|
||||
var failure any
|
||||
if err != nil {
|
||||
failure = map[string]string{"error": err.Error()}
|
||||
}
|
||||
record.Artifacts, _ = input.Interactions.WriteInteraction(structuredoutput.Interaction{
|
||||
Name: name,
|
||||
Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex, ProposalIndex: &proposalIndex, RepairAttempt: &attemptNumber},
|
||||
RequestMetadata: map[string]any{"stage_index": input.Spec.Index, "stage_id": input.Spec.ID, "proposal_index": proposalIndex, "repair_attempt": attemptNumber, "prompt_id": promptID, "prompt": completion.Prompt, "feedback": findings},
|
||||
RequestPayload: completion.Prepared,
|
||||
ResponsePayload: map[string]any{"decoded": response, "final_content": completion.FinalContent, "revision": record.Revision, "decisions": record.Decisions, "rejections": validated.rejected},
|
||||
ErrorPayload: failure,
|
||||
})
|
||||
branch.Attempts = append(branch.Attempts, record)
|
||||
}
|
||||
if callErr != nil {
|
||||
structuredoutput.RecordFailure(&operation, callErr)
|
||||
finishAttempt(callErr, nil, validateSectionCandidatesResult{})
|
||||
if operation.FailureClass == outcomes.FailureRecoverable {
|
||||
branch.Outcome = "unavailable"
|
||||
return nil, nil
|
||||
}
|
||||
return nil, callErr
|
||||
}
|
||||
var err error
|
||||
response, err = proposal_generation.DecodeRepairResponse(completion.Content, proposalRef)
|
||||
if err != nil {
|
||||
structuredoutput.RecordFailure(&operation, err)
|
||||
finishAttempt(err, nil, validateSectionCandidatesResult{})
|
||||
branch.Outcome = "unavailable"
|
||||
return nil, nil
|
||||
}
|
||||
operation.Succeeded = true
|
||||
operation.RecoveryReasons = outcomes.ExecutionRecoveries(operation.Counts)
|
||||
if response.Decision.Action == "withdraw" {
|
||||
finishAttempt(nil, nil, validateSectionCandidatesResult{})
|
||||
branch.Outcome = "withdrawn"
|
||||
return nil, nil
|
||||
}
|
||||
revision := response.Decision.Correction
|
||||
localRef = revision.SegmentReference
|
||||
current = original
|
||||
current.CorrectionProposal = proposals.CorrectionProposal{TargetSegmentID: proposals.SegmentIDAtReference(input.WorkingTranscript, localRef), OriginalText: revision.OriginalText, CorrectedText: revision.CorrectedText}
|
||||
if debug {
|
||||
branch.Final = current
|
||||
}
|
||||
next := input
|
||||
next.RepairProposalIndex, next.RepairAttempt = &proposalIndex, &attemptNumber
|
||||
next.SectionEnriched = []proposals.EnrichedCorrectionProposal{current}
|
||||
validated, validationErr := validateSectionCandidates(ctx, next)
|
||||
if len(validated.filtered) > 0 {
|
||||
// Each repair response contains only one proposal.
|
||||
operation.Filtered = validated.filtered[:1]
|
||||
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryProposalsFiltered)
|
||||
}
|
||||
for i := range validated.operations {
|
||||
validated.operations[i].Reference.ProposalIndex = &proposalIndex
|
||||
validated.operations[i].Reference.RepairAttempt = &attemptNumber
|
||||
}
|
||||
approved := validationErr == nil && len(validated.approved) == 1
|
||||
if approved {
|
||||
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryCorrectionRevised)
|
||||
}
|
||||
result.operations = append(result.operations, validated.operations...)
|
||||
finishAttempt(validationErr, ¤t, validated)
|
||||
if validationErr != nil {
|
||||
return nil, validationErr
|
||||
}
|
||||
if approved {
|
||||
branch.Outcome = "approved"
|
||||
result.approved = ¤t
|
||||
return nil, nil
|
||||
}
|
||||
if actionable(original.ProposalIndex, validated.rejected) {
|
||||
findings = repairFindings(original.ProposalIndex, validated.rejected)
|
||||
return buildRequest()
|
||||
}
|
||||
branch.Outcome = "unavailable"
|
||||
for _, d := range validated.rejected {
|
||||
if d.Disposition != validators.DispositionUnavailable {
|
||||
branch.Outcome = "rejected"
|
||||
return nil, nil
|
||||
}
|
||||
}
|
||||
return nil, nil
|
||||
})
|
||||
if coordinationErr != nil {
|
||||
branch.Outcome = "interrupted"
|
||||
result.approved = nil
|
||||
} else if exhausted {
|
||||
branch.Outcome = "exhausted"
|
||||
}
|
||||
return result, coordinationErr
|
||||
}
|
||||
|
||||
@@ -34,26 +34,13 @@ type repairClient struct {
|
||||
err error
|
||||
}
|
||||
|
||||
func (c *repairClient) CompleteStructured(_ context.Context, req structuredoutput.Request, out any) (structuredoutput.Response, error) {
|
||||
|
||||
c.calls = append(c.calls, req)
|
||||
if c.err != nil {
|
||||
return structuredoutput.Response{}, c.err
|
||||
}
|
||||
value := c.outputs[len(c.calls)-1]
|
||||
if err := json.Unmarshal([]byte(value), out); err != nil {
|
||||
return structuredoutput.Response{}, err
|
||||
}
|
||||
return structuredoutput.Response{Prepared: &structuredoutput.PreparedSnapshot{Messages: []structuredoutput.Message{{Role: "user", Content: "synthetic-secret"}}}, Content: json.RawMessage(value), FinalContent: value, Exchange: &structuredoutput.Exchange{}}, nil
|
||||
}
|
||||
|
||||
func TestRepairBranchesRetargetAndPreserveOtherApprovals(t *testing.T) {
|
||||
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":2,"decision":{"action":"revise","correction":{"id":2,"original_text":"uh ","corrected_text":""}}}`}}
|
||||
candidates := []proposals.EnrichedCorrectionProposal{
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 4}},
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 0, OriginalText: "x", CorrectedText: "y"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 5}},
|
||||
}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}, {ID: 2, Text: "uh hello"}}}, Validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.NonEmptyResultingSegmentValidator{}}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", RepairCoordinator: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}, {ID: 2, Text: "uh hello"}}}, Validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.NonEmptyResultingSegmentValidator{}}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{}}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -90,7 +77,7 @@ func TestRepairRequestCarriesLiteralPreview(t *testing.T) {
|
||||
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`}}
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 81, OriginalText: tc.original, CorrectedText: tc.replacement}}
|
||||
input := validateSectionCandidatesInput{
|
||||
Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}},
|
||||
Spec: contracts.ModuleRunSpec{RepairCoordinator: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}},
|
||||
Policy: tc.policy,
|
||||
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 81, Text: tc.segment}}},
|
||||
Validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.NonEmptyResultingSegmentValidator{}, rejectUntilRepair()},
|
||||
@@ -137,7 +124,7 @@ func TestRepairRequestRefreshesPreviewAfterRetarget(t *testing.T) {
|
||||
return validators.Result{Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Disposition: validators.DispositionRejected, ReasonCode: "llm_rejected", Message: "The context does not support this edit."}}}, nil
|
||||
}}
|
||||
input := validateSectionCandidatesInput{
|
||||
Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 2}}},
|
||||
Spec: contracts.ModuleRunSpec{RepairCoordinator: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 2}}},
|
||||
Policy: proposals.ReplacementPolicyRequireUnique,
|
||||
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 81, Text: "cat"}, {ID: 104, Text: "uh hello"}}},
|
||||
Validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validatormetadata.Wrap(reject, validatormetadata.ExecutionClassDeterministic)},
|
||||
@@ -194,7 +181,7 @@ func TestRepairBranchTerminalOutcomes(t *testing.T) {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
client := &repairClient{t: t, outputs: tc.outputs, err: tc.err}
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: tc.budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairCoordinator: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: tc.budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -214,7 +201,7 @@ func TestRepairBranchTerminalOutcomes(t *testing.T) {
|
||||
}
|
||||
}
|
||||
if tc.calls == 2 && client.calls[1].Context.RepairAttempt == nil {
|
||||
t.Fatalf("history: %+v", client.calls[1].ContinueFrom)
|
||||
t.Fatalf("second attempt lacks diagnostic coordinates: %+v", client.calls[1].Context)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -224,6 +211,27 @@ type repairModule struct{ outcomeModule }
|
||||
|
||||
func (m repairModule) RepairPromptID() string { return "repair" }
|
||||
|
||||
func TestRunnerChecksRepairCapabilityBeforeSubmittingProposals(t *testing.T) {
|
||||
calls := 0
|
||||
module := repairModule{outcomeModule{fakeModule: fakeModule{key: "grammar"}, proposeResult: func(contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
calls++
|
||||
return contracts.ProposalResult{}, nil
|
||||
}}}
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}})
|
||||
out, err := r.Run(context.Background(), RunInput{
|
||||
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}},
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{{
|
||||
ModuleKey: "grammar", InstanceName: "grammar",
|
||||
Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}},
|
||||
}},
|
||||
})
|
||||
var failure *structuredoutput.Failure
|
||||
if !errors.As(err, &failure) || failure.Class != outcomes.FailureFatal || failure.Reason != outcomes.ReasonSetupFailure ||
|
||||
calls != 0 || len(out.WorkUnits) != 1 || out.WorkUnits[0].State != outcomes.WorkNotStarted {
|
||||
t.Fatalf("repair setup did not stop before dispatch: calls=%d work=%+v error=%v", calls, out.WorkUnits, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerAppliesRevisionsInOriginalOrder(t *testing.T) {
|
||||
candidates := []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, {TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "fox"}, {TargetSegmentID: 1, OriginalText: "bird", CorrectedText: "owl"}}
|
||||
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":2,"decision":{"action":"revise","correction":{"id":1,"original_text":"cat","corrected_text":"fox"}}}`,
|
||||
@@ -233,7 +241,7 @@ func TestRunnerAppliesRevisionsInOriginalOrder(t *testing.T) {
|
||||
return contracts.ProposalResult{Proposals: candidates, SegmentReferences: []int{1, 1, 1}, Exchange: &structuredoutput.Exchange{}}, nil
|
||||
}}}
|
||||
runner := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}})
|
||||
out, err := runner.Run(context.Background(), RunInput{Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat and bird"}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}}})
|
||||
out, err := runner.Run(context.Background(), RunInput{Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat and bird"}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", RepairCoordinator: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -274,7 +282,7 @@ func TestRepairRevalidatesEarlierApprovalAndRetainsUnrelatedApprovalOnFailure(t
|
||||
if tc.failure {
|
||||
client.err = &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonProviderFailed, Message: "failed"}
|
||||
}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat bird"}}}, Validators: []validators.Validator{check, rejectUntilRepair(0)}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairCoordinator: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat bird"}}}, Validators: []validators.Validator{check, rejectUntilRepair(0)}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{}}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -297,7 +305,7 @@ func TestRepairRevalidatesEarlierApprovalAndRetainsUnrelatedApprovalOnFailure(t
|
||||
func TestRepairInterruptionRetainsAttempt(t *testing.T) {
|
||||
client := &repairClient{t: t, err: &structuredoutput.Failure{Class: structuredoutput.FailureCanceled, Reason: outcomes.ReasonCanceled, Message: "canceled", Cause: context.Canceled}}
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairCoordinator: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -315,7 +323,7 @@ func TestRepairArtifactsAreUnique(t *testing.T) {
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "fox"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}},
|
||||
}
|
||||
section := 4
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar_1", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{rejectUntilRepair()}, SectionEnriched: candidates, SectionIndex: §ion, Exchange: &structuredoutput.Exchange{}, Interactions: llm.NewDiagnosticsWriter(t.TempDir())}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar_1", RepairCoordinator: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{rejectUntilRepair()}, SectionEnriched: candidates, SectionIndex: §ion, Exchange: &structuredoutput.Exchange{}, Interactions: llm.NewDiagnosticsWriter(t.TempDir())}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -373,7 +381,7 @@ func TestRepairArtifactFailureDoesNotChangeOutcome(t *testing.T) {
|
||||
}
|
||||
sink := llm.NewDiagnosticsWriter(path)
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}, Interactions: sink}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairCoordinator: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}, Interactions: sink}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -391,7 +399,7 @@ func TestNoEffectAndUnavailableOnlyDoNotStartRepair(t *testing.T) {
|
||||
{{ProposalIndex: 0, Disposition: validators.DispositionUnavailable, ReasonCode: "provider_failed"}},
|
||||
} {
|
||||
candidate := proposals.EnrichedCorrectionProposal{ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairCoordinator: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}}
|
||||
_, branches, err := repairCandidates(context.Background(), input, nil, validateSectionCandidatesResult{rejected: rejected}, "repair")
|
||||
if err != nil || len(branches) != 0 || len(client.calls) != 0 {
|
||||
t.Fatalf("branches=%+v calls=%d err=%v", branches, len(client.calls), err)
|
||||
|
||||
@@ -441,6 +441,19 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
repairPromptID := ""
|
||||
if input.Spec.Settings.LLM.CorrectionRepairAttempts > 0 {
|
||||
provider, ok := input.Module.(interface{ RepairPromptID() string })
|
||||
if !ok || input.Spec.RepairCoordinator == nil {
|
||||
return out, &structuredoutput.Failure{Class: outcomes.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "correction repair coordinator or prompt provider is unavailable"}
|
||||
}
|
||||
repairPromptID = provider.RepairPromptID()
|
||||
if repairPromptID == "" {
|
||||
return out, &structuredoutput.Failure{Class: outcomes.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "correction repair prompt is unavailable"}
|
||||
}
|
||||
input.Spec.CaptureProposalExchange = true
|
||||
}
|
||||
|
||||
runCtx, proposalResults, cancel := collectSectionProposals(ctx, input)
|
||||
defer cancel()
|
||||
|
||||
@@ -548,12 +561,8 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
||||
}
|
||||
validated, err := validateSectionCandidates(runCtx, validationInput)
|
||||
var repairs []RepairOutcome
|
||||
if err == nil && input.Spec.Settings.LLM.CorrectionRepairAttempts > 0 {
|
||||
promptID := ""
|
||||
if provider, ok := input.Module.(interface{ RepairPromptID() string }); ok {
|
||||
promptID = provider.RepairPromptID()
|
||||
}
|
||||
validated, repairs, err = repairCandidates(runCtx, validationInput, sectionResult.proposal.SegmentReferences, validated, promptID)
|
||||
if err == nil && repairPromptID != "" {
|
||||
validated, repairs, err = repairCandidates(runCtx, validationInput, sectionResult.proposal.SegmentReferences, validated, repairPromptID)
|
||||
}
|
||||
setErr(err)
|
||||
validationResults <- sectionValidationResult{
|
||||
|
||||
Reference in New Issue
Block a user