2026-11-11 04:29:18 +00:00
package runner
import (
"context"
"encoding/json"
"errors"
2026-11-15 01:25:55 +00:00
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
ck "go.maximumdirect.net/configkit"
"io"
"net/http"
2026-11-11 04:37:52 +00:00
"os"
"path/filepath"
"strings"
2026-11-11 04:29:18 +00:00
"testing"
2026-11-15 01:25:55 +00:00
"time"
2026-11-11 04:29:18 +00:00
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
2026-11-16 14:11:02 +00:00
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
2026-11-11 04:29:18 +00:00
)
type repairClient struct {
2026-11-17 04:29:02 +00:00
t * testing . T
outputs [ ] string
calls [ ] structuredoutput . Request
err error
capturePrepared func ( * structuredoutput . PreparedSnapshot )
2026-11-11 04:29:18 +00:00
}
func TestRepairBranchesRetargetAndPreserveOtherApprovals ( t * testing . T ) {
2026-11-15 02:41:32 +00:00
client := & repairClient { t : t , outputs : [ ] string { ` { "proposal_ref":2,"decision": { "action":"revise","correction": { "id":2,"original_text":"uh ","corrected_text":""}}} ` } }
2026-11-11 04:29:18 +00:00
candidates := [ ] proposals . EnrichedCorrectionProposal {
2026-11-11 17:05:52 +00:00
{ CorrectionProposal : proposals . CorrectionProposal { TargetSegmentID : 1 , OriginalText : "cat" , CorrectedText : "dog" } , ProposalMetadata : proposals . ProposalMetadata { ProposalIndex : 4 } } ,
{ CorrectionProposal : proposals . CorrectionProposal { TargetSegmentID : 0 , OriginalText : "x" , CorrectedText : "y" } , ProposalMetadata : proposals . ProposalMetadata { ProposalIndex : 5 } } ,
2026-11-11 04:29:18 +00:00
}
2026-11-16 14:36:07 +00:00
input := validateSectionCandidatesInput { Spec : contracts . ModuleRunSpec { ModuleKey : "grammar" , RepairCoordinator : client , Settings : config . Stage { LLM : config . LLM { CorrectionRepairAttempts : 1 } } } , Policy : proposals . ReplacementPolicyRequireUnique , WorkingTranscript : & schema . Transcript { Segments : [ ] schema . Segment { { ID : 1 , Text : "cat" } , { ID : 2 , Text : "uh hello" } } } , Validators : [ ] validators . Validator { validators . ProposalShapeValidator { } , validators . NoEffectValidator { } , validators . OriginalTextPresenceValidator { } , validators . NonEmptyResultingSegmentValidator { } } , SectionEnriched : candidates , Exchange : & structuredoutput . Exchange { } }
2026-11-11 04:29:18 +00:00
initial , err := validateSectionCandidates ( context . Background ( ) , input )
if err != nil {
t . Fatal ( err )
}
result , branches , err := repairCandidates ( context . Background ( ) , input , [ ] int { 1 , 9 } , initial , "repair" )
if err != nil {
t . Fatal ( err )
}
if len ( result . approved ) != 2 || result . approved [ 1 ] . ProposalIndex != 5 || result . approved [ 1 ] . TargetSegmentID != 2 || result . approved [ 1 ] . CorrectedText != "" || len ( branches ) != 1 || branches [ 0 ] . Outcome != "approved" || len ( result . filtered ) != 0 {
t . Fatalf ( "result=%+v branches=%+v" , result , branches )
}
var payload proposal_generation . RepairRequest
if err := json . Unmarshal ( [ ] byte ( client . calls [ 0 ] . Variables [ "RepairRequestJSON" ] ) , & payload ) ; err != nil {
t . Fatal ( err )
}
2026-11-11 17:42:30 +00:00
if payload . ProposalRef != 2 || payload . Correction . SegmentReference != 9 || payload . OriginalSegmentText != nil || payload . CorrectedSegmentText != nil {
2026-11-11 04:29:18 +00:00
t . Fatalf ( "request: %+v" , payload )
}
}
2026-11-11 17:42:30 +00:00
func TestRepairRequestCarriesLiteralPreview ( t * testing . T ) {
for _ , tc := range [ ] struct {
name , segment , original , replacement , result string
policy proposals . ReplacementPolicy
preview bool
} {
{ "deletion" , "uh hello" , "uh " , "" , "hello" , proposals . ReplacementPolicyRequireUnique , true } ,
{ "all occurrences" , "gestures and gestures" , "gestures" , "Jesters" , "Jesters and Jesters" , proposals . ReplacementPolicyReplaceAll , true } ,
{ "ambiguous span" , "gestures and gestures" , "gestures" , "Jesters" , "" , proposals . ReplacementPolicyRequireUnique , false } ,
{ "missing span" , "hello" , "absent" , "world" , "" , proposals . ReplacementPolicyRequireUnique , false } ,
{ "blank result" , "uh" , "uh" , "" , "" , proposals . ReplacementPolicyRequireUnique , false } ,
} {
t . Run ( tc . name , func ( t * testing . T ) {
2026-11-15 02:41:32 +00:00
client := & repairClient { t : t , outputs : [ ] string { ` { "proposal_ref":1,"decision": { "action":"withdraw","correction":null}} ` } }
2026-11-11 17:42:30 +00:00
candidate := proposals . EnrichedCorrectionProposal { CorrectionProposal : proposals . CorrectionProposal { TargetSegmentID : 81 , OriginalText : tc . original , CorrectedText : tc . replacement } }
input := validateSectionCandidatesInput {
2026-11-16 14:36:07 +00:00
Spec : contracts . ModuleRunSpec { RepairCoordinator : client , Settings : config . Stage { LLM : config . LLM { CorrectionRepairAttempts : 1 } } } ,
2026-11-11 17:42:30 +00:00
Policy : tc . policy ,
WorkingTranscript : & schema . Transcript { Segments : [ ] schema . Segment { { ID : 81 , Text : tc . segment } } } ,
2026-11-16 14:11:02 +00:00
Validators : [ ] validators . Validator { validators . ProposalShapeValidator { } , validators . NoEffectValidator { } , validators . OriginalTextPresenceValidator { } , validators . NonEmptyResultingSegmentValidator { } , rejectUntilRepair ( ) } ,
2026-11-11 17:42:30 +00:00
SectionEnriched : [ ] proposals . EnrichedCorrectionProposal { candidate } ,
2026-11-15 01:25:55 +00:00
Exchange : & structuredoutput . Exchange { } ,
2026-11-11 17:42:30 +00:00
}
initial , err := validateSectionCandidates ( context . Background ( ) , input )
if err != nil {
t . Fatal ( err )
}
_ , branches , err := repairCandidates ( context . Background ( ) , input , [ ] int { 1 } , initial , "repair" )
if err != nil || len ( client . calls ) != 1 || len ( branches ) != 1 || branches [ 0 ] . Outcome != "withdrawn" {
t . Fatalf ( "repair: branches=%+v calls=%d err=%v" , branches , len ( client . calls ) , err )
}
var payload proposal_generation . RepairRequest
if err := json . Unmarshal ( [ ] byte ( client . calls [ 0 ] . Variables [ "RepairRequestJSON" ] ) , & payload ) ; err != nil {
t . Fatal ( err )
}
if payload . OriginalSegmentText == nil || * payload . OriginalSegmentText != tc . segment || len ( payload . Findings ) == 0 {
t . Fatalf ( "missing source or findings: %+v" , payload )
}
if tc . preview {
if payload . CorrectedSegmentText == nil || * payload . CorrectedSegmentText != tc . result {
t . Fatalf ( "incorrect preview: %+v" , payload )
}
} else if payload . CorrectedSegmentText != nil {
t . Fatalf ( "inapplicable proposal has a preview: %+v" , payload )
}
if input . WorkingTranscript . Segments [ 0 ] . Text != tc . segment {
t . Fatal ( "repair preview changed the transcript" )
}
} )
}
}
func TestRepairRequestRefreshesPreviewAfterRetarget ( t * testing . T ) {
2026-11-15 01:25:55 +00:00
client := & repairClient { t : t , outputs : [ ] string {
2026-11-15 02:41:32 +00:00
` { "proposal_ref":1,"decision": { "action":"revise","correction": { "id":2,"original_text":"uh ","corrected_text":""}}} ` ,
` { "proposal_ref":1,"decision": { "action":"withdraw","correction":null}} ` ,
2026-11-11 17:42:30 +00:00
} }
2026-11-16 04:09:25 +00:00
var validatedIDs [ ] int
2026-11-16 14:11:02 +00:00
reject := fakeValidator { name : "editorial_review" , validateF : func ( req validators . Request ) ( validators . Result , error ) {
2026-11-16 04:09:25 +00:00
validatedIDs = append ( validatedIDs , req . CandidateProposal [ 0 ] . TargetSegmentID )
2026-11-11 17:42:30 +00:00
return validators . Result { Decisions : [ ] validators . Decision { { ProposalIndex : req . CandidateProposal [ 0 ] . ProposalIndex , Disposition : validators . DispositionRejected , ReasonCode : "llm_rejected" , Message : "The context does not support this edit." } } } , nil
} }
input := validateSectionCandidatesInput {
2026-11-16 14:36:07 +00:00
Spec : contracts . ModuleRunSpec { RepairCoordinator : client , Settings : config . Stage { LLM : config . LLM { CorrectionRepairAttempts : 2 } } } ,
2026-11-11 17:42:30 +00:00
Policy : proposals . ReplacementPolicyRequireUnique ,
WorkingTranscript : & schema . Transcript { Segments : [ ] schema . Segment { { ID : 81 , Text : "cat" } , { ID : 104 , Text : "uh hello" } } } ,
2026-11-16 14:11:02 +00:00
Validators : [ ] validators . Validator { validators . ProposalShapeValidator { } , validators . NoEffectValidator { } , validators . OriginalTextPresenceValidator { } , validatormetadata . Wrap ( reject , validatormetadata . ExecutionClassDeterministic ) } ,
2026-11-11 17:42:30 +00:00
SectionEnriched : [ ] proposals . EnrichedCorrectionProposal { { CorrectionProposal : proposals . CorrectionProposal { TargetSegmentID : 81 , OriginalText : "cat" , CorrectedText : "dog" } } } ,
2026-11-15 01:25:55 +00:00
Exchange : & structuredoutput . Exchange { } ,
2026-11-16 14:11:02 +00:00
Interactions : llm . NewDiagnosticsWriter ( t . TempDir ( ) ) ,
2026-11-11 17:42:30 +00:00
}
initial , err := validateSectionCandidates ( context . Background ( ) , input )
if err != nil {
t . Fatal ( err )
}
_ , branches , err := repairCandidates ( context . Background ( ) , input , [ ] int { 1 } , initial , "repair" )
if err != nil || len ( client . calls ) != 2 || len ( branches ) != 1 || branches [ 0 ] . Outcome != "withdrawn" {
t . Fatalf ( "repair: branches=%+v calls=%d err=%v" , branches , len ( client . calls ) , err )
}
2026-11-16 04:09:25 +00:00
if len ( validatedIDs ) != 2 || validatedIDs [ 0 ] != 81 || validatedIDs [ 1 ] != 104 {
t . Fatalf ( "validated segment IDs: %v" , validatedIDs )
}
2026-11-16 14:11:02 +00:00
for _ , decisions := range [ ] [ ] ValidatorDecisionRecord { initial . decisions , branches [ 0 ] . Attempts [ 0 ] . Decisions } {
if len ( decisions ) == 0 {
t . Fatal ( "missing decision evidence" )
}
record := decisions [ len ( decisions ) - 1 ]
if record . ValidatorName != "editorial_review" || record . ExecutionClass != validatormetadata . ExecutionClassDeterministic {
t . Fatalf ( "lost execution class override: %+v" , record )
}
}
2026-11-11 17:42:30 +00:00
for i , want := range [ ] struct { before , after string } { { "cat" , "dog" } , { "uh hello" , "hello" } } {
var payload proposal_generation . RepairRequest
if err := json . Unmarshal ( [ ] byte ( client . calls [ i ] . Variables [ "RepairRequestJSON" ] ) , & payload ) ; err != nil {
t . Fatal ( err )
}
2026-11-17 04:22:20 +00:00
if payload . ProposalRef != 1 || int ( payload . Correction . SegmentReference ) != i + 1 || payload . OriginalSegmentText == nil || * payload . OriginalSegmentText != want . before || payload . CorrectedSegmentText == nil || * payload . CorrectedSegmentText != want . after {
2026-11-11 17:42:30 +00:00
t . Fatalf ( "attempt %d did not describe its current proposal: %+v" , i + 1 , payload )
}
}
}
2026-11-11 04:29:18 +00:00
func TestRepairBranchTerminalOutcomes ( t * testing . T ) {
for _ , tc := range [ ] struct {
name string
outputs [ ] string
err error
budget int
outcome string
calls int
} {
2026-11-15 02:41:32 +00:00
{ "withdraw" , [ ] string { ` { "proposal_ref":1,"decision": { "action":"withdraw","correction":null}} ` } , nil , 1 , "withdrawn" , 1 } ,
{ "exhausted" , [ ] string { ` { "proposal_ref":1,"decision": { "action":"revise","correction": { "id":1,"original_text":"absent","corrected_text":"dog"}}} ` , ` { "proposal_ref":1,"decision": { "action":"revise","correction": { "id":1,"original_text":"absent","corrected_text":"dog"}}} ` } , nil , 2 , "exhausted" , 2 } ,
{ "no-effect revision" , [ ] string { ` { "proposal_ref":1,"decision": { "action":"revise","correction": { "id":1,"original_text":"cat","corrected_text":"cat"}}} ` } , nil , 2 , "rejected" , 1 } ,
{ "wrong reference" , [ ] string { ` { "proposal_ref":2,"decision": { "action":"withdraw","correction":null}} ` } , nil , 1 , "unavailable" , 1 } ,
2026-11-16 14:27:54 +00:00
{ "provider failure" , nil , & structuredoutput . Failure { Class : structuredoutput . FailureRecoverable , Reason : outcomes . ReasonProviderFailed , Message : "provider failed" , Cause : errors . New ( "underlying provider error" ) } , 1 , "unavailable" , 1 } ,
2026-11-11 04:29:18 +00:00
} {
t . Run ( tc . name , func ( t * testing . T ) {
2026-11-15 01:25:55 +00:00
client := & repairClient { t : t , outputs : tc . outputs , err : tc . err }
2026-11-11 17:05:52 +00:00
candidate := proposals . EnrichedCorrectionProposal { CorrectionProposal : proposals . CorrectionProposal { TargetSegmentID : 1 , OriginalText : "cat" , CorrectedText : "dog" } , ProposalMetadata : proposals . ProposalMetadata { ProposalIndex : 0 } }
2026-11-16 14:36:07 +00:00
input := validateSectionCandidatesInput { Spec : contracts . ModuleRunSpec { RepairCoordinator : client , Settings : config . Stage { LLM : config . LLM { CorrectionRepairAttempts : tc . budget } } } , Policy : proposals . ReplacementPolicyRequireUnique , WorkingTranscript : & schema . Transcript { Segments : [ ] schema . Segment { { ID : 1 , Text : "cat" } } } , Validators : [ ] validators . Validator { validators . ProposalShapeValidator { } , validators . NoEffectValidator { } , validators . OriginalTextPresenceValidator { } , rejectUntilRepair ( ) } , SectionEnriched : [ ] proposals . EnrichedCorrectionProposal { candidate } , Exchange : & structuredoutput . Exchange { } }
2026-11-11 04:29:18 +00:00
initial , err := validateSectionCandidates ( context . Background ( ) , input )
if err != nil {
t . Fatal ( err )
}
2026-11-16 14:27:54 +00:00
result , branches , err := repairCandidates ( context . Background ( ) , input , [ ] int { 1 } , initial , "repair" )
2026-11-11 04:29:18 +00:00
if err != nil {
t . Fatal ( err )
}
2026-11-11 12:31:08 +00:00
if len ( branches ) != 1 || branches [ 0 ] . Outcome != tc . outcome || len ( client . calls ) != tc . calls || branches [ 0 ] . AttemptCount != tc . calls || len ( branches [ 0 ] . Attempts ) != 0 {
2026-11-11 04:29:18 +00:00
t . Fatalf ( "branches=%+v calls=%d" , branches , len ( client . calls ) )
}
2026-11-16 14:27:54 +00:00
if tc . err != nil {
operation := result . operations [ len ( result . operations ) - 1 ]
if operation . FailureClass != outcomes . FailureRecoverable ||
operation . FailureReason != outcomes . ReasonProviderFailed || ! errors . Is ( operation . Cause , tc . err ) {
t . Fatalf ( "repair lost typed failure facts: %+v" , operation )
}
}
2026-11-15 01:25:55 +00:00
if tc . calls == 2 && client . calls [ 1 ] . Context . RepairAttempt == nil {
2026-11-16 14:36:07 +00:00
t . Fatalf ( "second attempt lacks diagnostic coordinates: %+v" , client . calls [ 1 ] . Context )
2026-11-11 04:29:18 +00:00
}
} )
}
}
type repairModule struct { outcomeModule }
func ( m repairModule ) RepairPromptID ( ) string { return "repair" }
2026-11-16 14:36:07 +00:00
func TestRunnerChecksRepairCapabilityBeforeSubmittingProposals ( t * testing . T ) {
calls := 0
module := repairModule { outcomeModule { fakeModule : fakeModule { key : "grammar" } , proposeResult : func ( contracts . ProposalRequest ) ( contracts . ProposalResult , error ) {
calls ++
return contracts . ProposalResult { } , nil
} } }
r := New ( fakeFactory { modules : map [ string ] contracts . TranscriptModule { "grammar" : module } } )
out , err := r . Run ( context . Background ( ) , RunInput {
Transcript : & schema . Transcript { Segments : [ ] schema . Segment { { ID : 1 , Text : "cat" } } } ,
ModuleSpecs : [ ] contracts . ModuleRunSpec { {
ModuleKey : "grammar" , InstanceName : "grammar" ,
Settings : config . Stage { LLM : config . LLM { CorrectionRepairAttempts : 1 } } ,
} } ,
} )
var failure * structuredoutput . Failure
if ! errors . As ( err , & failure ) || failure . Class != outcomes . FailureFatal || failure . Reason != outcomes . ReasonSetupFailure ||
calls != 0 || len ( out . WorkUnits ) != 1 || out . WorkUnits [ 0 ] . State != outcomes . WorkNotStarted {
t . Fatalf ( "repair setup did not stop before dispatch: calls=%d work=%+v error=%v" , calls , out . WorkUnits , err )
}
}
2026-11-11 04:29:18 +00:00
func TestRunnerAppliesRevisionsInOriginalOrder ( t * testing . T ) {
2026-11-11 17:05:52 +00:00
candidates := [ ] proposals . CorrectionProposal { { TargetSegmentID : 1 , OriginalText : "cat" , CorrectedText : "dog" } , { TargetSegmentID : 1 , OriginalText : "cat" , CorrectedText : "fox" } , { TargetSegmentID : 1 , OriginalText : "bird" , CorrectedText : "owl" } }
2026-11-15 02:41:32 +00:00
client := & repairClient { t : t , outputs : [ ] string { ` { "proposal_ref":2,"decision": { "action":"revise","correction": { "id":1,"original_text":"cat","corrected_text":"fox"}}} ` ,
` { "proposal_ref":3,"decision": { "action":"revise","correction": { "id":1,"original_text":"bird","corrected_text":"owl"}}} ` ,
2026-11-11 12:29:42 +00:00
} }
2026-11-16 14:11:02 +00:00
module := repairModule { outcomeModule { fakeModule : fakeModule { key : "grammar" , policy : proposals . ReplacementPolicyRequireUnique , validators : [ ] validators . Validator { validators . ProposalShapeValidator { } , validators . NoEffectValidator { } , validators . OriginalTextPresenceValidator { } , rejectUntilRepair ( 0 ) } } , proposeResult : func ( req contracts . ProposalRequest ) ( contracts . ProposalResult , error ) {
2026-11-15 01:25:55 +00:00
return contracts . ProposalResult { Proposals : candidates , SegmentReferences : [ ] int { 1 , 1 , 1 } , Exchange : & structuredoutput . Exchange { } } , nil
2026-11-11 04:29:18 +00:00
} } }
runner := New ( fakeFactory { modules : map [ string ] contracts . TranscriptModule { "grammar" : module } } )
2026-11-16 14:36:07 +00:00
out , err := runner . Run ( context . Background ( ) , RunInput { Transcript : & schema . Transcript { Segments : [ ] schema . Segment { { ID : 1 , Text : "cat and bird" } } } , ModuleSpecs : [ ] contracts . ModuleRunSpec { { ModuleKey : "grammar" , InstanceName : "grammar" , RepairCoordinator : client , Settings : config . Stage { LLM : config . LLM { CorrectionRepairAttempts : 1 } } } } } )
2026-11-11 04:29:18 +00:00
if err != nil {
t . Fatal ( err )
}
2026-11-11 12:29:42 +00:00
if out . FinalTranscript . Segments [ 0 ] . Text != "dog and owl" || len ( out . WorkUnits ) != 1 || len ( out . WorkUnits [ 0 ] . Repairs ) != 2 || out . WorkUnits [ 0 ] . Repairs [ 0 ] . Outcome != "approved_skipped" || out . WorkUnits [ 0 ] . Repairs [ 1 ] . Outcome != "applied" || out . WorkUnits [ 0 ] . CorrectionsApplied != 2 || len ( out . ModuleResults [ 0 ] . SkippedChanges ) != 1 || out . ModuleResults [ 0 ] . SkippedChanges [ 0 ] . ProposalIndex != 1 {
t . Fatalf ( "output=%+v" , out )
2026-11-11 04:29:18 +00:00
}
2026-11-11 12:29:42 +00:00
operations := out . WorkUnits [ 0 ] . Operations
if len ( operations ) != 2 {
t . Fatalf ( "repair operations: %+v" , operations )
}
for _ , op := range operations {
if op . Reference . Kind != outcomes . OperationCorrectionRepair || len ( op . RecoveryReasons ) != 1 || op . RecoveryReasons [ 0 ] != outcomes . RecoveryCorrectionRevised {
t . Fatalf ( "repair operation: %+v" , op )
}
2026-11-11 04:31:44 +00:00
}
2026-11-11 04:29:18 +00:00
}
2026-11-17 04:34:31 +00:00
func TestRunnerAttributesApplicationFilteringToEachRepair ( t * testing . T ) {
client := & repairClient { t : t , outputs : [ ] string {
` { "proposal_ref":1,"decision": { "action":"revise","correction": { "id":1,"original_text":"missing","corrected_text":"new"}}} ` ,
` { "proposal_ref":2,"decision": { "action":"revise","correction": { "id":1,"original_text":"absent","corrected_text":"new"}}} ` ,
} }
module := repairModule { outcomeModule {
fakeModule : fakeModule { key : "grammar" , policy : proposals . ReplacementPolicyRequireUnique ,
validators : [ ] validators . Validator { rejectUntilRepair ( ) } } ,
proposeResult : func ( contracts . ProposalRequest ) ( contracts . ProposalResult , error ) {
return contracts . ProposalResult {
Proposals : [ ] proposals . CorrectionProposal {
{ TargetSegmentID : 1 , OriginalText : "cat" , CorrectedText : "dog" } ,
{ TargetSegmentID : 1 , OriginalText : "bird" , CorrectedText : "owl" } ,
} ,
SegmentReferences : [ ] int { 1 , 1 } ,
Exchange : & structuredoutput . Exchange { } ,
Operations : [ ] outcomes . Operation { { Reference : outcomes . Reference { Kind : outcomes . OperationProposal } , Succeeded : true } } ,
} , nil
} ,
} }
r := New ( fakeFactory { modules : map [ string ] contracts . TranscriptModule { "grammar" : module } } )
out , err := r . Run ( context . Background ( ) , RunInput {
Transcript : & schema . Transcript { Segments : [ ] schema . Segment { { ID : 1 , Text : "cat and bird" } } } ,
ModuleSpecs : [ ] contracts . ModuleRunSpec { {
ModuleKey : "grammar" , InstanceName : "grammar" , RepairCoordinator : client ,
Settings : config . Stage { LLM : config . LLM { CorrectionRepairAttempts : 1 } } ,
} } ,
} )
if err != nil {
t . Fatal ( err )
}
operations := out . WorkUnits [ 0 ] . Operations
if out . Totals . Corrections != 0 || out . Totals . FilteringResponses != 0 || out . Totals . Recoveries != 2 || len ( operations ) != 2 {
t . Fatalf ( "unexpected filtering totals or reports: %+v, %+v" , out . Totals , operations )
}
for i , operation := range operations {
if operation . Reference . Kind != outcomes . OperationCorrectionRepair || len ( operation . Filtered ) != 1 ||
operation . Filtered [ 0 ] . ProposalIndex != i || operation . Filtered [ 0 ] . Reason != outcomes . ReasonCode ( proposals . SkipReasonMissingOriginalText ) {
t . Fatalf ( "filtering reported against the wrong repair: %+v" , operation )
}
}
}
2026-11-11 04:29:18 +00:00
func TestRepairRevalidatesEarlierApprovalAndRetainsUnrelatedApprovalOnFailure ( t * testing . T ) {
candidates := [ ] proposals . EnrichedCorrectionProposal {
2026-11-11 17:05:52 +00:00
{ CorrectionProposal : proposals . CorrectionProposal { TargetSegmentID : 1 , OriginalText : "cat" , CorrectedText : "dog" } , ProposalMetadata : proposals . ProposalMetadata { ProposalIndex : 0 } } ,
{ CorrectionProposal : proposals . CorrectionProposal { TargetSegmentID : 1 , OriginalText : "bird" , CorrectedText : "owl" } , ProposalMetadata : proposals . ProposalMetadata { ProposalIndex : 1 } } ,
2026-11-11 04:29:18 +00:00
}
for _ , tc := range [ ] struct {
name string
failure bool
} { { "revision" , false } , { "provider failure" , true } } {
t . Run ( tc . name , func ( t * testing . T ) {
calls := 0
2026-11-16 14:11:02 +00:00
check := fakeValidator { name : "custom" , validateF : func ( req validators . Request ) ( validators . Result , error ) {
2026-11-11 04:29:18 +00:00
var decisions [ ] validators . Decision
for _ , candidate := range req . CandidateProposal {
calls ++
decisions = append ( decisions , validators . Decision { ProposalIndex : candidate . ProposalIndex , Approved : true , Disposition : validators . DispositionApproved } )
}
return validators . Result { Decisions : decisions } , nil
} }
2026-11-15 02:41:32 +00:00
client := & repairClient { t : t , outputs : [ ] string { ` { "proposal_ref":2,"decision": { "action":"revise","correction": { "id":1,"original_text":"bird","corrected_text":"owl"}}} ` } }
2026-11-11 04:29:18 +00:00
if tc . failure {
client . err = & structuredoutput . Failure { Class : structuredoutput . FailureRecoverable , Reason : outcomes . ReasonProviderFailed , Message : "failed" }
}
2026-11-16 14:36:07 +00:00
input := validateSectionCandidatesInput { Spec : contracts . ModuleRunSpec { RepairCoordinator : client , Settings : config . Stage { LLM : config . LLM { CorrectionRepairAttempts : 1 } } } , Policy : proposals . ReplacementPolicyRequireUnique , WorkingTranscript : & schema . Transcript { Segments : [ ] schema . Segment { { ID : 1 , Text : "cat bird" } } } , Validators : [ ] validators . Validator { check , rejectUntilRepair ( 0 ) } , SectionEnriched : candidates , Exchange : & structuredoutput . Exchange { } }
2026-11-11 04:29:18 +00:00
initial , err := validateSectionCandidates ( context . Background ( ) , input )
if err != nil {
t . Fatal ( err )
}
got , branches , err := repairCandidates ( context . Background ( ) , input , [ ] int { 1 , 1 } , initial , "repair" )
if err != nil {
t . Fatal ( err )
}
if tc . failure {
if len ( got . approved ) != 1 || got . approved [ 0 ] . ProposalIndex != 0 || branches [ 0 ] . Outcome != "unavailable" || calls != 2 {
t . Fatalf ( "result=%+v calls=%d" , got , calls )
}
} else if len ( got . approved ) != 2 || branches [ 0 ] . Outcome != "approved" || calls != 3 {
t . Fatalf ( "result=%+v calls=%d" , got , calls )
}
} )
}
}
func TestRepairInterruptionRetainsAttempt ( t * testing . T ) {
2026-11-15 01:25:55 +00:00
client := & repairClient { t : t , err : & structuredoutput . Failure { Class : structuredoutput . FailureCanceled , Reason : outcomes . ReasonCanceled , Message : "canceled" , Cause : context . Canceled } }
2026-11-11 17:05:52 +00:00
candidate := proposals . EnrichedCorrectionProposal { CorrectionProposal : proposals . CorrectionProposal { TargetSegmentID : 1 , OriginalText : "cat" , CorrectedText : "dog" } , ProposalMetadata : proposals . ProposalMetadata { ProposalIndex : 0 } }
2026-11-16 14:36:07 +00:00
input := validateSectionCandidatesInput { Spec : contracts . ModuleRunSpec { RepairCoordinator : client , Settings : config . Stage { LLM : config . LLM { CorrectionRepairAttempts : 1 } } } , Policy : proposals . ReplacementPolicyRequireUnique , WorkingTranscript : & schema . Transcript { Segments : [ ] schema . Segment { { ID : 1 , Text : "cat" } } } , Validators : [ ] validators . Validator { rejectUntilRepair ( ) } , SectionEnriched : [ ] proposals . EnrichedCorrectionProposal { candidate } , Exchange : & structuredoutput . Exchange { } }
2026-11-11 04:29:18 +00:00
initial , err := validateSectionCandidates ( context . Background ( ) , input )
if err != nil {
t . Fatal ( err )
}
_ , branches , err := repairCandidates ( context . Background ( ) , input , [ ] int { 1 } , initial , "repair" )
2026-11-11 12:31:08 +00:00
if ! errors . Is ( err , context . Canceled ) || len ( branches ) != 1 || branches [ 0 ] . Outcome != "interrupted" || branches [ 0 ] . AttemptCount != 1 || len ( branches [ 0 ] . Attempts ) != 0 {
2026-11-11 04:29:18 +00:00
t . Fatalf ( "branches=%+v error=%v" , branches , err )
}
}
2026-11-11 04:37:52 +00:00
2026-11-16 03:37:29 +00:00
func TestRepairArtifactsAreUnique ( t * testing . T ) {
client := & repairClient { t : t , outputs : [ ] string { ` { "proposal_ref":1,"decision": { "action":"withdraw","correction":null}} ` , ` { "proposal_ref":2,"decision": { "action":"withdraw","correction":null}} ` } }
2026-11-11 04:37:52 +00:00
candidates := [ ] proposals . EnrichedCorrectionProposal {
2026-11-11 17:05:52 +00:00
{ CorrectionProposal : proposals . CorrectionProposal { TargetSegmentID : 1 , OriginalText : "cat" , CorrectedText : "dog" } , ProposalMetadata : proposals . ProposalMetadata { ProposalIndex : 0 } } ,
{ CorrectionProposal : proposals . CorrectionProposal { TargetSegmentID : 1 , OriginalText : "cat" , CorrectedText : "fox" } , ProposalMetadata : proposals . ProposalMetadata { ProposalIndex : 1 } } ,
2026-11-11 04:37:52 +00:00
}
2026-11-16 03:43:29 +00:00
section := 4
2026-11-16 14:36:07 +00:00
input := validateSectionCandidatesInput { Spec : contracts . ModuleRunSpec { InstanceName : "grammar_1" , RepairCoordinator : client , Settings : config . Stage { LLM : config . LLM { CorrectionRepairAttempts : 1 } } } , Policy : proposals . ReplacementPolicyRequireUnique , WorkingTranscript : & schema . Transcript { Segments : [ ] schema . Segment { { ID : 1 , Text : "cat" } } } , Validators : [ ] validators . Validator { rejectUntilRepair ( ) } , SectionEnriched : candidates , SectionIndex : & section , Exchange : & structuredoutput . Exchange { } , Interactions : llm . NewDiagnosticsWriter ( t . TempDir ( ) ) }
2026-11-11 04:37:52 +00:00
initial , err := validateSectionCandidates ( context . Background ( ) , input )
if err != nil {
t . Fatal ( err )
}
_ , branches , err := repairCandidates ( context . Background ( ) , input , [ ] int { 1 , 1 } , initial , "repair" )
if err != nil {
t . Fatal ( err )
}
if len ( branches ) != 2 {
t . Fatalf ( "branches: %+v" , branches )
}
a , b := branches [ 0 ] . Attempts [ 0 ] . Artifacts . RequestPayloadPath , branches [ 1 ] . Attempts [ 0 ] . Artifacts . RequestPayloadPath
if a == "" || a == b {
t . Fatalf ( "artifact paths: %q %q" , a , b )
}
2026-11-16 03:37:29 +00:00
for i , path := range [ ] string { a , b } {
2026-11-11 04:37:52 +00:00
data , err := os . ReadFile ( path )
if err != nil {
t . Fatal ( err )
}
2026-11-16 03:37:29 +00:00
if ! json . Valid ( data ) {
t . Fatalf ( "invalid JSON artifact %s: %s" , filepath . Base ( path ) , data )
}
2026-11-16 03:43:29 +00:00
raw , err := os . ReadFile ( branches [ i ] . Attempts [ 0 ] . Artifacts . RequestMetadataPath )
if err != nil {
t . Fatal ( err )
}
var metadata struct {
Context structuredoutput . DiagnosticContext ` json:"context" `
}
if err := json . Unmarshal ( raw , & metadata ) ; err != nil {
t . Fatal ( err )
}
c := metadata . Context
if c . Role != "correction_repair" || c . ModuleInstance != "grammar_1" || c . SectionIndex == nil || * c . SectionIndex != section ||
c . ProposalIndex == nil || * c . ProposalIndex != i || c . RepairAttempt == nil || * c . RepairAttempt != 1 {
t . Fatalf ( "repair metadata: %s" , raw )
}
2026-11-16 03:37:29 +00:00
var request proposal_generation . RepairRequest
if err := json . Unmarshal ( [ ] byte ( client . calls [ i ] . Variables [ "RepairRequestJSON" ] ) , & request ) ; err != nil {
t . Fatal ( err )
}
if request . ProposalRef != i + 1 || request . Correction . CorrectedText != candidates [ i ] . CorrectedText ||
len ( request . Findings ) != 1 || request . Findings [ 0 ] . Explanation != "The proposed replacement is unsupported by the supplied context." {
t . Fatalf ( "repair request for proposal %d: %+v" , i , request )
2026-11-11 04:37:52 +00:00
}
}
}
func TestRepairArtifactFailureDoesNotChangeOutcome ( t * testing . T ) {
2026-11-15 02:41:32 +00:00
client := & repairClient { t : t , outputs : [ ] string { ` { "proposal_ref":1,"decision": { "action":"withdraw","correction":null}} ` } }
2026-11-11 04:37:52 +00:00
path := filepath . Join ( t . TempDir ( ) , "file" )
if err := os . WriteFile ( path , [ ] byte ( "x" ) , 0600 ) ; err != nil {
t . Fatal ( err )
}
2026-11-16 03:43:29 +00:00
sink := llm . NewDiagnosticsWriter ( path )
2026-11-11 17:05:52 +00:00
candidate := proposals . EnrichedCorrectionProposal { CorrectionProposal : proposals . CorrectionProposal { TargetSegmentID : 1 , OriginalText : "cat" , CorrectedText : "dog" } , ProposalMetadata : proposals . ProposalMetadata { ProposalIndex : 0 } }
2026-11-16 14:36:07 +00:00
input := validateSectionCandidatesInput { Spec : contracts . ModuleRunSpec { InstanceName : "grammar" , RepairCoordinator : client , Settings : config . Stage { LLM : config . LLM { CorrectionRepairAttempts : 1 } } } , Policy : proposals . ReplacementPolicyRequireUnique , WorkingTranscript : & schema . Transcript { Segments : [ ] schema . Segment { { ID : 1 , Text : "cat" } } } , Validators : [ ] validators . Validator { rejectUntilRepair ( ) } , SectionEnriched : [ ] proposals . EnrichedCorrectionProposal { candidate } , Exchange : & structuredoutput . Exchange { } , Interactions : sink }
2026-11-11 04:37:52 +00:00
initial , err := validateSectionCandidates ( context . Background ( ) , input )
if err != nil {
t . Fatal ( err )
}
_ , branches , err := repairCandidates ( context . Background ( ) , input , [ ] int { 1 } , initial , "repair" )
2026-11-16 03:43:29 +00:00
if err != nil || len ( branches ) != 1 || branches [ 0 ] . Outcome != "withdrawn" || len ( sink . Warnings ( ) ) != 1 {
t . Fatalf ( "branches=%+v warnings=%v err=%v" , branches , sink . Warnings ( ) , err )
2026-11-11 04:37:52 +00:00
}
}
2026-11-11 04:47:43 +00:00
func TestNoEffectAndUnavailableOnlyDoNotStartRepair ( t * testing . T ) {
2026-11-15 01:25:55 +00:00
client := & repairClient { t : t }
2026-11-11 04:47:43 +00:00
for _ , rejected := range [ ] [ ] ValidatorRejectedChange {
{ { ProposalIndex : 0 , Disposition : validators . DispositionRejected , ReasonCode : validators . ReasonNoEffect } } ,
{ { ProposalIndex : 0 , Disposition : validators . DispositionUnavailable , ReasonCode : "provider_failed" } } ,
} {
candidate := proposals . EnrichedCorrectionProposal { ProposalMetadata : proposals . ProposalMetadata { ProposalIndex : 0 } }
2026-11-16 14:36:07 +00:00
input := validateSectionCandidatesInput { Spec : contracts . ModuleRunSpec { RepairCoordinator : client , Settings : config . Stage { LLM : config . LLM { CorrectionRepairAttempts : 1 } } } , SectionEnriched : [ ] proposals . EnrichedCorrectionProposal { candidate } }
2026-11-11 04:47:43 +00:00
_ , branches , err := repairCandidates ( context . Background ( ) , input , nil , validateSectionCandidatesResult { rejected : rejected } , "repair" )
if err != nil || len ( branches ) != 0 || len ( client . calls ) != 0 {
t . Fatalf ( "branches=%+v calls=%d err=%v" , branches , len ( client . calls ) , err )
}
}
}
2026-11-11 17:05:52 +00:00
// rejectUntilRepair supplies a semantic rejection at the external validator seam.
2026-11-16 14:11:02 +00:00
func rejectUntilRepair ( preserved ... int ) validators . Validator {
return fakeValidator { name : "editorial_review" , validateF : func ( req validators . Request ) ( validators . Result , error ) {
2026-11-11 17:05:52 +00:00
var decisions [ ] validators . Decision
for _ , p := range req . CandidateProposal {
approved := req . RepairAttempt != nil
for _ , index := range preserved {
if p . ProposalIndex == index {
approved = true
}
}
disposition , reason := validators . DispositionRejected , "llm_rejected"
if approved {
disposition , reason = validators . DispositionApproved , validators . ReasonApproved
}
decisions = append ( decisions , validators . Decision { ProposalIndex : p . ProposalIndex , Approved : approved , Disposition : disposition , ReasonCode : reason , Message : "The proposed replacement is unsupported by the supplied context." } )
}
return validators . Result { Decisions : decisions } , nil
} }
}
2026-11-15 01:25:55 +00:00
type repairTransport func ( * http . Request ) ( * http . Response , error )
func ( f repairTransport ) RoundTrip ( r * http . Request ) ( * http . Response , error ) { return f ( r ) }
// Exercise the real adapter and upstream coordinator; only model responses are scripted.
func ( c * repairClient ) Coordinate ( ctx context . Context , _ * structuredoutput . Exchange , budget int , next func ( context . Context , structuredoutput . Turn ) ( * structuredoutput . Request , error ) ) ( bool , error ) {
t := c . t
dir := t . TempDir ( )
for name , content := range map [ string ] string {
"config.yml" : "version: 3\n" ,
2026-11-16 03:37:29 +00:00
"prompts.yml" : "profiles:\n - id: fixture\n endpoint: https://example.invalid/v1\n model: fixture\nprompts:\n - id: repair\n messages:\n - role: user\n content: '{{.RepairRequestJSON}}'\n output:\n format: text\n validation_mode: none\n" ,
2026-11-15 01:25:55 +00:00
} {
if err := os . WriteFile ( filepath . Join ( dir , name ) , [ ] byte ( content ) , 0600 ) ; err != nil {
t . Fatal ( err )
}
}
loaded , err := config . Load ( ck . Input { Source : ck . File ( filepath . Join ( dir , "config.yml" ) ) } , "" )
if err != nil {
t . Fatal ( err )
}
var selected * structuredoutput . Request
initial := true
runtime , err := llm . NewRuntime ( ctx , loaded , & http . Client { Transport : repairTransport ( func ( * http . Request ) ( * http . Response , error ) {
content := ` { "corrections":[]} `
if initial {
initial = false
} else {
c . calls = append ( c . calls , * selected )
content = c . outputs [ len ( c . calls ) - 1 ]
}
body , _ := json . Marshal ( map [ string ] any { "choices" : [ ] any { map [ string ] any { "message" : map [ string ] any { "role" : "assistant" , "content" : content } , "finish_reason" : "stop" } } } )
return & http . Response { StatusCode : 200 , Header : make ( http . Header ) , Body : io . NopCloser ( strings . NewReader ( string ( body ) ) ) } , nil
} ) } )
if err != nil {
t . Fatal ( err )
}
client , err := runtime . NewRoleClient ( "repair" , "fixture" , config . LLM { OperationTimeout : time . Second } , llm . CallHooks { } )
if err != nil {
t . Fatal ( err )
}
var output map [ string ] any
base , err := client . CompleteStructured ( ctx , structuredoutput . Request { PromptID : prompts . PromptIDModuleGrammarProposal , Schema : responseschema . CorrectionSetKey , CaptureExchange : true , Variables : map [ string ] string { "TranscriptDescriptionBlock" : "" , "GlossaryJSON" : "[]" , "SectionJSON" : "[]" } } , & output )
if err != nil {
return false , err
}
return client . Coordinate ( ctx , base . Exchange , budget , func ( ctx context . Context , turn structuredoutput . Turn ) ( * structuredoutput . Request , error ) {
2026-11-17 04:29:02 +00:00
if turn . Response != nil && c . capturePrepared != nil {
c . capturePrepared ( turn . Response . Prepared )
}
2026-11-15 01:25:55 +00:00
req , err := next ( ctx , turn )
if req != nil && err == nil {
selected = req
if c . err != nil {
c . calls = append ( c . calls , * req )
_ , err = next ( ctx , structuredoutput . Turn { Response : & structuredoutput . Response { } , Err : c . err } )
return nil , err
}
// Test prompt IDs vary; all use the same supplied-data fixture definition.
copy := * req
copy . PromptID = "repair"
return & copy , nil
}
return req , err
} )
}