2026-05-11 14:22:22 +00:00
package runner
import (
"context"
2026-11-16 03:37:29 +00:00
"encoding/json"
2026-05-11 14:22:22 +00:00
"errors"
2026-05-12 01:25:52 +00:00
"os"
2026-05-12 02:25:33 +00:00
"path/filepath"
2026-11-10 20:50:50 +00:00
"reflect"
2026-05-12 18:39:26 -05:00
"strconv"
2026-05-12 01:25:52 +00:00
"strings"
2026-05-12 12:59:46 -05:00
"sync"
2026-05-11 14:22:22 +00:00
"testing"
2026-05-12 12:59:46 -05:00
"time"
2026-05-11 14:22:22 +00:00
2026-10-11 01:40:38 +00:00
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
2026-11-16 03:43:29 +00:00
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
2026-10-11 01:40:38 +00:00
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
2026-05-11 14:22:22 +00:00
)
2026-09-24 02:18:31 +00:00
func runnerTestConfig ( ) config . Config {
return config . Config { Chunking : config . Chunking { MaxSectionTokens : 8192 , MinSectionTokens : 2048 } , Validation : config . Validation { MaxPromptTokens : 2048 } }
}
2026-05-11 14:22:22 +00:00
type fakeFactory struct {
modules map [ string ] contracts . TranscriptModule
}
func ( f fakeFactory ) ModuleForSpec ( spec contracts . ModuleRunSpec ) ( contracts . TranscriptModule , error ) {
m , ok := f . modules [ spec . InstanceName ]
if ! ok {
return nil , errors . New ( "module not registered" )
}
return m , nil
}
type fakeModule struct {
2026-05-12 00:26:27 +00:00
key string
policy proposals . ReplacementPolicy
2026-11-16 14:11:02 +00:00
validators [ ] validators . Validator
2026-05-12 00:26:27 +00:00
proposeF func ( req contracts . ProposalRequest ) ( [ ] proposals . CorrectionProposal , error )
2026-05-11 14:22:22 +00:00
}
2026-09-24 17:23:59 +00:00
type outcomeModule struct {
fakeModule
proposeResult func ( contracts . ProposalRequest ) ( contracts . ProposalResult , error )
}
2026-11-16 14:11:02 +00:00
func TestRunnerStopsOnContradictoryValidatorDecision ( t * testing . T ) {
candidate := proposals . EnrichedCorrectionProposal {
CorrectionProposal : proposals . CorrectionProposal { TargetSegmentID : 1 , OriginalText : "cat" , CorrectedText : "dog" } ,
ProposalMetadata : proposals . ProposalMetadata { ProposalIndex : 7 } ,
}
contractFailure := errors . New ( "validator also returned a runtime error" )
for _ , cause := range [ ] error { nil , contractFailure } {
contradictory := fakeValidator { name : "invalid" , validateF : func ( validators . Request ) ( validators . Result , error ) {
return validators . Result { Decisions : [ ] validators . Decision { {
ProposalIndex : 7 , Approved : true , Disposition : validators . DispositionUnavailable ,
} } } , cause
} }
later := fakeValidator { name : "later" , validateF : func ( validators . Request ) ( validators . Result , error ) {
t . Fatal ( "candidate advanced past a contradictory decision" )
return validators . Result { } , nil
} }
result , err := validateSectionCandidates ( context . Background ( ) , validateSectionCandidatesInput {
Validators : [ ] validators . Validator { validators . ProposalShapeValidator { } , contradictory , later } ,
SectionEnriched : [ ] proposals . EnrichedCorrectionProposal { candidate } ,
} )
if err == nil || ! strings . Contains ( err . Error ( ) , "contradictory" ) || ( cause != nil && ! errors . Is ( err , cause ) ) ||
len ( result . approved ) != 0 || len ( result . decisions ) != 1 || result . decisions [ 0 ] . ValidatorName != "proposal_shape" {
t . Fatalf ( "contradictory result = %+v (%v)" , result , err )
}
}
}
2026-09-24 17:23:59 +00:00
func TestRunnerRecoverableProposalFailureAllowsLaterModule ( t * testing . T ) {
failed := outcomeModule { fakeModule : fakeModule { key : "first" , policy : proposals . ReplacementPolicyRequireUnique } , proposeResult : func ( req contracts . ProposalRequest ) ( contracts . ProposalResult , error ) {
return contracts . ProposalResult { Operations : [ ] outcomes . Operation { { Reference : outcomes . Reference { ModuleKey : "first" , ModuleInstance : "first" , Kind : outcomes . OperationProposal } , FailureReason : outcomes . ReasonTransportFailed } } } , nil
} }
second := fakeModule { key : "second" , policy : proposals . ReplacementPolicyRequireUnique , proposeF : func ( req contracts . ProposalRequest ) ( [ ] proposals . CorrectionProposal , error ) {
2026-11-11 17:05:52 +00:00
return [ ] proposals . CorrectionProposal { { TargetSegmentID : 1 , OriginalText : "teh" , CorrectedText : "the" } } , nil
2026-09-24 17:23:59 +00:00
} }
r := New ( fakeFactory { modules : map [ string ] contracts . TranscriptModule { "first" : failed , "second" : second } } )
out , err := r . Run ( context . Background ( ) , RunInput { Transcript : & schema . Transcript { Segments : [ ] schema . Segment { { ID : 1 , Text : "teh" } } } , ModuleSpecs : [ ] contracts . ModuleRunSpec { { ModuleKey : "first" , InstanceName : "first" } , { ModuleKey : "second" , InstanceName : "second" } } } )
if err != nil {
t . Fatal ( err )
}
if out . FinalTranscript . Segments [ 0 ] . Text != "the" || len ( out . WorkUnits ) != 2 || out . WorkUnits [ 0 ] . State != outcomes . WorkCompleted || out . WorkUnits [ 1 ] . CorrectionsApplied != 1 {
t . Fatalf ( "recoverable failure lost later work: %+v" , out )
}
if len ( out . WorkUnits [ 0 ] . Operations ) != 1 || out . WorkUnits [ 0 ] . Operations [ 0 ] . FailureReason != outcomes . ReasonTransportFailed {
t . Fatalf ( "missing proposal failure: %+v" , out . WorkUnits [ 0 ] )
}
}
func TestRunnerOnlyFullyValidatedCandidatesApplyAndRecoveriesMerge ( t * testing . T ) {
secondSaw := make ( [ ] int , 0 )
2026-11-16 14:11:02 +00:00
partial := fakeValidator { name : "partial" , validateF : func ( req validators . Request ) ( validators . Result , error ) {
2026-09-24 17:23:59 +00:00
if len ( req . CandidateProposal ) != 2 {
t . Fatalf ( "expected two candidates after shape filtering, got %d" , len ( req . CandidateProposal ) )
}
return validators . Result { ValidatorName : "partial" , Decisions : [ ] validators . Decision {
{ ProposalIndex : req . CandidateProposal [ 0 ] . ProposalIndex , Approved : false , Disposition : validators . DispositionUnavailable , ReasonCode : string ( outcomes . ReasonTransportFailed ) } ,
{ ProposalIndex : req . CandidateProposal [ 1 ] . ProposalIndex , Approved : true , Disposition : validators . DispositionApproved } ,
} , Operations : [ ] outcomes . Operation {
{ Reference : outcomes . Reference { Kind : outcomes . OperationValidation } , FailureReason : outcomes . ReasonTransportFailed } ,
{ Reference : outcomes . Reference { Kind : outcomes . OperationValidation } , FailureReason : outcomes . ReasonProviderFailed } ,
} } , nil
} }
2026-11-16 14:11:02 +00:00
last := fakeValidator { name : "last" , validateF : func ( req validators . Request ) ( validators . Result , error ) {
2026-09-24 17:23:59 +00:00
for _ , candidate := range req . CandidateProposal {
secondSaw = append ( secondSaw , candidate . ProposalIndex )
}
return validators . Result { ValidatorName : "last" , Decisions : [ ] validators . Decision { { ProposalIndex : req . CandidateProposal [ 0 ] . ProposalIndex , Approved : true , Disposition : validators . DispositionApproved } } } , nil
} }
2026-11-16 14:11:02 +00:00
module := outcomeModule { fakeModule : fakeModule { key : "m" , policy : proposals . ReplacementPolicyRequireUnique , validators : [ ] validators . Validator { validators . ProposalShapeValidator { } , partial , last } } , proposeResult : func ( req contracts . ProposalRequest ) ( contracts . ProposalResult , error ) {
2026-09-24 17:23:59 +00:00
return contracts . ProposalResult { Proposals : [ ] proposals . CorrectionProposal {
2026-11-11 17:05:52 +00:00
{ TargetSegmentID : 0 , OriginalText : "bad" , CorrectedText : "x" } ,
{ TargetSegmentID : 1 , OriginalText : "teh" , CorrectedText : "the" } ,
{ TargetSegmentID : 1 , OriginalText : "cat" , CorrectedText : "dog" } ,
2026-09-24 17:23:59 +00:00
} , Operations : [ ] outcomes . Operation { { Reference : outcomes . Reference { Kind : outcomes . OperationProposal } , Succeeded : true , RecoveryReasons : [ ] outcomes . RecoveryReason { outcomes . RecoveryOutputRepaired } } } } , nil
} }
r := New ( fakeFactory { modules : map [ string ] contracts . TranscriptModule { "m" : module } } )
out , err := r . Run ( context . Background ( ) , RunInput { Transcript : & schema . Transcript { Segments : [ ] schema . Segment { { ID : 1 , Text : "teh cat" } } } , ModuleSpecs : [ ] contracts . ModuleRunSpec { { ModuleKey : "m" , InstanceName : "m" } } } )
if err != nil {
t . Fatal ( err )
}
if out . FinalTranscript . Segments [ 0 ] . Text != "teh dog" || len ( secondSaw ) != 1 || secondSaw [ 0 ] != 2 {
t . Fatalf ( "unchecked candidate applied or survivor lost: text=%q final=%v" , out . FinalTranscript . Segments [ 0 ] . Text , secondSaw )
}
unit := out . WorkUnits [ 0 ]
if unit . State != outcomes . WorkCompleted || unit . CorrectionsApplied != 1 || len ( unit . Operations ) != 3 {
t . Fatalf ( "expected one completed unit with two operation failures: %+v" , unit )
}
proposal := unit . Operations [ 0 ]
if len ( proposal . RecoveryReasons ) != 2 || proposal . RecoveryReasons [ 0 ] != outcomes . RecoveryOutputRepaired || proposal . RecoveryReasons [ 1 ] != outcomes . RecoveryProposalsFiltered || len ( proposal . Filtered ) != 1 || proposal . Filtered [ 0 ] . ProposalIndex != 0 {
t . Fatalf ( "repair and filtering must share one recovery: %+v" , proposal )
}
}
func TestFatalValidatorNeverReturnsUncheckedCandidates ( t * testing . T ) {
origin := errors . New ( "validator setup failed" )
2026-11-11 17:05:52 +00:00
proposal := proposals . EnrichedCorrectionProposal { CorrectionProposal : proposals . CorrectionProposal { TargetSegmentID : 1 , OriginalText : "teh" , CorrectedText : "the" } , ProposalMetadata : proposals . ProposalMetadata { ProposalIndex : 7 } }
2026-11-16 14:11:02 +00:00
broken := fakeValidator { name : "broken" , validateF : func ( req validators . Request ) ( validators . Result , error ) {
2026-09-24 17:23:59 +00:00
return validators . Result { Operations : [ ] outcomes . Operation { { Reference : outcomes . Reference { Kind : outcomes . OperationValidation } , FailureReason : outcomes . ReasonSetupFailure } } } , origin
} }
2026-11-16 14:11:02 +00:00
result , err := validateSectionCandidates ( context . Background ( ) , validateSectionCandidatesInput { Spec : contracts . ModuleRunSpec { ModuleKey : "m" , InstanceName : "m" } , Validators : [ ] validators . Validator { broken } , SectionEnriched : [ ] proposals . EnrichedCorrectionProposal { proposal } } )
2026-09-24 17:23:59 +00:00
if ! errors . Is ( err , origin ) || len ( result . approved ) != 0 || len ( result . operations ) != 1 {
t . Fatalf ( "fatal result must retain facts without approving candidates: result=%+v err=%v" , result , err )
}
}
2026-09-24 17:41:10 +00:00
func TestCandidateCaptureIsExplicitAndIncludesFilteredProposals ( t * testing . T ) {
module := fakeModule { key : "grammar" , policy : proposals . ReplacementPolicyRequireUnique ,
2026-11-16 14:11:02 +00:00
validators : [ ] validators . Validator { validators . ProposalShapeValidator { } } ,
2026-09-24 17:41:10 +00:00
proposeF : func ( req contracts . ProposalRequest ) ( [ ] proposals . CorrectionProposal , error ) {
2026-11-11 17:05:52 +00:00
return [ ] proposals . CorrectionProposal { { TargetSegmentID : 0 , OriginalText : "bad" , CorrectedText : "good" } } , nil
2026-09-24 17:41:10 +00:00
} }
r := New ( fakeFactory { modules : map [ string ] contracts . TranscriptModule { "grammar" : module } } )
input := RunInput { Transcript : & schema . Transcript { Segments : [ ] schema . Segment { { ID : 1 , Text : "bad" } } } , ModuleSpecs : [ ] contracts . ModuleRunSpec { { ModuleKey : "grammar" , InstanceName : "grammar" } } }
without , err := r . Run ( context . Background ( ) , input )
if err != nil {
t . Fatal ( err )
}
if len ( without . Candidates ) != 0 {
t . Fatalf ( "debug-off candidate capture: %+v" , without . Candidates )
}
input . CaptureCandidates = true
with , err := r . Run ( context . Background ( ) , input )
if err != nil {
t . Fatal ( err )
}
if len ( with . Candidates ) != 1 || with . Candidates [ 0 ] . ProposalIndex != 0 || len ( with . ModuleResults [ 0 ] . ValidatorRejected ) != 1 {
t . Fatalf ( "filtered candidate missing from debug evidence: %+v" , with )
}
}
2026-09-24 17:23:59 +00:00
func ( m outcomeModule ) Propose ( _ context . Context , req contracts . ProposalRequest ) ( contracts . ProposalResult , error ) {
return m . proposeResult ( req )
}
2026-05-11 14:22:22 +00:00
func ( m fakeModule ) Key ( ) string { return m . key }
func ( m fakeModule ) ReplacementPolicy ( ) proposals . ReplacementPolicy { return m . policy }
2026-11-16 14:11:02 +00:00
func ( m fakeModule ) Validators ( ) [ ] validators . Validator { return m . validators }
2026-05-23 10:07:06 -05:00
func ( m fakeModule ) Propose ( ctx context . Context , req contracts . ProposalRequest ) ( contracts . ProposalResult , error ) {
2026-05-11 14:22:22 +00:00
if m . proposeF == nil {
2026-05-23 10:07:06 -05:00
return contracts . ProposalResult { } , nil
2026-05-11 14:22:22 +00:00
}
2026-05-23 10:07:06 -05:00
proposalsOut , err := m . proposeF ( req )
return contracts . ProposalResult { Proposals : proposalsOut } , err
2026-05-11 14:22:22 +00:00
}
2026-05-12 00:26:27 +00:00
type fakeValidator struct {
name string
2026-11-16 14:11:02 +00:00
validateF func ( req validators . Request ) ( validators . Result , error )
2026-05-12 00:26:27 +00:00
}
func ( v fakeValidator ) Name ( ) string { return v . name }
2026-11-16 14:11:02 +00:00
func ( v fakeValidator ) Validate ( ctx context . Context , req validators . Request ) ( validators . Result , error ) {
2026-05-12 00:26:27 +00:00
_ = ctx
return v . validateF ( req )
}
2026-05-14 00:28:28 +00:00
type classifiedFakeValidator struct {
fakeValidator
class validatormetadata . ExecutionClass
}
func ( v classifiedFakeValidator ) ExecutionClass ( ) validatormetadata . ExecutionClass {
return v . class
}
func TestReorderValidatorsDeterministicBeforeLLMBacked ( t * testing . T ) {
llm := classifiedFakeValidator {
2026-11-16 14:11:02 +00:00
fakeValidator : fakeValidator { name : "llm" , validateF : func ( req validators . Request ) ( validators . Result , error ) {
2026-05-14 00:28:28 +00:00
return validators . Result { ValidatorName : "llm" } , nil
} } ,
class : validatormetadata . ExecutionClassLLMBacked ,
}
deterministic := classifiedFakeValidator {
2026-11-16 14:11:02 +00:00
fakeValidator : fakeValidator { name : "deterministic" , validateF : func ( req validators . Request ) ( validators . Result , error ) {
2026-05-14 00:28:28 +00:00
return validators . Result { ValidatorName : "deterministic" } , nil
} } ,
class : validatormetadata . ExecutionClassDeterministic ,
}
2026-11-16 14:11:02 +00:00
ordered , _ := reorderValidatorsForPipeline ( [ ] validators . Validator { llm , deterministic } )
2026-05-14 00:28:28 +00:00
if len ( ordered ) != 2 {
t . Fatalf ( "expected 2 validators, got %d" , len ( ordered ) )
}
if ordered [ 0 ] . Name ( ) != "deterministic" || ordered [ 1 ] . Name ( ) != "llm" {
t . Fatalf ( "unexpected validator order: %s, %s" , ordered [ 0 ] . Name ( ) , ordered [ 1 ] . Name ( ) )
}
}
func TestReorderValidatorsDefaultsUnclassifiedToDeterministic ( t * testing . T ) {
2026-11-16 14:11:02 +00:00
unclassified := fakeValidator { name : "plain" , validateF : func ( req validators . Request ) ( validators . Result , error ) {
2026-05-14 00:28:28 +00:00
return validators . Result { ValidatorName : "plain" } , nil
} }
llm := classifiedFakeValidator {
2026-11-16 14:11:02 +00:00
fakeValidator : fakeValidator { name : "llm" , validateF : func ( req validators . Request ) ( validators . Result , error ) {
2026-05-14 00:28:28 +00:00
return validators . Result { ValidatorName : "llm" } , nil
} } ,
class : validatormetadata . ExecutionClassLLMBacked ,
}
2026-11-16 14:11:02 +00:00
ordered , _ := reorderValidatorsForPipeline ( [ ] validators . Validator { llm , unclassified } )
2026-05-14 00:28:28 +00:00
if len ( ordered ) != 2 {
t . Fatalf ( "expected 2 validators, got %d" , len ( ordered ) )
}
if ordered [ 0 ] . Name ( ) != "plain" || ordered [ 1 ] . Name ( ) != "llm" {
t . Fatalf ( "unexpected validator order with unclassified validator: %s, %s" , ordered [ 0 ] . Name ( ) , ordered [ 1 ] . Name ( ) )
}
}
2026-05-11 14:22:22 +00:00
func TestRunnerOneModuleAppliesProposal ( t * testing . T ) {
transcript := & schema . Transcript { Segments : [ ] schema . Segment { { ID : 1 , Speaker : "A" , Start : 0 , End : 1 , Text : "teh cat" } } }
r := New ( fakeFactory { modules : map [ string ] contracts . TranscriptModule {
"grammar" : fakeModule { key : "grammar" , policy : proposals . ReplacementPolicyRequireUnique , proposeF : func ( req contracts . ProposalRequest ) ( [ ] proposals . CorrectionProposal , error ) {
2026-11-11 17:05:52 +00:00
return [ ] proposals . CorrectionProposal { { TargetSegmentID : 1 , OriginalText : "teh" , CorrectedText : "the" } } , nil
2026-05-11 14:22:22 +00:00
} } ,
} } )
out , err := r . Run ( context . Background ( ) , RunInput { Transcript : transcript , ModuleSpecs : [ ] contracts . ModuleRunSpec { { ModuleKey : "grammar" , InstanceName : "grammar" } } } )
if err != nil {
t . Fatalf ( "Run error: %v" , err )
}
if out . FinalTranscript . Segments [ 0 ] . Text != "the cat" {
t . Fatalf ( "expected corrected text, got %q" , out . FinalTranscript . Segments [ 0 ] . Text )
}
if len ( out . ModuleResults ) != 1 || len ( out . ModuleResults [ 0 ] . AppliedChanges ) != 1 {
t . Fatalf ( "expected one module with one applied change, got %+v" , out . ModuleResults )
}
}
func TestRunnerModulesRunSequentially ( t * testing . T ) {
transcript := & schema . Transcript { Segments : [ ] schema . Segment { { ID : 1 , Speaker : "A" , Start : 0 , End : 1 , Text : "teh cat" } } }
r := New ( fakeFactory { modules : map [ string ] contracts . TranscriptModule {
"m1" : fakeModule { key : "m1" , policy : proposals . ReplacementPolicyRequireUnique , proposeF : func ( req contracts . ProposalRequest ) ( [ ] proposals . CorrectionProposal , error ) {
2026-11-11 17:05:52 +00:00
return [ ] proposals . CorrectionProposal { { TargetSegmentID : 1 , OriginalText : "teh" , CorrectedText : "the" } } , nil
2026-05-11 14:22:22 +00:00
} } ,
"m2" : fakeModule { key : "m2" , policy : proposals . ReplacementPolicyRequireUnique , proposeF : func ( req contracts . ProposalRequest ) ( [ ] proposals . CorrectionProposal , error ) {
if req . WorkingTranscript . Segments [ 0 ] . Text != "the cat" {
t . Fatalf ( "second module did not see first module changes: %q" , req . WorkingTranscript . Segments [ 0 ] . Text )
}
2026-11-11 17:05:52 +00:00
return [ ] proposals . CorrectionProposal { { TargetSegmentID : 1 , OriginalText : "cat" , CorrectedText : "dog" } } , nil
2026-05-11 14:22:22 +00:00
} } ,
} } )
out , err := r . Run ( context . Background ( ) , RunInput { Transcript : transcript , ModuleSpecs : [ ] contracts . ModuleRunSpec { { ModuleKey : "m1" , InstanceName : "m1" } , { ModuleKey : "m2" , InstanceName : "m2" } } } )
if err != nil {
t . Fatalf ( "Run error: %v" , err )
}
if out . FinalTranscript . Segments [ 0 ] . Text != "the dog" {
t . Fatalf ( "expected sequential updates, got %q" , out . FinalTranscript . Segments [ 0 ] . Text )
}
}
2026-09-24 17:17:14 +00:00
func TestRunnerKeepsSectionMembershipAfterTextGrowth ( t * testing . T ) {
cfg := runnerTestConfig ( )
cfg . Chunking . MaxSectionTokens = 3
cfg . Chunking . MinSectionTokens = 0
transcript := & schema . Transcript { Segments : [ ] schema . Segment {
2026-09-24 18:03:25 +00:00
{ ID : 1 , Speaker : "A" , Start : 0 , End : 1 , Text : "a" } ,
{ ID : 2 , Speaker : "B" , Start : 1 , End : 2 , Text : "b" } ,
{ ID : 3 , Speaker : "A" , Start : 2 , End : 3 , Text : "c" } ,
{ ID : 4 , Speaker : "B" , Start : 3 , End : 4 , Text : "d" } ,
2026-09-24 17:17:14 +00:00
} }
var mu sync . Mutex
seen := make ( map [ int ] [ ] int )
updatedText := "a one two three four five"
r := New ( fakeFactory { modules : map [ string ] contracts . TranscriptModule {
"grow" : fakeModule { key : "grow" , policy : proposals . ReplacementPolicyRequireUnique , proposeF : func ( req contracts . ProposalRequest ) ( [ ] proposals . CorrectionProposal , error ) {
if req . Section . Index != 0 {
return nil , nil
}
2026-11-11 17:05:52 +00:00
return [ ] proposals . CorrectionProposal { { TargetSegmentID : 1 , OriginalText : "a" , CorrectedText : updatedText } } , nil
2026-09-24 17:17:14 +00:00
} } ,
"inspect" : fakeModule { key : "inspect" , policy : proposals . ReplacementPolicyRequireUnique , proposeF : func ( req contracts . ProposalRequest ) ( [ ] proposals . CorrectionProposal , error ) {
ids := make ( [ ] int , len ( req . WorkingTranscript . Segments ) )
for i , segment := range req . WorkingTranscript . Segments {
ids [ i ] = segment . ID
}
mu . Lock ( )
seen [ req . Section . Index ] = ids
mu . Unlock ( )
if req . Section . Index == 0 && req . WorkingTranscript . Segments [ 0 ] . Text != updatedText {
return nil , errors . New ( "later module did not see updated section text" )
}
2026-09-24 18:03:25 +00:00
if req . Section . Index == 0 {
2026-11-11 17:05:52 +00:00
return [ ] proposals . CorrectionProposal { { TargetSegmentID : 3 , OriginalText : "c" , CorrectedText : "C" } } , nil
2026-09-24 18:03:25 +00:00
}
2026-09-24 17:17:14 +00:00
return nil , nil
} } ,
} } )
out , err := r . Run ( context . Background ( ) , RunInput { Config : & cfg , Transcript : transcript , ModuleSpecs : [ ] contracts . ModuleRunSpec { { ModuleKey : "grow" , InstanceName : "grow" } , { ModuleKey : "inspect" , InstanceName : "inspect" } } } )
if err != nil {
t . Fatal ( err )
}
if len ( out . Sections ) != 2 || out . Sections [ 0 ] . StartSegmentID != 1 || out . Sections [ 0 ] . EndSegmentID != 2 || out . Sections [ 0 ] . EstimatedTokens != 2 || len ( out . WorkUnits ) != 4 {
t . Fatalf ( "initial inventory changed: sections=%+v units=%+v" , out . Sections , out . WorkUnits )
}
if len ( seen ) != 2 || len ( seen [ 0 ] ) != 2 || seen [ 0 ] [ 0 ] != 1 || seen [ 0 ] [ 1 ] != 2 || len ( seen [ 1 ] ) != 2 || seen [ 1 ] [ 0 ] != 3 || seen [ 1 ] [ 1 ] != 4 {
t . Fatalf ( "later module saw changed section membership: %+v" , seen )
}
2026-09-24 18:03:25 +00:00
if len ( out . FinalTranscript . Segments ) != 4 || out . FinalTranscript . Segments [ 0 ] . Text != updatedText || out . FinalTranscript . Segments [ 2 ] . Text != "c" || out . FinalTranscript . Segments [ 3 ] . ID != 4 {
2026-09-24 17:17:14 +00:00
t . Fatalf ( "recombined transcript: %+v" , out . FinalTranscript )
}
2026-09-24 18:03:25 +00:00
for i , segment := range out . FinalTranscript . Segments {
original := transcript . Segments [ i ]
if segment . ID != original . ID || segment . Speaker != original . Speaker || segment . Start != original . Start || segment . End != original . End {
t . Fatalf ( "segment %d metadata changed: %+v" , i , segment )
}
}
2026-09-24 17:17:14 +00:00
if out . WorkUnits [ 0 ] . CorrectionsApplied != 1 || out . WorkUnits [ 1 ] . CorrectionsApplied != 0 {
t . Fatalf ( "section application counts: %+v" , out . WorkUnits )
}
}
func TestRunnerInventoriesEmptyWorkAndRejectsImpossibleChunking ( t * testing . T ) {
r := New ( fakeFactory { modules : map [ string ] contracts . TranscriptModule { } } )
empty , err := r . Run ( context . Background ( ) , RunInput { Transcript : & schema . Transcript { } } )
if err != nil || len ( empty . Sections ) != 0 || len ( empty . WorkUnits ) != 0 || len ( empty . FinalTranscript . Segments ) != 0 {
t . Fatalf ( "empty transcript and pipeline: %+v, %v" , empty , err )
}
input := & schema . Transcript { Segments : [ ] schema . Segment { { ID : 1 , Text : "one" } , { ID : 2 , Text : "two" } } }
noModules , err := r . Run ( context . Background ( ) , RunInput { Transcript : input } )
if err != nil || len ( noModules . Sections ) != 1 || len ( noModules . WorkUnits ) != 0 || len ( noModules . FinalTranscript . Segments ) != 2 {
t . Fatalf ( "empty pipeline inventory: %+v, %v" , noModules , err )
}
cfg := runnerTestConfig ( )
target := 3
cfg . Chunking . TargetSections = & target
if _ , err := r . Run ( context . Background ( ) , RunInput { Config : & cfg , Transcript : input } ) ; err == nil {
t . Fatal ( "impossible section target was accepted" )
}
}
2026-05-12 12:25:14 -05:00
func TestRunnerProposalsExecutePerChunkSection ( t * testing . T ) {
2026-09-24 02:18:31 +00:00
cfg := runnerTestConfig ( )
cfg . Chunking . MaxSectionTokens = 3
cfg . Chunking . MinSectionTokens = 0
2026-05-12 12:25:14 -05:00
transcript := & schema . Transcript { Segments : [ ] schema . Segment {
{ ID : 1 , Text : "one two" } ,
{ ID : 2 , Text : "three four" } ,
{ ID : 3 , Text : "five six" } ,
} }
2026-05-12 12:59:46 -05:00
var (
mu sync . Mutex
seenBySection = make ( map [ int ] contracts . SectionMetadata )
segmentCountBySec = make ( map [ int ] int )
)
2026-05-12 12:25:14 -05:00
r := New ( fakeFactory { modules : map [ string ] contracts . TranscriptModule {
"m" : fakeModule { key : "m" , policy : proposals . ReplacementPolicyRequireUnique , proposeF : func ( req contracts . ProposalRequest ) ( [ ] proposals . CorrectionProposal , error ) {
if req . Section == nil {
t . Fatalf ( "expected section metadata on proposal request" )
}
2026-05-12 12:59:46 -05:00
mu . Lock ( )
seenBySection [ req . Section . Index ] = * req . Section
segmentCountBySec [ req . Section . Index ] = len ( req . WorkingTranscript . Segments )
mu . Unlock ( )
2026-05-12 12:25:14 -05:00
return nil , nil
} } ,
} } )
out , err := r . Run ( context . Background ( ) , RunInput {
Config : & cfg ,
Transcript : transcript ,
ModuleSpecs : [ ] contracts . ModuleRunSpec {
{ ModuleKey : "m" , InstanceName : "m" } ,
} ,
} )
if err != nil {
t . Fatalf ( "Run error: %v" , err )
}
if len ( out . ModuleResults ) != 1 {
t . Fatalf ( "expected one module result, got %+v" , out . ModuleResults )
}
2026-11-10 20:50:50 +00:00
if ! reflect . DeepEqual ( out . FinalTranscript , transcript ) || len ( out . WorkUnits ) != 3 {
t . Fatalf ( "no-edit run changed the transcript or inventory: %+v" , out )
}
for _ , unit := range out . WorkUnits {
if unit . State != outcomes . WorkCompleted || unit . CorrectionsApplied != 0 {
t . Fatalf ( "no-edit section was not completed: %+v" , unit )
}
}
2026-05-12 12:59:46 -05:00
if len ( seenBySection ) != 3 {
t . Fatalf ( "expected 3 chunked proposal calls, got %d" , len ( seenBySection ) )
2026-05-12 12:25:14 -05:00
}
2026-05-12 12:59:46 -05:00
for i := 0 ; i < 3 ; i ++ {
section , ok := seenBySection [ i ]
if ! ok {
t . Fatalf ( "expected section %d to be observed" , i )
}
2026-05-12 12:25:14 -05:00
if section . Index != i {
t . Fatalf ( "expected section index %d, got %+v" , i , section )
}
2026-05-12 12:59:46 -05:00
if segmentCountBySec [ i ] != 1 {
t . Fatalf ( "expected one segment per section call, got %d at index %d" , segmentCountBySec [ i ] , i )
}
}
}
func TestRunnerProposalIndexOrderingIsDeterministicAcrossParallelSections ( t * testing . T ) {
2026-09-24 02:18:31 +00:00
cfg := runnerTestConfig ( )
cfg . Chunking . MaxSectionTokens = 3
cfg . Chunking . MinSectionTokens = 0
2026-05-12 12:59:46 -05:00
transcript := & schema . Transcript { Segments : [ ] schema . Segment {
{ ID : 1 , Text : "alpha one" } ,
{ ID : 2 , Text : "bravo two" } ,
{ ID : 3 , Text : "charlie three" } ,
} }
2026-05-12 21:23:57 +00:00
started := make ( chan int , len ( transcript . Segments ) )
releaseByID := map [ int ] chan struct { } {
1 : make ( chan struct { } ) ,
2 : make ( chan struct { } ) ,
3 : make ( chan struct { } ) ,
}
2026-05-12 12:59:46 -05:00
r := New ( fakeFactory { modules : map [ string ] contracts . TranscriptModule {
"m" : fakeModule { key : "m" , policy : proposals . ReplacementPolicyRequireUnique , proposeF : func ( req contracts . ProposalRequest ) ( [ ] proposals . CorrectionProposal , error ) {
if len ( req . WorkingTranscript . Segments ) != 1 {
t . Fatalf ( "expected one segment per section, got %d" , len ( req . WorkingTranscript . Segments ) )
}
seg := req . WorkingTranscript . Segments [ 0 ]
2026-05-12 21:23:57 +00:00
started <- seg . ID
<- releaseByID [ seg . ID ]
2026-05-12 12:59:46 -05:00
return [ ] proposals . CorrectionProposal {
2026-11-11 17:05:52 +00:00
{ TargetSegmentID : seg . ID , OriginalText : seg . Text , CorrectedText : strings . ToUpper ( seg . Text ) } ,
2026-05-12 12:59:46 -05:00
} , nil
} } ,
} } )
2026-05-12 21:23:57 +00:00
resultCh := make ( chan struct {
out RunOutput
err error
} , 1 )
go func ( ) {
out , err := r . Run ( context . Background ( ) , RunInput {
Config : & cfg ,
Transcript : transcript ,
ModuleSpecs : [ ] contracts . ModuleRunSpec {
{ ModuleKey : "m" , InstanceName : "m" } ,
} ,
} )
resultCh <- struct {
out RunOutput
err error
} { out : out , err : err }
} ( )
waitForRunnerSectionIDs ( t , started , map [ int ] struct { } { 1 : { } , 2 : { } , 3 : { } } )
close ( releaseByID [ 3 ] )
close ( releaseByID [ 2 ] )
close ( releaseByID [ 1 ] )
result := <- resultCh
out , err := result . out , result . err
2026-05-12 12:59:46 -05:00
if err != nil {
t . Fatalf ( "Run error: %v" , err )
}
if len ( out . ModuleResults ) != 1 {
t . Fatalf ( "expected one module result, got %+v" , out . ModuleResults )
}
if got := len ( out . ModuleResults [ 0 ] . AppliedChanges ) ; got != 3 {
t . Fatalf ( "expected three applied changes, got %d" , got )
}
for i , change := range out . ModuleResults [ 0 ] . AppliedChanges {
wantID := i + 1
if change . TargetSegmentID != wantID {
t . Fatalf ( "expected deterministic applied-change order by section/segment, got %+v" , out . ModuleResults [ 0 ] . AppliedChanges )
2026-05-12 12:25:14 -05:00
}
}
2026-05-12 21:17:35 +00:00
wantFinal := [ ] string { "ALPHA ONE" , "BRAVO TWO" , "CHARLIE THREE" }
for i , seg := range out . FinalTranscript . Segments {
if seg . Text != wantFinal [ i ] {
t . Fatalf ( "expected deterministic final transcript regardless of section completion order; got %+v" , out . FinalTranscript . Segments )
}
}
2026-05-12 12:25:14 -05:00
}
2026-05-12 21:23:57 +00:00
func waitForRunnerEntries ( t * testing . T , entered <- chan struct { } , want int , label string ) {
t . Helper ( )
deadline := time . After ( 350 * time . Millisecond )
got := 0
for got < want {
select {
case <- entered :
got ++
case <- deadline :
t . Fatalf ( "timed out waiting for %d entries (%s), got %d" , want , label , got )
}
}
}
func waitForRunnerSectionIDs ( t * testing . T , started <- chan int , want map [ int ] struct { } ) {
t . Helper ( )
deadline := time . After ( 350 * time . Millisecond )
seen := map [ int ] struct { } { }
for len ( seen ) < len ( want ) {
select {
case id := <- started :
seen [ id ] = struct { } { }
case <- deadline :
t . Fatalf ( "timed out waiting for section IDs %v, got %v" , mapKeys ( want ) , mapKeys ( seen ) )
}
}
}
func mapKeys ( values map [ int ] struct { } ) [ ] int {
keys := make ( [ ] int , 0 , len ( values ) )
for key := range values {
keys = append ( keys , key )
}
return keys
}
2026-05-12 18:39:26 -05:00
func TestRunnerValidationStartsBeforeAllSectionProposalsComplete ( t * testing . T ) {
2026-09-24 02:18:31 +00:00
cfg := runnerTestConfig ( )
cfg . Chunking . MaxSectionTokens = 3
cfg . Chunking . MinSectionTokens = 0
2026-05-12 18:39:26 -05:00
transcript := & schema . Transcript { Segments : [ ] schema . Segment {
{ ID : 1 , Text : "alpha one" } ,
{ ID : 2 , Text : "bravo two" } ,
} }
releaseSectionOne := make ( chan struct { } )
validatorStarted := make ( chan struct { } , 1 )
validator := fakeValidator {
name : "capture" ,
2026-11-16 14:11:02 +00:00
validateF : func ( req validators . Request ) ( validators . Result , error ) {
2026-05-12 18:39:26 -05:00
if len ( req . CandidateProposal ) == 1 && req . CandidateProposal [ 0 ] . TargetSegmentID == 1 {
select {
case validatorStarted <- struct { } { } :
default :
}
}
decisions := make ( [ ] validators . Decision , 0 , len ( req . CandidateProposal ) )
for _ , p := range req . CandidateProposal {
decisions = append ( decisions , validators . Decision {
ProposalIndex : p . ProposalIndex ,
Approved : true ,
ReasonCode : validators . ReasonApproved ,
Message : "ok" ,
} )
}
return validators . Result { ValidatorName : "capture" , Decisions : decisions } , nil
} ,
}
r := New ( fakeFactory { modules : map [ string ] contracts . TranscriptModule {
"m" : fakeModule {
key : "m" ,
policy : proposals . ReplacementPolicyRequireUnique ,
2026-11-16 14:11:02 +00:00
validators : [ ] validators . Validator { validator } ,
2026-05-12 18:39:26 -05:00
proposeF : func ( req contracts . ProposalRequest ) ( [ ] proposals . CorrectionProposal , error ) {
seg := req . WorkingTranscript . Segments [ 0 ]
if req . Section != nil && req . Section . Index == 1 {
<- releaseSectionOne
}
return [ ] proposals . CorrectionProposal {
2026-11-11 17:05:52 +00:00
{ TargetSegmentID : seg . ID , OriginalText : seg . Text , CorrectedText : strings . ToUpper ( seg . Text ) } ,
2026-05-12 18:39:26 -05:00
} , nil
} ,
} ,
} } )
resultCh := make ( chan error , 1 )
go func ( ) {
_ , err := r . Run ( context . Background ( ) , RunInput {
Config : & cfg ,
Transcript : transcript ,
ModuleSpecs : [ ] contracts . ModuleRunSpec {
{ ModuleKey : "m" , InstanceName : "m" } ,
} ,
} )
resultCh <- err
} ( )
select {
case <- validatorStarted :
case <- time . After ( 350 * time . Millisecond ) :
t . Fatal ( "expected section-level validator work before all proposal sections complete" )
}
close ( releaseSectionOne )
if err := <- resultCh ; err != nil {
t . Fatalf ( "Run error: %v" , err )
}
}
type stageAwareStructuredClient struct {
startedSection chan int
releaseValidation <- chan struct { }
eventSink chan <- string
}
2026-05-13 02:10:24 +00:00
type captureContractStructuredClient struct {
lastRequest contracts . StructuredCompletionRequest
}
func ( c * captureContractStructuredClient ) CompleteStructured ( ctx context . Context , req contracts . StructuredCompletionRequest , out any ) ( contracts . StructuredCompletionResponse , error ) {
_ = ctx
c . lastRequest = req
if target , ok := out . ( * validators . LLMValidationResponse ) ; ok {
* target = validators . LLMValidationResponse {
Validations : [ ] validators . LLMValidationDecision {
2026-11-16 14:11:02 +00:00
{ CorrectionIndex : "1" , Approved : true , Reason : "ok" } ,
2026-05-13 02:10:24 +00:00
} ,
}
}
return contracts . StructuredCompletionResponse { } , nil
}
2026-05-12 18:39:26 -05:00
func ( c * stageAwareStructuredClient ) CompleteStructured ( ctx context . Context , req contracts . StructuredCompletionRequest , out any ) ( contracts . StructuredCompletionResponse , error ) {
2026-09-24 02:18:31 +00:00
section := 0
2026-09-24 15:32:13 +00:00
if req . Context . SectionIndex != nil {
section = * req . Context . SectionIndex
2026-09-24 02:18:31 +00:00
}
2026-05-12 18:39:26 -05:00
if c . startedSection != nil {
select {
case c . startedSection <- section :
default :
}
}
if c . eventSink != nil {
c . eventSink <- "v" + strconv . Itoa ( section )
}
if c . releaseValidation != nil {
select {
case <- c . releaseValidation :
case <- ctx . Done ( ) :
return contracts . StructuredCompletionResponse { } , ctx . Err ( )
}
}
target := out . ( * validators . LLMValidationResponse )
* target = validators . LLMValidationResponse {
Validations : [ ] validators . LLMValidationDecision {
2026-11-16 14:11:02 +00:00
{ CorrectionIndex : "1" , Approved : true , Reason : "ok" } ,
2026-05-12 18:39:26 -05:00
} ,
}
return contracts . StructuredCompletionResponse { } , nil
}
2026-05-11 14:22:22 +00:00
func TestRunnerSkippedRecorded ( t * testing . T ) {
transcript := & schema . Transcript { Segments : [ ] schema . Segment { { ID : 1 , Speaker : "A" , Start : 0 , End : 1 , Text : "word word" } } }
r := New ( fakeFactory { modules : map [ string ] contracts . TranscriptModule {
"m" : fakeModule { key : "m" , policy : proposals . ReplacementPolicyRequireUnique , proposeF : func ( req contracts . ProposalRequest ) ( [ ] proposals . CorrectionProposal , error ) {
2026-11-11 17:05:52 +00:00
return [ ] proposals . CorrectionProposal { { TargetSegmentID : 1 , OriginalText : "word" , CorrectedText : "term" } } , nil
2026-05-11 14:22:22 +00:00
} } ,
} } )
out , err := r . Run ( context . Background ( ) , RunInput { Transcript : transcript , ModuleSpecs : [ ] contracts . ModuleRunSpec { { ModuleKey : "m" , InstanceName : "m" } } } )
if err != nil {
t . Fatalf ( "Run error: %v" , err )
}
if len ( out . ModuleResults [ 0 ] . SkippedChanges ) != 1 {
t . Fatalf ( "expected one skipped change, got %+v" , out . ModuleResults [ 0 ] . SkippedChanges )
}
}
func TestRunnerFailureReturnsPartialProgress ( t * testing . T ) {
transcript := & schema . Transcript { Segments : [ ] schema . Segment { { ID : 1 , Speaker : "A" , Start : 0 , End : 1 , Text : "teh cat" } } }
r := New ( fakeFactory { modules : map [ string ] contracts . TranscriptModule {
"m1" : fakeModule { key : "m1" , policy : proposals . ReplacementPolicyRequireUnique , proposeF : func ( req contracts . ProposalRequest ) ( [ ] proposals . CorrectionProposal , error ) {
2026-11-11 17:05:52 +00:00
return [ ] proposals . CorrectionProposal { { TargetSegmentID : 1 , OriginalText : "teh" , CorrectedText : "the" } } , nil
2026-05-11 14:22:22 +00:00
} } ,
"m2" : fakeModule { key : "m2" , policy : proposals . ReplacementPolicyRequireUnique , proposeF : func ( req contracts . ProposalRequest ) ( [ ] proposals . CorrectionProposal , error ) {
return nil , errors . New ( "boom" )
} } ,
} } )
out , err := r . Run ( context . Background ( ) , RunInput { Transcript : transcript , ModuleSpecs : [ ] contracts . ModuleRunSpec { { ModuleKey : "m1" , InstanceName : "m1" } , { ModuleKey : "m2" , InstanceName : "m2" } } } )
if err == nil {
t . Fatal ( "expected error" )
}
if out . FinalTranscript . Segments [ 0 ] . Text != "the cat" {
t . Fatalf ( "expected partial transcript progress preserved, got %q" , out . FinalTranscript . Segments [ 0 ] . Text )
}
if len ( out . ModuleResults ) != 2 || out . ModuleResults [ 1 ] . Status != ModuleStatusFailed {
t . Fatalf ( "expected second module failed in results, got %+v" , out . ModuleResults )
}
}
func TestRunnerDoesNotMutateInputTranscript ( t * testing . T ) {
transcript := & schema . Transcript { Segments : [ ] schema . Segment { { ID : 1 , Speaker : "A" , Start : 0 , End : 1 , Text : "teh" } } }
before := transcript . Segments [ 0 ] . Text
r := New ( fakeFactory { modules : map [ string ] contracts . TranscriptModule {
"m" : fakeModule { key : "m" , policy : proposals . ReplacementPolicyRequireUnique , proposeF : func ( req contracts . ProposalRequest ) ( [ ] proposals . CorrectionProposal , error ) {
2026-11-11 17:05:52 +00:00
return [ ] proposals . CorrectionProposal { { TargetSegmentID : 1 , OriginalText : "teh" , CorrectedText : "the" } } , nil
2026-05-11 14:22:22 +00:00
} } ,
} } )
2026-09-24 02:18:31 +00:00
_ , err := r . Run ( context . Background ( ) , RunInput { Config : ptrConfig ( runnerTestConfig ( ) ) , Transcript : transcript , ModuleSpecs : [ ] contracts . ModuleRunSpec { { ModuleKey : "m" , InstanceName : "m" } } } )
2026-05-11 14:22:22 +00:00
if err != nil {
t . Fatalf ( "Run error: %v" , err )
}
if transcript . Segments [ 0 ] . Text != before {
t . Fatalf ( "expected input transcript unchanged, got %q" , transcript . Segments [ 0 ] . Text )
}
}
func TestRunnerRepeatedModuleInstanceNames ( t * testing . T ) {
2026-11-10 18:13:12 +00:00
specs := contracts . ResolveModuleRunSpecs ( & config . Config { Pipeline : config . Pipeline { Stages : [ ] config . Stage { { Module : "glossary" } , { Module : "glossary" } } } } )
2026-05-11 14:22:22 +00:00
if specs [ 0 ] . InstanceName != "glossary_1" || specs [ 1 ] . InstanceName != "glossary_2" {
t . Fatalf ( "unexpected instance names: %+v" , specs )
}
}
2026-05-12 00:26:27 +00:00
func TestRunnerValidatorApprovedProposalApplied ( t * testing . T ) {
transcript := & schema . Transcript { Segments : [ ] schema . Segment { { ID : 1 , Text : "teh cat" } } }
2026-11-16 14:11:02 +00:00
allowAll := fakeValidator { name : "allow" , validateF : func ( req validators . Request ) ( validators . Result , error ) {
2026-05-12 00:26:27 +00:00
decisions := make ( [ ] validators . Decision , len ( req . CandidateProposal ) )
for i , p := range req . CandidateProposal {
decisions [ i ] = validators . Decision { ProposalIndex : p . ProposalIndex , Approved : true , ReasonCode : validators . ReasonApproved , Message : "approved" }
}
return validators . Result { ValidatorName : "allow" , Decisions : decisions } , nil
} }
r := New ( fakeFactory { modules : map [ string ] contracts . TranscriptModule {
"m" : fakeModule {
key : "m" ,
policy : proposals . ReplacementPolicyRequireUnique ,
2026-11-16 14:11:02 +00:00
validators : [ ] validators . Validator { allowAll } ,
2026-05-12 00:26:27 +00:00
proposeF : func ( req contracts . ProposalRequest ) ( [ ] proposals . CorrectionProposal , error ) {
2026-11-11 17:05:52 +00:00
return [ ] proposals . CorrectionProposal { { TargetSegmentID : 1 , OriginalText : "teh" , CorrectedText : "the" } } , nil
2026-05-12 00:26:27 +00:00
} ,
} ,
} } )
out , err := r . Run ( context . Background ( ) , RunInput { Transcript : transcript , ModuleSpecs : [ ] contracts . ModuleRunSpec { { ModuleKey : "m" , InstanceName : "m" } } } )
if err != nil {
t . Fatalf ( "Run error: %v" , err )
}
if out . FinalTranscript . Segments [ 0 ] . Text != "the cat" {
t . Fatalf ( "expected proposal applied, got %q" , out . FinalTranscript . Segments [ 0 ] . Text )
}
if len ( out . ModuleResults [ 0 ] . ValidatorDecisions ) != 1 {
t . Fatalf ( "expected validator decisions recorded" )
}
}
func TestRunnerValidatorRejectedProposalNotApplied ( t * testing . T ) {
transcript := & schema . Transcript { Segments : [ ] schema . Segment { { ID : 1 , Text : "teh cat" } } }
2026-11-16 14:11:02 +00:00
rejectAll := fakeValidator { name : "reject" , validateF : func ( req validators . Request ) ( validators . Result , error ) {
2026-05-12 00:26:27 +00:00
decisions := make ( [ ] validators . Decision , len ( req . CandidateProposal ) )
for i , p := range req . CandidateProposal {
decisions [ i ] = validators . Decision { ProposalIndex : p . ProposalIndex , Approved : false , ReasonCode : validators . ReasonNoEffect , Message : "rejected" }
}
return validators . Result { ValidatorName : "reject" , Decisions : decisions } , nil
} }
r := New ( fakeFactory { modules : map [ string ] contracts . TranscriptModule {
2026-11-16 14:11:02 +00:00
"m" : fakeModule { key : "m" , policy : proposals . ReplacementPolicyRequireUnique , validators : [ ] validators . Validator { rejectAll } , proposeF : func ( req contracts . ProposalRequest ) ( [ ] proposals . CorrectionProposal , error ) {
2026-11-11 17:05:52 +00:00
return [ ] proposals . CorrectionProposal { { TargetSegmentID : 1 , OriginalText : "teh" , CorrectedText : "the" } } , nil
2026-05-12 00:26:27 +00:00
} } ,
} } )
out , err := r . Run ( context . Background ( ) , RunInput { Transcript : transcript , ModuleSpecs : [ ] contracts . ModuleRunSpec { { ModuleKey : "m" , InstanceName : "m" } } } )
if err != nil {
t . Fatalf ( "Run error: %v" , err )
}
if out . FinalTranscript . Segments [ 0 ] . Text != "teh cat" {
t . Fatalf ( "expected rejected proposal not applied, got %q" , out . FinalTranscript . Segments [ 0 ] . Text )
}
if len ( out . ModuleResults [ 0 ] . ValidatorRejected ) != 1 {
t . Fatalf ( "expected validator rejection recorded, got %+v" , out . ModuleResults [ 0 ] . ValidatorRejected )
}
}
func TestRunnerMultipleValidatorsRunInOrderAndFilterSurvivors ( t * testing . T ) {
transcript := & schema . Transcript { Segments : [ ] schema . Segment { { ID : 1 , Text : "one two" } } }
2026-11-16 14:11:02 +00:00
first := fakeValidator { name : "first" , validateF : func ( req validators . Request ) ( validators . Result , error ) {
2026-05-12 00:26:27 +00:00
if len ( req . CandidateProposal ) != 2 {
t . Fatalf ( "expected first validator to see 2 candidates, got %d" , len ( req . CandidateProposal ) )
}
return validators . Result {
ValidatorName : "first" ,
Decisions : [ ] validators . Decision {
{ ProposalIndex : req . CandidateProposal [ 0 ] . ProposalIndex , Approved : true , ReasonCode : validators . ReasonApproved , Message : "ok" } ,
{ ProposalIndex : req . CandidateProposal [ 1 ] . ProposalIndex , Approved : false , ReasonCode : validators . ReasonNoEffect , Message : "reject" } ,
} ,
} , nil
} }
2026-11-16 14:11:02 +00:00
second := fakeValidator { name : "second" , validateF : func ( req validators . Request ) ( validators . Result , error ) {
2026-05-12 00:26:27 +00:00
if len ( req . CandidateProposal ) != 1 {
t . Fatalf ( "expected second validator to see only survivors, got %d" , len ( req . CandidateProposal ) )
}
return validators . Result { ValidatorName : "second" , Decisions : [ ] validators . Decision { { ProposalIndex : req . CandidateProposal [ 0 ] . ProposalIndex , Approved : true , ReasonCode : validators . ReasonApproved , Message : "ok" } } } , nil
} }
r := New ( fakeFactory { modules : map [ string ] contracts . TranscriptModule {
2026-11-16 14:11:02 +00:00
"m" : fakeModule { key : "m" , policy : proposals . ReplacementPolicyRequireUnique , validators : [ ] validators . Validator { first , second } , proposeF : func ( req contracts . ProposalRequest ) ( [ ] proposals . CorrectionProposal , error ) {
2026-05-12 00:26:27 +00:00
return [ ] proposals . CorrectionProposal {
2026-11-11 17:05:52 +00:00
{ TargetSegmentID : 1 , OriginalText : "one" , CorrectedText : "ONE" } ,
{ TargetSegmentID : 1 , OriginalText : "two" , CorrectedText : "TWO" } ,
2026-05-12 00:26:27 +00:00
} , nil
} } ,
} } )
out , err := r . Run ( context . Background ( ) , RunInput { Transcript : transcript , ModuleSpecs : [ ] contracts . ModuleRunSpec { { ModuleKey : "m" , InstanceName : "m" } } } )
if err != nil {
t . Fatalf ( "Run error: %v" , err )
}
if out . FinalTranscript . Segments [ 0 ] . Text != "ONE two" {
t . Fatalf ( "expected only survivor applied, got %q" , out . FinalTranscript . Segments [ 0 ] . Text )
}
}
func TestRunnerValidatorCardinalityErrorStopsPipelineWithPartialProgress ( t * testing . T ) {
transcript := & schema . Transcript { Segments : [ ] schema . Segment { { ID : 1 , Text : "teh cat" } } }
good := fakeModule { key : "m1" , policy : proposals . ReplacementPolicyRequireUnique , proposeF : func ( req contracts . ProposalRequest ) ( [ ] proposals . CorrectionProposal , error ) {
2026-11-11 17:05:52 +00:00
return [ ] proposals . CorrectionProposal { { TargetSegmentID : 1 , OriginalText : "teh" , CorrectedText : "the" } } , nil
2026-05-12 00:26:27 +00:00
} }
2026-11-16 14:11:02 +00:00
badValidator := fakeValidator { name : "bad" , validateF : func ( req validators . Request ) ( validators . Result , error ) {
2026-05-12 00:26:27 +00:00
// Missing one decision triggers cardinality error.
return validators . Result { ValidatorName : "bad" , Decisions : nil } , nil
} }
2026-11-16 14:11:02 +00:00
bad := fakeModule { key : "m2" , policy : proposals . ReplacementPolicyRequireUnique , validators : [ ] validators . Validator { badValidator } , proposeF : func ( req contracts . ProposalRequest ) ( [ ] proposals . CorrectionProposal , error ) {
2026-11-11 17:05:52 +00:00
return [ ] proposals . CorrectionProposal { { TargetSegmentID : 1 , OriginalText : "cat" , CorrectedText : "dog" } } , nil
2026-05-12 00:26:27 +00:00
} }
r := New ( fakeFactory { modules : map [ string ] contracts . TranscriptModule { "m1" : good , "m2" : bad } } )
out , err := r . Run ( context . Background ( ) , RunInput { Transcript : transcript , ModuleSpecs : [ ] contracts . ModuleRunSpec { { ModuleKey : "m1" , InstanceName : "m1" } , { ModuleKey : "m2" , InstanceName : "m2" } } } )
if err == nil {
t . Fatal ( "expected cardinality error" )
}
if out . FinalTranscript . Segments [ 0 ] . Text != "the cat" {
t . Fatalf ( "expected partial progress preserved, got %q" , out . FinalTranscript . Segments [ 0 ] . Text )
}
if len ( out . ModuleResults ) != 2 || out . ModuleResults [ 1 ] . Status != ModuleStatusFailed {
t . Fatalf ( "expected second module failed" )
}
}
func TestRunnerApplicationSkipAfterValidatorApprovalReported ( t * testing . T ) {
transcript := & schema . Transcript { Segments : [ ] schema . Segment { { ID : 1 , Text : "word word" } } }
2026-11-16 14:11:02 +00:00
allow := fakeValidator { name : "allow" , validateF : func ( req validators . Request ) ( validators . Result , error ) {
2026-05-12 00:26:27 +00:00
return validators . Result { ValidatorName : "allow" , Decisions : [ ] validators . Decision { { ProposalIndex : req . CandidateProposal [ 0 ] . ProposalIndex , Approved : true , ReasonCode : validators . ReasonApproved , Message : "ok" } } } , nil
} }
r := New ( fakeFactory { modules : map [ string ] contracts . TranscriptModule {
2026-11-16 14:11:02 +00:00
"m" : fakeModule { key : "m" , policy : proposals . ReplacementPolicyRequireUnique , validators : [ ] validators . Validator { allow } , proposeF : func ( req contracts . ProposalRequest ) ( [ ] proposals . CorrectionProposal , error ) {
2026-11-11 17:05:52 +00:00
return [ ] proposals . CorrectionProposal { { TargetSegmentID : 1 , OriginalText : "word" , CorrectedText : "term" } } , nil
2026-05-12 00:26:27 +00:00
} } ,
} } )
out , err := r . Run ( context . Background ( ) , RunInput { Transcript : transcript , ModuleSpecs : [ ] contracts . ModuleRunSpec { { ModuleKey : "m" , InstanceName : "m" } } } )
if err != nil {
t . Fatalf ( "Run error: %v" , err )
}
if len ( out . ModuleResults [ 0 ] . SkippedChanges ) != 1 {
t . Fatalf ( "expected application skip recorded" )
}
if len ( out . ModuleResults [ 0 ] . ValidatorRejected ) != 0 {
t . Fatalf ( "expected no validator rejection" )
}
}
2026-09-25 02:48:57 +00:00
func TestRunnerApplyConflictAfterTwoInitiallyValidDeletions ( t * testing . T ) {
transcript := & schema . Transcript { Segments : [ ] schema . Segment { { ID : 1 , Text : "a b" } } }
r := New ( fakeFactory { modules : map [ string ] contracts . TranscriptModule {
2026-09-25 03:06:57 +00:00
"m" : outcomeModule {
2026-11-16 14:11:02 +00:00
fakeModule : fakeModule { key : "m" , policy : proposals . ReplacementPolicyRequireUnique , validators : [ ] validators . Validator { validators . ProposalShapeValidator { } , validators . NonEmptyResultingSegmentValidator { } } } ,
2026-09-25 03:06:57 +00:00
proposeResult : func ( req contracts . ProposalRequest ) ( contracts . ProposalResult , error ) {
return contracts . ProposalResult {
Proposals : [ ] proposals . CorrectionProposal {
2026-11-11 17:05:52 +00:00
{ TargetSegmentID : 1 , OriginalText : "a" , CorrectedText : "" } ,
{ TargetSegmentID : 1 , OriginalText : "b" , CorrectedText : "" } ,
2026-09-25 03:06:57 +00:00
} ,
Operations : [ ] outcomes . Operation { { Reference : outcomes . Reference { Kind : outcomes . OperationProposal } , Succeeded : true } } ,
} , nil
} ,
} ,
2026-09-25 02:48:57 +00:00
} } )
out , err := r . Run ( context . Background ( ) , RunInput { Transcript : transcript , ModuleSpecs : [ ] contracts . ModuleRunSpec { { ModuleKey : "m" , InstanceName : "m" } } , CaptureCandidates : true } )
if err != nil {
t . Fatal ( err )
}
if got := out . FinalTranscript . Segments [ 0 ] . Text ; got != " b" {
t . Fatalf ( "expected first deletion only, got %q" , got )
}
if len ( out . ModuleResults ) != 1 || len ( out . ModuleResults [ 0 ] . SkippedChanges ) != 1 || out . ModuleResults [ 0 ] . SkippedChanges [ 0 ] . SkipReason != proposals . SkipReasonEmptyResultingText {
t . Fatalf ( "expected skipped debug evidence: %+v" , out . ModuleResults )
}
if len ( out . ModuleResults [ 0 ] . ValidatorRejected ) != 0 || len ( out . WorkUnits ) != 1 || out . WorkUnits [ 0 ] . CorrectionsApplied != 1 {
t . Fatalf ( "unexpected validation or application outcome: %+v" , out )
}
2026-09-25 03:06:57 +00:00
foundProposal := false
2026-09-25 02:48:57 +00:00
for _ , operation := range out . WorkUnits [ 0 ] . Operations {
2026-09-25 03:06:57 +00:00
if operation . Reference . Kind == outcomes . OperationProposal && operation . Succeeded {
foundProposal = true
}
if operation . FailureReason != "" || len ( operation . RecoveryReasons ) != 0 || len ( operation . Filtered ) != 0 {
2026-09-25 02:48:57 +00:00
t . Fatalf ( "apply-time conflict must not count as model error or filtering recovery: %+v" , operation )
}
}
2026-09-25 03:06:57 +00:00
if ! foundProposal {
t . Fatalf ( "expected successful proposal operation: %+v" , out . WorkUnits [ 0 ] . Operations )
}
2026-09-25 02:48:57 +00:00
}
2026-05-11 14:22:22 +00:00
func ptrConfig ( c config . Config ) * config . Config { return & c }
2026-05-12 01:25:52 +00:00
type fakeStructuredClient struct {
2026-11-16 03:37:29 +00:00
responses [ ] validators . LLMValidationResponse
err error
calls int
2026-05-12 01:25:52 +00:00
}
func ( f * fakeStructuredClient ) CompleteStructured ( ctx context . Context , req contracts . StructuredCompletionRequest , out any ) ( contracts . StructuredCompletionResponse , error ) {
_ = ctx
_ = req
f . calls ++
if f . err != nil {
return contracts . StructuredCompletionResponse { } , f . err
}
if len ( f . responses ) == 0 {
return contracts . StructuredCompletionResponse { } , errors . New ( "unexpected call" )
}
resp := f . responses [ 0 ]
f . responses = f . responses [ 1 : ]
target := out . ( * validators . LLMValidationResponse )
* target = resp
return contracts . StructuredCompletionResponse { } , nil
}
func TestRunnerLLMValidatorApprovalApplied ( t * testing . T ) {
2026-11-16 14:11:02 +00:00
client := & fakeStructuredClient { responses : [ ] validators . LLMValidationResponse { { Validations : [ ] validators . LLMValidationDecision { { CorrectionIndex : "1" , Approved : true , Reason : "ok" } } } } }
2026-09-24 02:18:31 +00:00
llmValidator , err := validators . NewLLMBackedValidator ( "spoken_form_plausibility_review" , validators . LLMValidatorTypeSpokenFormPlausibility )
2026-05-12 01:25:52 +00:00
if err != nil {
t . Fatalf ( "NewLLMBackedValidator: %v" , err )
}
r := New ( fakeFactory { modules : map [ string ] contracts . TranscriptModule {
2026-11-16 14:11:02 +00:00
"m" : fakeModule { key : "m" , policy : proposals . ReplacementPolicyRequireUnique , validators : [ ] validators . Validator { llmValidator } , proposeF : func ( req contracts . ProposalRequest ) ( [ ] proposals . CorrectionProposal , error ) {
2026-11-11 17:05:52 +00:00
return [ ] proposals . CorrectionProposal { { TargetSegmentID : 1 , OriginalText : "gestures" , CorrectedText : "Jesters" } } , nil
2026-05-12 01:25:52 +00:00
} } ,
} } )
2026-09-24 02:18:31 +00:00
cfg := runnerTestConfig ( )
2026-05-12 01:25:52 +00:00
out , err := r . Run ( context . Background ( ) , RunInput {
2026-11-10 18:13:12 +00:00
Config : & cfg ,
Transcript : & schema . Transcript { Segments : [ ] schema . Segment { { ID : 1 , Text : "There were gestures at the temple." } } } ,
ModuleSpecs : [ ] contracts . ModuleRunSpec { { ModuleKey : "m" , InstanceName : "m" , ValidationClient : client , Settings : config . Stage { Validation : cfg . Validation } } } ,
2026-05-12 01:25:52 +00:00
} )
if err != nil {
t . Fatalf ( "Run error: %v" , err )
}
if out . FinalTranscript . Segments [ 0 ] . Text != "There were Jesters at the temple." {
t . Fatalf ( "expected applied LLM-approved proposal, got %q" , out . FinalTranscript . Segments [ 0 ] . Text )
}
}
func TestRunnerLLMValidatorRejectionPreventsApplication ( t * testing . T ) {
2026-11-16 14:11:02 +00:00
client := & fakeStructuredClient { responses : [ ] validators . LLMValidationResponse { { Validations : [ ] validators . LLMValidationDecision { { CorrectionIndex : "1" , Approved : false , Reason : "reject" } } } } }
2026-09-24 02:18:31 +00:00
llmValidator , _ := validators . NewLLMBackedValidator ( "spoken_form_plausibility_review" , validators . LLMValidatorTypeSpokenFormPlausibility )
2026-05-12 01:25:52 +00:00
r := New ( fakeFactory { modules : map [ string ] contracts . TranscriptModule {
2026-11-16 14:11:02 +00:00
"m" : fakeModule { key : "m" , policy : proposals . ReplacementPolicyRequireUnique , validators : [ ] validators . Validator { llmValidator } , proposeF : func ( req contracts . ProposalRequest ) ( [ ] proposals . CorrectionProposal , error ) {
2026-11-11 17:05:52 +00:00
return [ ] proposals . CorrectionProposal { { TargetSegmentID : 1 , OriginalText : "gestures" , CorrectedText : "Jesters" } } , nil
2026-05-12 01:25:52 +00:00
} } ,
} } )
2026-09-24 02:18:31 +00:00
cfg := runnerTestConfig ( )
2026-05-12 01:25:52 +00:00
out , err := r . Run ( context . Background ( ) , RunInput {
2026-11-10 18:13:12 +00:00
Config : & cfg ,
Transcript : & schema . Transcript { Segments : [ ] schema . Segment { { ID : 1 , Text : "There were gestures at the temple." } } } ,
ModuleSpecs : [ ] contracts . ModuleRunSpec { { ModuleKey : "m" , InstanceName : "m" , ValidationClient : client , Settings : config . Stage { Validation : cfg . Validation } } } ,
2026-05-12 01:25:52 +00:00
} )
if err != nil {
t . Fatalf ( "Run error: %v" , err )
}
if out . FinalTranscript . Segments [ 0 ] . Text != "There were gestures at the temple." {
t . Fatalf ( "expected rejected proposal not applied" )
}
if len ( out . ModuleResults [ 0 ] . ValidatorRejected ) != 1 {
t . Fatalf ( "expected validator rejection record" )
}
}
2026-05-23 10:07:06 -05:00
func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress ( t * testing . T ) {
2026-11-16 14:11:02 +00:00
client := & fakeStructuredClient { responses : [ ] validators . LLMValidationResponse { { Validations : [ ] validators . LLMValidationDecision { { CorrectionIndex : "99" , Approved : true , Reason : "bad index" } } } } }
2026-09-24 02:18:31 +00:00
llmValidator , _ := validators . NewLLMBackedValidator ( "spoken_form_plausibility_review" , validators . LLMValidatorTypeSpokenFormPlausibility )
2026-05-12 01:25:52 +00:00
r := New ( fakeFactory { modules : map [ string ] contracts . TranscriptModule {
"m1" : fakeModule { key : "m1" , policy : proposals . ReplacementPolicyRequireUnique , proposeF : func ( req contracts . ProposalRequest ) ( [ ] proposals . CorrectionProposal , error ) {
2026-11-11 17:05:52 +00:00
return [ ] proposals . CorrectionProposal { { TargetSegmentID : 1 , OriginalText : "teh" , CorrectedText : "the" } } , nil
2026-05-12 01:25:52 +00:00
} } ,
2026-11-16 14:11:02 +00:00
"m2" : fakeModule { key : "m2" , policy : proposals . ReplacementPolicyRequireUnique , validators : [ ] validators . Validator { llmValidator } , proposeF : func ( req contracts . ProposalRequest ) ( [ ] proposals . CorrectionProposal , error ) {
2026-11-11 17:05:52 +00:00
return [ ] proposals . CorrectionProposal { { TargetSegmentID : 1 , OriginalText : "cat" , CorrectedText : "dog" } } , nil
2026-05-12 01:25:52 +00:00
} } ,
} } )
2026-09-24 02:18:31 +00:00
cfg := runnerTestConfig ( )
2026-05-12 01:25:52 +00:00
out , err := r . Run ( context . Background ( ) , RunInput {
2026-11-10 18:13:12 +00:00
Config : & cfg ,
Transcript : & schema . Transcript { Segments : [ ] schema . Segment { { ID : 1 , Text : "teh cat" } } } ,
ModuleSpecs : [ ] contracts . ModuleRunSpec { { ModuleKey : "m1" , InstanceName : "m1" } , { ModuleKey : "m2" , InstanceName : "m2" , ValidationClient : client , Settings : config . Stage { Validation : cfg . Validation } } } ,
2026-05-12 01:25:52 +00:00
} )
2026-05-23 10:07:06 -05:00
if err != nil {
t . Fatalf ( "expected malformed validator response to downgrade, got %v" , err )
2026-05-12 01:25:52 +00:00
}
if out . FinalTranscript . Segments [ 0 ] . Text != "the cat" {
t . Fatalf ( "expected partial progress retained" )
}
2026-05-23 10:07:06 -05:00
if len ( out . ModuleResults ) != 2 || len ( out . ModuleResults [ 1 ] . ValidatorRejected ) != 1 {
t . Fatalf ( "expected second module rejection, got %+v" , out . ModuleResults )
}
2026-09-24 17:23:59 +00:00
if len ( out . ModuleResults [ 1 ] . Operations ) != 1 || out . ModuleResults [ 1 ] . Operations [ 0 ] . FailureReason != outcomes . ReasonValidationDecisionsInvalid {
t . Fatalf ( "expected invalid decision-set operation, got %+v" , out . ModuleResults [ 1 ] . Operations )
2026-05-23 10:07:06 -05:00
}
2026-05-12 01:25:52 +00:00
}
2026-09-25 11:25:22 +00:00
func TestRunnerReconciledSurvivorsCompleteLaterValidation ( t * testing . T ) {
client := & fakeStructuredClient { responses : [ ] validators . LLMValidationResponse { { Validations : [ ] validators . LLMValidationDecision {
2026-11-16 14:11:02 +00:00
{ CorrectionIndex : "1" , Approved : true , Reason : "ok" } ,
{ CorrectionIndex : "3" , Approved : true , Reason : "ok" } ,
{ CorrectionIndex : "99" , Approved : true , Reason : "unexpected" } ,
2026-09-25 11:25:22 +00:00
} } } }
first , _ := validators . NewLLMBackedValidator ( "editorial_review" , validators . LLMValidatorTypeEditorialReview )
var laterIndices [ ] int
2026-11-16 14:11:02 +00:00
later := fakeValidator { name : "later_review" , validateF : func ( req validators . Request ) ( validators . Result , error ) {
2026-09-25 11:25:22 +00:00
decisions := make ( [ ] validators . Decision , 0 , len ( req . CandidateProposal ) )
for _ , candidate := range req . CandidateProposal {
laterIndices = append ( laterIndices , candidate . ProposalIndex )
approved := candidate . ProposalIndex == 0
disposition := validators . DispositionRejected
if approved {
disposition = validators . DispositionApproved
}
decisions = append ( decisions , validators . Decision { ProposalIndex : candidate . ProposalIndex , Approved : approved , Disposition : disposition , ReasonCode : "later_review" } )
}
return validators . Result { ValidatorName : "later_review" , Decisions : decisions } , nil
} }
2026-11-16 14:11:02 +00:00
module := outcomeModule { fakeModule : fakeModule { key : "m" , policy : proposals . ReplacementPolicyRequireUnique , validators : [ ] validators . Validator { first , validatormetadata . Wrap ( later , validatormetadata . ExecutionClassLLMBacked ) } } , proposeResult : func ( contracts . ProposalRequest ) ( contracts . ProposalResult , error ) {
2026-09-25 11:25:22 +00:00
return contracts . ProposalResult { Proposals : [ ] proposals . CorrectionProposal {
2026-11-11 17:05:52 +00:00
{ TargetSegmentID : 1 , OriginalText : "teh" , CorrectedText : "the" } ,
{ TargetSegmentID : 1 , OriginalText : "cat" , CorrectedText : "dog" } ,
{ TargetSegmentID : 1 , OriginalText : "bad" , CorrectedText : "good" } ,
2026-09-25 11:25:22 +00:00
} , Operations : [ ] outcomes . Operation { { Reference : outcomes . Reference { Kind : outcomes . OperationProposal } , Succeeded : true } } } , nil
} }
r := New ( fakeFactory { modules : map [ string ] contracts . TranscriptModule { "m" : module } } )
cfg := runnerTestConfig ( )
out , err := r . Run ( context . Background ( ) , RunInput {
Config : & cfg , Transcript : & schema . Transcript { Segments : [ ] schema . Segment { { ID : 1 , Text : "teh cat bad" } } } ,
2026-11-10 18:13:12 +00:00
ModuleSpecs : [ ] contracts . ModuleRunSpec { { ModuleKey : "m" , InstanceName : "m" , ValidationClient : client , Settings : config . Stage { Validation : cfg . Validation } } } ,
2026-09-25 11:25:22 +00:00
} )
if err != nil || out . FinalTranscript . Segments [ 0 ] . Text != "the cat bad" || len ( laterIndices ) != 2 || laterIndices [ 0 ] != 0 || laterIndices [ 1 ] != 2 || client . calls != 1 {
t . Fatalf ( "survivor routing: text=%q later=%v calls=%d err=%v" , out . FinalTranscript . Segments [ 0 ] . Text , laterIndices , client . calls , err )
}
unit := out . WorkUnits [ 0 ]
if unit . CorrectionsApplied != 1 || len ( unit . Operations ) != 2 || ! unit . Operations [ 0 ] . Succeeded || unit . Operations [ 1 ] . FailureReason != outcomes . ReasonValidationDecisionsInvalid || len ( unit . Operations [ 1 ] . ProposalIndices ) != 1 || unit . Operations [ 1 ] . ProposalIndices [ 0 ] != 1 || len ( unit . Operations [ 1 ] . RecoveryReasons ) != 0 {
t . Fatalf ( "operation accounting: %+v" , unit )
}
2026-09-25 11:28:44 +00:00
moduleResult := out . ModuleResults [ 0 ]
if len ( moduleResult . ValidatorDecisions ) != 5 || len ( moduleResult . ValidatorRejected ) != 2 || moduleResult . ValidatorDecisions [ 2 ] . Disposition != validators . DispositionUnavailable || moduleResult . ValidatorRejected [ 0 ] . Disposition != validators . DispositionUnavailable || moduleResult . ValidatorRejected [ 1 ] . Disposition != validators . DispositionRejected {
t . Fatalf ( "validator dispositions: %+v" , moduleResult )
}
2026-09-25 11:25:22 +00:00
}
2026-05-23 10:07:06 -05:00
func TestRunnerLLMValidatorMissingDecisionRejectsBatch ( t * testing . T ) {
2026-05-12 01:25:52 +00:00
client := & fakeStructuredClient { responses : [ ] validators . LLMValidationResponse { { Validations : [ ] validators . LLMValidationDecision { } } } }
2026-09-24 02:18:31 +00:00
llmValidator , _ := validators . NewLLMBackedValidator ( "spoken_form_plausibility_review" , validators . LLMValidatorTypeSpokenFormPlausibility )
2026-05-12 01:25:52 +00:00
r := New ( fakeFactory { modules : map [ string ] contracts . TranscriptModule {
2026-11-16 14:11:02 +00:00
"m" : fakeModule { key : "m" , policy : proposals . ReplacementPolicyRequireUnique , validators : [ ] validators . Validator { llmValidator } , proposeF : func ( req contracts . ProposalRequest ) ( [ ] proposals . CorrectionProposal , error ) {
2026-11-11 17:05:52 +00:00
return [ ] proposals . CorrectionProposal { { TargetSegmentID : 1 , OriginalText : "gestures" , CorrectedText : "Jesters" } } , nil
2026-05-12 01:25:52 +00:00
} } ,
} } )
2026-09-24 02:18:31 +00:00
cfg := runnerTestConfig ( )
2026-05-12 01:25:52 +00:00
_ , err := r . Run ( context . Background ( ) , RunInput {
2026-11-10 18:13:12 +00:00
Config : & cfg ,
Transcript : & schema . Transcript { Segments : [ ] schema . Segment { { ID : 1 , Text : "There were gestures at the temple." } } } ,
ModuleSpecs : [ ] contracts . ModuleRunSpec { { ModuleKey : "m" , InstanceName : "m" , ValidationClient : client , Settings : config . Stage { Validation : cfg . Validation } } } ,
2026-05-12 01:25:52 +00:00
} )
2026-05-23 10:07:06 -05:00
if err != nil {
t . Fatalf ( "expected missing decision downgrade, got %v" , err )
2026-05-12 01:25:52 +00:00
}
}
2026-05-23 10:07:06 -05:00
func TestRunnerLLMValidatorDuplicateDecisionRejectsBatch ( t * testing . T ) {
2026-05-12 01:25:52 +00:00
client := & fakeStructuredClient { responses : [ ] validators . LLMValidationResponse { { Validations : [ ] validators . LLMValidationDecision {
2026-11-16 14:11:02 +00:00
{ CorrectionIndex : "1" , Approved : true , Reason : "ok" } ,
{ CorrectionIndex : "1" , Approved : false , Reason : "dup" } ,
2026-05-12 01:25:52 +00:00
} } } }
2026-09-24 02:18:31 +00:00
llmValidator , _ := validators . NewLLMBackedValidator ( "spoken_form_plausibility_review" , validators . LLMValidatorTypeSpokenFormPlausibility )
2026-05-12 01:25:52 +00:00
r := New ( fakeFactory { modules : map [ string ] contracts . TranscriptModule {
2026-11-16 14:11:02 +00:00
"m" : fakeModule { key : "m" , policy : proposals . ReplacementPolicyRequireUnique , validators : [ ] validators . Validator { llmValidator } , proposeF : func ( req contracts . ProposalRequest ) ( [ ] proposals . CorrectionProposal , error ) {
2026-11-11 17:05:52 +00:00
return [ ] proposals . CorrectionProposal { { TargetSegmentID : 1 , OriginalText : "gestures" , CorrectedText : "Jesters" } } , nil
2026-05-12 01:25:52 +00:00
} } ,
} } )
2026-09-24 02:18:31 +00:00
cfg := runnerTestConfig ( )
2026-05-12 01:25:52 +00:00
_ , err := r . Run ( context . Background ( ) , RunInput {
2026-11-10 18:13:12 +00:00
Config : & cfg ,
Transcript : & schema . Transcript { Segments : [ ] schema . Segment { { ID : 1 , Text : "There were gestures at the temple." } } } ,
ModuleSpecs : [ ] contracts . ModuleRunSpec { { ModuleKey : "m" , InstanceName : "m" , ValidationClient : client , Settings : config . Stage { Validation : cfg . Validation } } } ,
2026-05-12 01:25:52 +00:00
} )
2026-05-23 10:07:06 -05:00
if err != nil {
t . Fatalf ( "expected duplicate decision downgrade, got %v" , err )
2026-05-12 01:25:52 +00:00
}
}
2026-09-24 02:18:31 +00:00
type lenEstimator struct { }
func ( lenEstimator ) EstimateTokens ( text string ) int { return len ( text ) }
func TestRunnerLLMValidatorBatching ( t * testing . T ) {
2026-05-12 01:25:52 +00:00
client := & fakeStructuredClient { responses : [ ] validators . LLMValidationResponse {
2026-11-16 14:11:02 +00:00
{ Validations : [ ] validators . LLMValidationDecision { { CorrectionIndex : "1" , Approved : true , Reason : "ok" } } } ,
{ Validations : [ ] validators . LLMValidationDecision { { CorrectionIndex : "1" , Approved : true , Reason : "ok" } } } ,
2026-05-12 01:25:52 +00:00
} }
2026-09-24 02:18:31 +00:00
llmValidator , _ := validators . NewLLMBackedValidator ( "spoken_form_plausibility_review" , validators . LLMValidatorTypeSpokenFormPlausibility )
2026-05-12 01:25:52 +00:00
llmValidator . SetTokenEstimator ( lenEstimator { } )
2026-09-24 02:18:31 +00:00
cfg := runnerTestConfig ( )
cfg . Validation . MaxPromptTokens = 260
2026-05-12 01:25:52 +00:00
r := New ( fakeFactory { modules : map [ string ] contracts . TranscriptModule {
2026-11-16 14:11:02 +00:00
"m" : fakeModule { key : "m" , policy : proposals . ReplacementPolicyReplaceAll , validators : [ ] validators . Validator { llmValidator } , proposeF : func ( req contracts . ProposalRequest ) ( [ ] proposals . CorrectionProposal , error ) {
2026-05-12 01:25:52 +00:00
return [ ] proposals . CorrectionProposal {
2026-11-11 17:05:52 +00:00
{ TargetSegmentID : 1 , OriginalText : "alpha" , CorrectedText : strings . Repeat ( "B" , 40 ) } ,
{ TargetSegmentID : 1 , OriginalText : "gamma" , CorrectedText : strings . Repeat ( "D" , 40 ) } ,
2026-05-12 01:25:52 +00:00
} , nil
} } ,
} } )
out , err := r . Run ( context . Background ( ) , RunInput {
2026-11-10 18:13:12 +00:00
Config : & cfg ,
Transcript : & schema . Transcript { Segments : [ ] schema . Segment { { ID : 1 , Text : "alpha gamma" } } } ,
ModuleSpecs : [ ] contracts . ModuleRunSpec { { ModuleKey : "m" , InstanceName : "m" , ValidationClient : client , Settings : config . Stage { Validation : cfg . Validation } } } ,
2026-05-12 01:25:52 +00:00
} )
if err != nil {
t . Fatalf ( "unexpected run error: %v" , err )
}
2026-09-24 02:18:31 +00:00
2026-05-12 01:25:52 +00:00
if client . calls < 2 {
t . Fatalf ( "expected multiple llm calls for batching, got %d" , client . calls )
}
if len ( out . ModuleResults [ 0 ] . AppliedChanges ) != 2 {
t . Fatalf ( "expected both approved proposals applied" )
}
}
2026-11-16 03:37:29 +00:00
func TestRunnerLLMValidatorDiagnosticsWritten ( t * testing . T ) {
2026-11-16 14:11:02 +00:00
client := & fakeStructuredClient { responses : [ ] validators . LLMValidationResponse { { Validations : [ ] validators . LLMValidationDecision { { CorrectionIndex : "1" , Approved : true , Reason : "supported by context" } } } } }
2026-09-24 02:18:31 +00:00
llmValidator , _ := validators . NewLLMBackedValidator ( "spoken_form_plausibility_review" , validators . LLMValidatorTypeSpokenFormPlausibility )
2026-11-10 18:13:12 +00:00
cfg := runnerTestConfig ( )
2026-05-12 01:25:52 +00:00
diagDir := t . TempDir ( )
r := New ( fakeFactory { modules : map [ string ] contracts . TranscriptModule {
2026-11-16 14:11:02 +00:00
"m" : fakeModule { key : "m" , policy : proposals . ReplacementPolicyRequireUnique , validators : [ ] validators . Validator { llmValidator } , proposeF : func ( req contracts . ProposalRequest ) ( [ ] proposals . CorrectionProposal , error ) {
2026-11-11 17:05:52 +00:00
return [ ] proposals . CorrectionProposal { { TargetSegmentID : 1 , OriginalText : "gestures" , CorrectedText : "Jesters" } } , nil
2026-05-12 01:25:52 +00:00
} } ,
} } )
out , err := r . Run ( context . Background ( ) , RunInput {
2026-11-16 03:43:29 +00:00
Config : & cfg ,
Transcript : & schema . Transcript { Segments : [ ] schema . Segment { { ID : 1 , Text : "There were gestures at the temple." } } } ,
ModuleSpecs : [ ] contracts . ModuleRunSpec { { ModuleKey : "m" , InstanceName : "m" , ValidationClient : client , Settings : config . Stage { Validation : cfg . Validation } } } ,
Interactions : llm . NewDiagnosticsWriter ( diagDir ) ,
2026-05-12 01:25:52 +00:00
} )
if err != nil {
t . Fatalf ( "Run error: %v" , err )
}
if len ( out . ModuleResults [ 0 ] . ValidatorDecisions ) == 0 || out . ModuleResults [ 0 ] . ValidatorDecisions [ 0 ] . DiagnosticArtifactPath == "" {
t . Fatalf ( "expected diagnostic artifact path on decision" )
}
2026-05-23 18:11:36 +00:00
matches , globErr := filepath . Glob ( filepath . Join ( diagDir , "m" , "*.json" ) )
if globErr != nil {
t . Fatalf ( "glob diagnostics: %v" , globErr )
}
if len ( matches ) == 0 {
t . Fatalf ( "expected diagnostics JSON artifacts under %s" , filepath . Join ( diagDir , "m" ) )
}
for _ , path := range matches {
raw , readErr := os . ReadFile ( path )
if readErr != nil {
t . Fatalf ( "read diagnostic %q: %v" , path , readErr )
}
2026-11-16 03:37:29 +00:00
if ! json . Valid ( raw ) {
t . Fatalf ( "invalid diagnostic %q: %s" , path , raw )
2026-05-23 18:11:36 +00:00
}
}
2026-05-12 01:25:52 +00:00
raw , readErr := os . ReadFile ( out . ModuleResults [ 0 ] . ValidatorDecisions [ 0 ] . DiagnosticArtifactPath )
if readErr != nil {
2026-05-23 18:11:36 +00:00
t . Fatalf ( "read decision diagnostic: %v" , readErr )
2026-05-12 01:25:52 +00:00
}
2026-11-16 03:37:29 +00:00
if ! strings . Contains ( string ( raw ) , "supported by context" ) {
t . Fatalf ( "missing decision evidence: %s" , raw )
2026-05-12 01:25:52 +00:00
}
}
2026-05-12 02:25:33 +00:00
type proposalGenerationModule struct {
key string
policy proposals . ReplacementPolicy
}
func ( m proposalGenerationModule ) Key ( ) string { return m . key }
func ( m proposalGenerationModule ) ReplacementPolicy ( ) proposals . ReplacementPolicy { return m . policy }
2026-11-16 14:11:02 +00:00
func ( m proposalGenerationModule ) Validators ( ) [ ] validators . Validator { return nil }
2026-05-23 10:07:06 -05:00
func ( m proposalGenerationModule ) Propose ( ctx context . Context , req contracts . ProposalRequest ) ( contracts . ProposalResult , error ) {
2026-05-12 02:25:33 +00:00
result , err := proposal_generation . GenerateCandidates ( ctx , proposal_generation . Request {
ModuleKey : req . RunSpec . ModuleKey ,
ModuleInstance : req . RunSpec . InstanceName ,
ReplacementPolicy : req . RunSpec . ReplacementPolicy ,
WorkingTranscript : req . WorkingTranscript ,
2026-09-24 02:18:31 +00:00
PromptID : "modules.grammar.proposal" ,
Variables : map [ string ] string { "TranscriptDescriptionBlock" : "" , "GlossaryJSON" : "[]" , "SectionJSON" : "[]" } ,
LLMClient : req . LLMClient ,
2026-11-16 03:43:29 +00:00
Interactions : req . Interactions ,
2026-05-12 02:25:33 +00:00
} )
if err != nil {
2026-05-23 10:07:06 -05:00
return contracts . ProposalResult { } , err
2026-05-12 02:25:33 +00:00
}
2026-09-24 17:23:59 +00:00
return contracts . ProposalResult { Proposals : result . Corrections , Operations : result . Operations } , nil
2026-05-12 02:25:33 +00:00
}
type fakeProposalStructuredClient struct {
responses [ ] proposal_generation . StructuredCorrectionSet
}
func ( f * fakeProposalStructuredClient ) CompleteStructured ( ctx context . Context , req contracts . StructuredCompletionRequest , out any ) ( contracts . StructuredCompletionResponse , error ) {
_ = ctx
_ = req
if len ( f . responses ) == 0 {
return contracts . StructuredCompletionResponse { } , errors . New ( "unexpected call" )
}
target , ok := out . ( * proposal_generation . StructuredCorrectionSet )
if ! ok {
return contracts . StructuredCompletionResponse { } , errors . New ( "unexpected output type" )
}
* target = f . responses [ 0 ]
f . responses = f . responses [ 1 : ]
return contracts . StructuredCompletionResponse { } , nil
}
func TestRunnerProposalGenerationHelperFlowsThroughPipeline ( t * testing . T ) {
client := & fakeProposalStructuredClient {
responses : [ ] proposal_generation . StructuredCorrectionSet {
{
Corrections : [ ] proposal_generation . StructuredCorrectionProposal {
2026-11-11 17:05:52 +00:00
{ SegmentReference : 1 , OriginalText : "teh" , CorrectedText : "the" } ,
2026-05-12 02:25:33 +00:00
} ,
} ,
} ,
}
2026-09-24 02:18:31 +00:00
cfg := runnerTestConfig ( )
2026-05-12 02:25:33 +00:00
diagDir := t . TempDir ( )
r := New ( fakeFactory { modules : map [ string ] contracts . TranscriptModule {
"m" : proposalGenerationModule { key : "m" , policy : proposals . ReplacementPolicyRequireUnique } ,
} } )
out , err := r . Run ( context . Background ( ) , RunInput {
2026-11-16 03:43:29 +00:00
Config : & cfg ,
Transcript : & schema . Transcript { Segments : [ ] schema . Segment { { ID : 1 , Text : "teh cat" } } } ,
ModuleSpecs : [ ] contracts . ModuleRunSpec { { ModuleKey : "m" , InstanceName : "m" , ProposalClient : client , Settings : config . Stage { } } } ,
Interactions : llm . NewDiagnosticsWriter ( diagDir ) ,
2026-05-12 02:25:33 +00:00
} )
if err != nil {
t . Fatalf ( "unexpected run error: %v" , err )
}
if out . FinalTranscript . Segments [ 0 ] . Text != "the cat" {
t . Fatalf ( "expected proposal-generated correction to apply, got %q" , out . FinalTranscript . Segments [ 0 ] . Text )
}
if len ( out . ModuleResults ) != 1 || len ( out . ModuleResults [ 0 ] . AppliedChanges ) != 1 {
t . Fatalf ( "expected one applied change, got %+v" , out . ModuleResults )
}
matches , globErr := filepath . Glob ( filepath . Join ( diagDir , "m" , "*proposal-generation*response-payload.json" ) )
if globErr != nil {
t . Fatalf ( "glob diagnostics: %v" , globErr )
}
if len ( matches ) == 0 {
t . Fatalf ( "expected proposal-generation diagnostics artifacts in %s" , filepath . Join ( diagDir , "m" ) )
}
}
2026-05-12 02:57:06 +00:00
2026-09-24 21:48:18 +00:00
func TestFatalPartialDecisionsRejectUnknownAndDuplicateIdentity ( t * testing . T ) {
origin := errors . New ( "original fatal failure" )
candidate := proposals . EnrichedCorrectionProposal { ProposalMetadata : proposals . ProposalMetadata { ProposalIndex : 7 } }
for _ , decisions := range [ ] [ ] validators . Decision {
{ { ProposalIndex : 8 , Approved : true } } ,
{ { ProposalIndex : 7 , Approved : true } , { ProposalIndex : 7 , Approved : true } } ,
} {
2026-11-16 14:11:02 +00:00
v := fakeValidator { name : "broken" , validateF : func ( validators . Request ) ( validators . Result , error ) {
2026-09-24 21:48:18 +00:00
return validators . Result { Decisions : decisions } , origin
} }
2026-11-16 14:11:02 +00:00
result , err := validateSectionCandidates ( context . Background ( ) , validateSectionCandidatesInput { Validators : [ ] validators . Validator { v } , SectionEnriched : [ ] proposals . EnrichedCorrectionProposal { candidate } } )
2026-09-24 21:48:18 +00:00
if ! errors . Is ( err , origin ) || len ( result . decisions ) != 0 || len ( result . approved ) != 0 {
t . Fatalf ( "invalid partial evidence accepted or origin lost: %+v %v" , result , err )
}
}
}