2026-11-21 23:48:51 +00:00
package pipeline
import (
"context"
"errors"
"fmt"
"reflect"
"sync"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"go.maximumdirect.net/promptkit"
)
type runnerProbe struct {
mu sync . Mutex
calls map [ string ] int
eligible map [ string ] [ ] bool
}
func newRunnerProbe ( ) * runnerProbe {
return & runnerProbe { calls : map [ string ] int { } , eligible : map [ string ] [ ] bool { } }
}
func ( p * runnerProbe ) record ( key string , eligible bool ) {
p . mu . Lock ( )
defer p . mu . Unlock ( )
p . calls [ key ] ++
p . eligible [ key ] = append ( p . eligible [ key ] , eligible )
}
type runnerNotesProducer struct {
name string
probe * runnerProbe
engine * promptkit . Engine
native bool
references bool
failure error
cancel context . CancelFunc
}
func ( p * runnerNotesProducer ) Key ( ) string { return p . name }
func ( p * runnerNotesProducer ) ReferenceSlots ( ) [ ] contracts . ReferenceSlot {
if ! p . references {
return nil
}
return [ ] contracts . ReferenceSlot { { Name : "context" , AcceptedArtifactKinds : [ ] contracts . ArtifactKind { "test/notes" } , AcceptedMediaTypes : [ ] string { "application/json" } } }
}
func ( p * runnerNotesProducer ) fallback ( ctx context . Context , role string ) ( codecNotes , error ) {
p . probe . record ( p . name + "/" + role + "/fallback" , false )
return codecNotes { Items : [ ] string { } } , ctx . Err ( )
}
func ( p * runnerNotesProducer ) Extract ( ctx context . Context , request contracts . TypedExtractionRequest [ codecNotes ] ) ( contracts . TypedExtractionResult [ codecNotes ] , error ) {
p . probe . record ( p . name + "/extract" , request . Operation . ReuseEligible )
if p . cancel != nil {
p . cancel ( )
}
if p . failure != nil {
return contracts . TypedExtractionResult [ codecNotes ] { } , p . failure
}
if p . native {
initial := nativeProducerInitial ( )
initial . ProfileID , initial . SessionID = request . LLMProfile , request . SessionID
value , outcome , err := RunProducer ( ctx , p . engine , request . Operation , initial ,
func ( ctx context . Context , result * promptkit . RunResult ) ( * codecNotes , [ ] string , error ) {
number , _ , err := interpretProducerNumber ( ctx , result )
if err != nil {
return nil , nil , err
}
return & codecNotes { Items : [ ] string { fmt . Sprint ( * number ) } } , nil , nil
} , producerCorrection , func ( ctx context . Context , _ * codecNotes ) ( codecNotes , error ) { return p . fallback ( ctx , "extract" ) } )
return contracts . TypedExtractionResult [ codecNotes ] { Value : value , Outcome : outcome } , err
}
return contracts . TypedExtractionResult [ codecNotes ] { Value : codecNotes { Items : [ ] string { p . name } } , Fallback : func ( ctx context . Context ) ( codecNotes , error ) { return p . fallback ( ctx , "extract" ) } } , nil
}
func ( p * runnerNotesProducer ) Merge ( ctx context . Context , request contracts . TypedMergeRequest [ codecNotes ] ) ( contracts . TypedMergeResult [ codecNotes ] , error ) {
p . probe . record ( p . name + "/merge" , request . Operation . ReuseEligible )
if p . references {
if slot , ok := request . References . Slots [ "context" ] ; ok && len ( slot . Items ) > 0 {
p . probe . record ( p . name + "/reference" , request . Operation . ReuseEligible )
}
}
value := codecNotes { Items : [ ] string { } }
for _ , output := range request . ExtractOutputs {
value . Items = append ( value . Items , output . Value . Items ... )
}
return contracts . TypedMergeResult [ codecNotes ] { Value : value , Fallback : func ( ctx context . Context ) ( codecNotes , error ) { return p . fallback ( ctx , "merge" ) } } , nil
}
func ( p * runnerNotesProducer ) Normalize ( ctx context . Context , request contracts . TypedNormalizeRequest [ codecNotes ] ) ( contracts . TypedNormalizeResult [ codecNotes ] , error ) {
p . probe . record ( p . name + "/normalize" , request . Operation . ReuseEligible )
return contracts . TypedNormalizeResult [ codecNotes ] { Value : request . MergeOutput . Value , Fallback : func ( ctx context . Context ) ( codecNotes , error ) { return p . fallback ( ctx , "normalize" ) } } , nil
}
func registerRunnerNotes ( t * testing . T , catalog ModuleCatalog , producer runnerNotesProducer ) ArtifactLaneProfile {
t . Helper ( )
class := contracts . ExecutionClassDeterministic
if producer . native {
class = contracts . ExecutionClassLLMBacked
}
spec := func ( stage ModuleStage ) ModuleSpec {
return ModuleSpec { Key : producer . name , Stage : stage , ExecutionClass : contracts . ExecutionClassDeterministic , ArtifactKind : "test/notes" , ReferenceSlots : producer . ReferenceSlots ( ) }
}
extractSpec := spec ( StageExtract )
extractSpec . ExecutionClass = class
if err := RegisterExtractorBuilder ( catalog . Extractors , extractSpec , rejectUnconfiguredOptions , func ( request BuildRequest ) ( contracts . Extractor [ codecNotes ] , error ) {
copy := producer
copy . engine = request . Dependencies . Engine
return & copy , nil
} ) ; err != nil {
t . Fatal ( err )
}
if err := RegisterMerger ( catalog . Mergers , spec ( StageMerge ) , func ( ) ( contracts . Merger [ codecNotes ] , error ) { copy := producer ; return & copy , nil } ) ; err != nil {
t . Fatal ( err )
}
if err := RegisterNormalizer ( catalog . Normalizers , spec ( StageNormalize ) , func ( ) ( contracts . Normalizer [ codecNotes ] , error ) { copy := producer ; return & copy , nil } ) ; err != nil {
t . Fatal ( err )
}
return ArtifactLaneProfile { Extract : Binding ( producer . name ) , Merge : Binding ( producer . name ) , Normalize : Binding ( producer . name ) }
}
func prepareRunnerProfile ( t * testing . T , catalog ModuleCatalog , profile PipelineProfile , engine * promptkit . Engine ) * PreparedPipeline {
t . Helper ( )
resolved , err := ResolvePipeline ( profile , ResolveOptions { } , catalog )
if err != nil {
t . Fatal ( err )
}
resolved , _ , err = MaterializeReferences ( resolved , catalog , ReferenceMaterializationOptions { } )
if err != nil {
t . Fatal ( err )
}
prepared , err := Prepare ( resolved , registriesFromModuleCatalog ( catalog ) , ModuleDependencies { Engine : engine } )
if err != nil {
t . Fatal ( err )
}
return prepared
}
type namedRunnerValidator struct {
name string
notesAssessmentValidator
}
func ( v namedRunnerValidator ) Name ( ) string { return v . name }
func registerRunnerRejection ( t * testing . T , catalog ModuleCatalog , probe * runnerProbe ) ValidatorOverride {
t . Helper ( )
if err := RegisterTypedValidator ( catalog . Validators , "test/notes" , ValidatorSpec { Key : "reject" , ExecutionClass : contracts . ExecutionClassDeterministic } , func ( ) ( contracts . TypedValidator [ codecNotes ] , error ) {
return namedRunnerValidator { "reject" , notesAssessmentValidator ( func ( _ context . Context , request contracts . TypedValidationRequest [ codecNotes ] ) ( contracts . ValidationResult , error ) {
probe . record ( request . ModuleKey + "/" + request . Stage + "/validator" , false )
return contracts . ValidationResult { ReasonCode : "domain_rejection" , CorrectionGuidance : "Return a supported value." } , nil
} ) } , nil
} ) ; err != nil {
t . Fatal ( err )
}
return ValidatorOverride { Set : true , Validators : [ ] ModuleBinding { Binding ( "reject" ) } }
}
func TestRunnerPropagatesLaneFallbackWithoutCountingDependents ( t * testing . T ) {
for _ , stage := range [ ] ModuleStage { StageExtract , StageMerge , StageNormalize } {
t . Run ( string ( stage ) , func ( t * testing . T ) {
probe := newRunnerProbe ( )
catalog := typedResolutionCatalog ( t , completeTypedCatalogOptions ( ) )
affected := registerRunnerNotes ( t , catalog , runnerNotesProducer { name : "affected" , probe : probe } )
unrelated := registerRunnerNotes ( t , catalog , runnerNotesProducer { name : "unrelated" , probe : probe } )
reject := registerRunnerRejection ( t , catalog , probe )
permissive := contracts . ModePermissive
switch stage {
case StageExtract :
affected . Extract . Validators , affected . Extract . Mode = reject , & permissive
case StageMerge :
affected . Merge . Validators , affected . Merge . Mode = reject , & permissive
case StageNormalize :
affected . Normalize . Validators , affected . Normalize . Mode = reject , & permissive
}
profile := typedResolutionProfile ( )
profile . Artifacts = map [ string ] ArtifactLaneProfile { "affected" : affected , "unrelated" : unrelated }
2026-11-22 01:45:31 +00:00
output , err := New ( ) . Run ( context . Background ( ) , RunInput { Prepared : prepareRunnerProfile ( t , catalog , profile , nil ) , RawInput : [ ] byte ( "input" ) , Workers : 1 } )
2026-11-21 23:48:51 +00:00
if err != nil {
t . Fatal ( err )
}
2026-11-22 02:34:40 +00:00
if output . FallbackCount != 1 || len ( output . normalizeOutputs ) != 2 {
t . Fatalf ( "fallback count=%d, outputs=%d" , output . FallbackCount , len ( output . normalizeOutputs ) )
2026-11-21 23:48:51 +00:00
}
after := false
for _ , role := range [ ] ModuleStage { StageExtract , StageMerge , StageNormalize } {
for _ , name := range [ ] string { "affected" , "unrelated" } {
key := name + "/" + string ( role )
want := name != "affected" || ! after
if probe . calls [ key ] != 1 || ! reflect . DeepEqual ( probe . eligible [ key ] , [ ] bool { want } ) {
t . Fatalf ( "%s calls=%d eligibility=%#v, want %t" , key , probe . calls [ key ] , probe . eligible [ key ] , want )
}
}
if role == stage {
after = true
}
}
if probe . calls [ "affected/" + string ( stage ) + "/validator" ] != 1 || probe . calls [ "affected/" + string ( stage ) + "/fallback" ] != 1 {
t . Fatalf ( "deterministic work was retried or fallback revalidated: %#v" , probe . calls )
}
lanes := output . normalizeReuseEligibility
for key , eligible := range lanes {
if eligible != ( key . laneID == "unrelated" ) {
t . Fatalf ( "normalized eligibility = %#v" , lanes )
}
}
} )
}
}
func TestRunnerAssessesNativeRevisionsWithoutOuterRevalidation ( t * testing . T ) {
probe := newRunnerProbe ( )
providerCalls := 0
engine := nativeProducerEngine ( t , producerClientFunc ( func ( _ context . Context , request promptkit . GenerateRequest ) ( * promptkit . GenerateResponse , error ) {
if request . Target . Model != "runner-model" || request . Prompt . SessionID != "runner-session" {
t . Fatalf ( "run-wide override missing: %#v" , request )
}
value := providerCalls
providerCalls ++
return & promptkit . GenerateResponse { Content : fmt . Sprintf ( ` { "value":%d} ` , value ) } , nil
} ) )
catalog := typedResolutionCatalog ( t , completeTypedCatalogOptions ( ) )
lane := registerRunnerNotes ( t , catalog , runnerNotesProducer { name : "native" , probe : probe , native : true } )
for _ , name := range [ ] string { "first" , "second" } {
if err := RegisterTypedValidator ( catalog . Validators , "test/notes" , ValidatorSpec { Key : name , ExecutionClass : contracts . ExecutionClassDeterministic } , func ( ) ( contracts . TypedValidator [ codecNotes ] , error ) {
return namedRunnerValidator { name , notesAssessmentValidator ( func ( _ context . Context , request contracts . TypedValidationRequest [ codecNotes ] ) ( contracts . ValidationResult , error ) {
probe . record ( name + "/" + request . Value . Items [ 0 ] , true )
if name == "second" && request . Value . Items [ 0 ] == "0" {
return contracts . ValidationResult { ReasonCode : "not_one" , CorrectionGuidance : "Return value one." } , nil
}
return contracts . ValidationResult { Approved : true } , nil
} ) } , nil
} ) ; err != nil {
t . Fatal ( err )
}
}
lane . Extract . Validators = ValidatorOverride { Set : true , Validators : [ ] ModuleBinding { Binding ( "first" ) , Binding ( "second" ) } }
profile := typedResolutionProfile ( )
profile . Artifacts = map [ string ] ArtifactLaneProfile { "native" : lane }
profile . LLMProfile = "chosen"
turns := 1
profile . MaxAdditionalTurns = & turns
output , err := New ( ) . Run ( context . Background ( ) , RunInput {
Prepared : prepareRunnerProfile ( t , catalog , profile , engine ) , RawInput : [ ] byte ( "input" ) , SessionID : "runner-session" ,
Execution : contracts . ExecutionOptions { Execution : & promptkit . ExecutionTargetOverride { Model : "runner-model" } } ,
} )
if err != nil {
t . Fatal ( err )
}
2026-11-22 02:34:40 +00:00
if providerCalls != 2 || output . FallbackCount != 0 || len ( output . normalizeOutputs ) != 1 {
2026-11-21 23:48:51 +00:00
t . Fatalf ( "native calls=%d, output=%#v" , providerCalls , output )
}
for _ , key := range [ ] string { "first/0" , "second/0" , "first/1" , "second/1" , "native/extract" , "native/merge" , "native/normalize" } {
if probe . calls [ key ] != 1 {
t . Fatalf ( "%s called %d times" , key , probe . calls [ key ] )
}
}
}
func TestRunnerGeneratedFallbackReferencesTaintOnlyActualConsumers ( t * testing . T ) {
probe := newRunnerProbe ( )
catalog := typedResolutionCatalog ( t , completeTypedCatalogOptions ( ) )
producer := registerRunnerNotes ( t , catalog , runnerNotesProducer { name : "producer" , probe : probe } )
earlier := registerRunnerNotes ( t , catalog , runnerNotesProducer { name : "earlier" , probe : probe } )
consumer := registerRunnerNotes ( t , catalog , runnerNotesProducer { name : "consumer" , probe : probe , references : true } )
unrelated := registerRunnerNotes ( t , catalog , runnerNotesProducer { name : "later" , probe : probe } )
producer . Normalize . Validators = registerRunnerRejection ( t , catalog , probe )
permissive := contracts . ModePermissive
producer . Normalize . Mode = & permissive
consumer . Merge . References = map [ string ] ReferenceSource { "context" : GeneratedReference ( "build" , "producer" ) }
profile := typedResolutionProfile ( )
profile . Artifacts = nil
profile . Steps = [ ] PipelineStepProfile {
{ ID : "build" , Artifacts : map [ string ] ArtifactLaneProfile { "producer" : producer , "earlier" : earlier } } ,
{ ID : "consume" , Artifacts : map [ string ] ArtifactLaneProfile { "consumer" : consumer , "later" : unrelated } } ,
}
2026-11-22 01:45:31 +00:00
output , err := New ( ) . Run ( context . Background ( ) , RunInput { Prepared : prepareRunnerProfile ( t , catalog , profile , nil ) , RawInput : [ ] byte ( "input" ) , Workers : 1 } )
2026-11-21 23:48:51 +00:00
if err != nil {
t . Fatal ( err )
}
2026-11-22 02:34:40 +00:00
if output . FallbackCount != 1 || len ( output . normalizeOutputs ) != 4 || probe . calls [ "consumer/reference" ] != 1 {
t . Fatalf ( "handoff failed: count=%d outputs=%d observations=%#v" , output . FallbackCount , len ( output . normalizeOutputs ) , probe . calls )
2026-11-21 23:48:51 +00:00
}
for _ , key := range [ ] string { "earlier/extract" , "earlier/merge" , "earlier/normalize" , "later/extract" , "later/merge" , "later/normalize" , "consumer/extract" } {
if ! reflect . DeepEqual ( probe . eligible [ key ] , [ ] bool { true } ) {
t . Fatalf ( "unaffected work %s tainted: %#v" , key , probe . eligible [ key ] )
}
}
for _ , key := range [ ] string { "consumer/merge" , "consumer/normalize" } {
if ! reflect . DeepEqual ( probe . eligible [ key ] , [ ] bool { false } ) {
t . Fatalf ( "dependent %s remained reusable: %#v" , key , probe . eligible [ key ] )
}
}
if output . normalizeReuseEligibility [ generatedOutputKeyFor ( "consume" , "consumer" ) ] || ! output . normalizeReuseEligibility [ generatedOutputKeyFor ( "consume" , "later" ) ] {
t . Fatalf ( "generated lineage = %#v" , output . normalizeReuseEligibility )
}
}
type runnerFallbackChunker struct { }
func ( runnerFallbackChunker ) Key ( ) string { return "fallback-chunk" }
func ( runnerFallbackChunker ) ReferenceSlots ( ) [ ] contracts . ReferenceSlot { return nil }
func ( runnerFallbackChunker ) Plan ( _ context . Context , request contracts . ChunkRequest ) ( contracts . ChunkPlanResult , error ) {
return contracts . ChunkPlanResult { Plan : typedTestPlan ( request . Source ) , Outcome : contracts . ProducerOutcome { Disposition : contracts . ProducerFallback , FallbackReason : "selected test fallback" , ReuseEligible : false } } , nil
}
func TestRunnerChunkFallbackTaintsEveryDerivedOperation ( t * testing . T ) {
probe := newRunnerProbe ( )
catalog := typedResolutionCatalog ( t , completeTypedCatalogOptions ( ) )
if err := catalog . Chunkers . RegisterWithSpec ( ModuleSpec { Key : "fallback-chunk" , Stage : StageChunk , ExecutionClass : contracts . ExecutionClassLLMBacked } , func ( ) ( contracts . Chunker , error ) { return runnerFallbackChunker { } , nil } ) ; err != nil {
t . Fatal ( err )
}
lane := registerRunnerNotes ( t , catalog , runnerNotesProducer { name : "notes" , probe : probe } )
profile := typedResolutionProfile ( )
profile . Chunk = Binding ( "fallback-chunk" )
permissive := contracts . ModePermissive
profile . Chunk . Mode = & permissive
profile . Artifacts = map [ string ] ArtifactLaneProfile { "notes" : lane }
output , err := New ( ) . Run ( context . Background ( ) , RunInput { Prepared : prepareRunnerProfile ( t , catalog , profile , nil ) , RawInput : [ ] byte ( "input" ) } )
if err != nil {
t . Fatal ( err )
}
2026-11-22 02:34:40 +00:00
if output . FallbackCount != 1 || len ( output . normalizeOutputs ) != 1 {
2026-11-21 23:48:51 +00:00
t . Fatalf ( "chunk fallback = %#v" , output )
}
for _ , role := range [ ] string { "extract" , "merge" , "normalize" } {
if ! reflect . DeepEqual ( probe . eligible [ "notes/" + role ] , [ ] bool { false } ) {
t . Fatalf ( "chunk fallback did not taint %s: %#v" , role , probe . eligible )
}
}
}
func TestRunnerDoesNotRetryOrFallbackDeterministicOperationErrors ( t * testing . T ) {
probe := newRunnerProbe ( )
failure := errors . New ( "deterministic operation failed" )
catalog := typedResolutionCatalog ( t , completeTypedCatalogOptions ( ) )
lane := registerRunnerNotes ( t , catalog , runnerNotesProducer { name : "notes" , probe : probe , failure : failure } )
permissive := contracts . ModePermissive
profile := typedResolutionProfile ( )
profile . Mode = & permissive
profile . Artifacts = map [ string ] ArtifactLaneProfile { "notes" : lane }
_ , err := New ( ) . Run ( context . Background ( ) , RunInput { Prepared : prepareRunnerProfile ( t , catalog , profile , nil ) , RawInput : [ ] byte ( "input" ) } )
if ! errors . Is ( err , failure ) || probe . calls [ "notes/extract" ] != 1 || probe . calls [ "notes/extract/fallback" ] != 0 || probe . calls [ "notes/merge" ] != 0 {
t . Fatalf ( "operation failure = %v, calls %#v" , err , probe . calls )
}
}
func TestRunnerDoesNotAcceptModuleFallbackUnderStrictControls ( t * testing . T ) {
probe := newRunnerProbe ( )
catalog := typedResolutionCatalog ( t , completeTypedCatalogOptions ( ) )
if err := catalog . Chunkers . RegisterWithSpec ( ModuleSpec { Key : "fallback-chunk" , Stage : StageChunk , ExecutionClass : contracts . ExecutionClassLLMBacked } , func ( ) ( contracts . Chunker , error ) { return runnerFallbackChunker { } , nil } ) ; err != nil {
t . Fatal ( err )
}
profile := typedResolutionProfile ( )
profile . Chunk = Binding ( "fallback-chunk" )
profile . Artifacts = map [ string ] ArtifactLaneProfile { "notes" : registerRunnerNotes ( t , catalog , runnerNotesProducer { name : "notes" , probe : probe } ) }
output , err := New ( ) . Run ( context . Background ( ) , RunInput { Prepared : prepareRunnerProfile ( t , catalog , profile , nil ) , RawInput : [ ] byte ( "input" ) } )
2026-11-22 02:34:40 +00:00
if err == nil || output . FallbackCount != 0 || len ( output . normalizeOutputs ) != 0 || probe . calls [ "notes/extract" ] != 0 {
2026-11-21 23:48:51 +00:00
t . Fatalf ( "strict controls accepted module fallback: %#v %v" , output , err )
}
}
func TestRunnerDiscardsDeterministicOutputReturnedAfterCancellation ( t * testing . T ) {
ctx , cancel := context . WithCancel ( context . Background ( ) )
defer cancel ( )
probe := newRunnerProbe ( )
catalog := typedResolutionCatalog ( t , completeTypedCatalogOptions ( ) )
lane := registerRunnerNotes ( t , catalog , runnerNotesProducer { name : "notes" , probe : probe , cancel : cancel } )
profile := typedResolutionProfile ( )
profile . Artifacts = map [ string ] ArtifactLaneProfile { "notes" : lane }
output , err := New ( ) . Run ( ctx , RunInput { Prepared : prepareRunnerProfile ( t , catalog , profile , nil ) , RawInput : [ ] byte ( "input" ) } )
2026-11-22 02:34:40 +00:00
if ! errors . Is ( err , context . Canceled ) || output . FallbackCount != 0 || len ( output . normalizeOutputs ) != 0 || probe . calls [ "notes/merge" ] != 0 || probe . calls [ "notes/extract/fallback" ] != 0 {
2026-11-21 23:48:51 +00:00
t . Fatalf ( "cancellation = %#v %v, calls %#v" , output , err , probe . calls )
}
}
var _ contracts . Chunker = runnerFallbackChunker { }
var _ contracts . Extractor [ codecNotes ] = ( * runnerNotesProducer ) ( nil )
var _ contracts . Merger [ codecNotes ] = ( * runnerNotesProducer ) ( nil )
var _ contracts . Normalizer [ codecNotes ] = ( * runnerNotesProducer ) ( nil )