2026-07-27 18:17:54 +00:00
package pipeline
import (
"context"
"errors"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/evidencecontext"
2026-11-22 02:34:40 +00:00
"go.maximumdirect.net/promptkit"
2026-07-27 18:17:54 +00:00
)
type capturingEvidenceOutput struct {
requests [ ] contracts . OutputRequest
}
func ( * capturingEvidenceOutput ) Key ( ) string { return "capture/evidence-context" }
func ( output * capturingEvidenceOutput ) Encode ( _ context . Context , request contracts . OutputRequest ) ( contracts . OutputResult , error ) {
request . EvidenceContext = contracts . CloneSerializedArtifactPointer ( request . EvidenceContext )
output . requests = append ( output . requests , request )
return contracts . OutputResult { } , nil
}
func installEvidencePlan ( prepared * PreparedPipeline , window int , selected [ ] string , project func ( codecNotes ) ( [ ] source . SourceRef , error ) ) {
lanes := make ( [ ] preparedEvidenceLane , 0 , len ( selected ) )
for _ , laneID := range selected {
for _ , step := range prepared . Steps {
for _ , lane := range step . lanes {
if lane . resolved . ID != laneID {
continue
}
lanes = append ( lanes , preparedEvidenceLane {
laneID : laneID ,
kind : lane . resolved . ArtifactKind ,
project : func ( value any ) ( [ ] source . SourceRef , error ) {
notes , ok := value . ( codecNotes )
if ! ok {
return nil , errors . New ( "unexpected artifact type" )
}
return project ( notes )
} ,
} )
}
}
}
prepared . evidencePlan = & preparedEvidencePlan { policy : EvidenceContextPolicy { Enabled : true , WindowUnits : window , LaneIDs : append ( [ ] string ( nil ) , selected ... ) } , lanes : lanes }
}
func setNormalizedNotes ( prepared * PreparedPipeline , values map [ string ] codecNotes ) {
for stepIndex := range prepared . Steps {
for laneIndex := range prepared . Steps [ stepIndex ] . lanes {
lane := & prepared . Steps [ stepIndex ] . lanes [ laneIndex ]
value , ok := values [ lane . resolved . ID ]
if ! ok {
continue
}
lane . typed . normalize = func ( context . Context , any , contracts . TypedNormalizeRequest [ any ] ) ( erasedTypedResult , error ) {
return erasedTypedResult { Value : value } , nil
}
}
}
}
func decodeCapturedEvidence ( t * testing . T , output * capturingEvidenceOutput ) evidencecontext . Document {
t . Helper ( )
if len ( output . requests ) != 1 || output . requests [ 0 ] . EvidenceContext == nil {
t . Fatalf ( "output requests = %#v, want one evidence context" , output . requests )
}
value , err := evidencecontext . New ( ) . Decode ( output . requests [ 0 ] . EvidenceContext . Content )
if err != nil {
t . Fatalf ( "Decode(evidence context): %v" , err )
}
return value
}
func TestRunnerBuildsEvidenceContextFromSelectedNormalizedOutputs ( t * testing . T ) {
prepared := preparedOrderedPipeline ( t , 3 , orderedLaneSpec { id : "alpha" , profile : "notes" } , orderedLaneSpec { id : "beta" , profile : "notes" } )
encoder := & capturingEvidenceOutput { }
prepared . output = encoder
setNormalizedNotes ( prepared , map [ string ] codecNotes { "alpha" : { Items : [ ] string { "one" } } , "beta" : { Items : [ ] string { "two" } } } )
installEvidencePlan ( prepared , 1 , [ ] string { "alpha" , "beta" , "inactive" } , func ( notes codecNotes ) ( [ ] source . SourceRef , error ) {
switch notes . Items [ 0 ] {
case "one" :
return [ ] source . SourceRef { { SourceID : "source" , StartUnitID : 1 , EndUnitID : 1 } } , nil
case "two" :
return [ ] source . SourceRef { { SourceID : "source" , StartUnitID : 2 , EndUnitID : 2 } } , nil
default :
return nil , nil
}
} )
2026-11-22 02:34:40 +00:00
if _ , err := New ( ) . Run ( context . Background ( ) , RunInput { Prepared : prepared , RawInput : [ ] byte ( "input" ) } ) ; err != nil {
2026-07-27 18:17:54 +00:00
t . Fatalf ( "Run() error = %v" , err )
}
value := decodeCapturedEvidence ( t , encoder )
2026-08-09 19:42:33 +00:00
if actual := [ ] int { value [ 0 ] . ID , value [ 1 ] . ID , value [ 2 ] . ID } ; ! reflect . DeepEqual ( actual , [ ] int { 1 , 2 , 3 } ) {
2026-07-27 18:17:54 +00:00
t . Fatalf ( "evidence context = %#v, want selected union" , value )
}
2026-11-22 02:34:40 +00:00
2026-07-27 18:17:54 +00:00
}
2026-11-22 02:34:40 +00:00
func TestEvidenceContextOmitsUnselectedAndAbsentLanes ( t * testing . T ) {
prepared := preparedOrderedPipeline ( t , 2 , orderedLaneSpec { id : "present" , profile : "notes" } , orderedLaneSpec { id : "unselected" , profile : "notes" } )
doc := prepared . input . ( * typedTestInput ) . doc
installEvidencePlan ( prepared , 0 , [ ] string { "absent" , "present" } , func ( codecNotes ) ( [ ] source . SourceRef , error ) {
return [ ] source . SourceRef { { SourceID : doc . ID , StartUnitID : 1 , EndUnitID : 1 } } , nil
2026-07-27 18:17:54 +00:00
} )
2026-11-22 02:34:40 +00:00
artifact , err := prepared . artifactCodecs . Encode ( "test/notes" , codecNotes { Items : [ ] string { "accepted" } } )
2026-07-27 18:17:54 +00:00
if err != nil {
2026-11-22 02:34:40 +00:00
t . Fatal ( err )
2026-07-27 18:17:54 +00:00
}
2026-11-22 02:34:40 +00:00
result , _ , err := buildOutputEvidenceContext ( prepared , doc , [ ] contracts . SerializedOutput { { LaneID : "present" , SourceID : doc . ID , Artifact : artifact } , { LaneID : "unselected" , SourceID : doc . ID , Artifact : artifact } } )
if err != nil {
t . Fatal ( err )
2026-07-27 18:17:54 +00:00
}
2026-11-22 02:34:40 +00:00
value , err := evidencecontext . New ( ) . Decode ( result . Content )
if err != nil || len ( value ) != 1 || value [ 0 ] . ID != 1 {
t . Fatalf ( "evidence=%#v %v" , value , err )
2026-07-27 18:17:54 +00:00
}
}
func TestRunnerEvidenceContextFailurePreventsOutputEncoding ( t * testing . T ) {
prepared := preparedConcurrentPipeline ( t , 1 )
encoder := & capturingEvidenceOutput { }
prepared . output = encoder
setNormalizedNotes ( prepared , map [ string ] codecNotes { "notes" : { Items : [ ] string { "invalid" } } } )
installEvidencePlan ( prepared , 0 , [ ] string { "notes" } , func ( codecNotes ) ( [ ] source . SourceRef , error ) {
return [ ] source . SourceRef { { SourceID : "source" , StartUnitID : 99 , EndUnitID : 99 } } , nil
} )
result , err := New ( ) . Run ( context . Background ( ) , RunInput { Prepared : prepared , RawInput : [ ] byte ( "input" ) } )
if err == nil || ! strings . Contains ( err . Error ( ) , "accepted evidence references are invalid" ) {
t . Fatalf ( "Run() error = %v, want evidence context failure" , err )
}
2026-11-22 02:34:40 +00:00
if len ( encoder . requests ) != 0 || result . ArtifactCount ( ) != 0 {
t . Fatalf ( "invalid evidence was published: %#v" , result )
2026-07-27 18:17:54 +00:00
}
}
func TestBuildOutputEvidenceContextRejectsIncompatibleAcceptedOutputs ( t * testing . T ) {
prepared := preparedConcurrentPipeline ( t , 1 )
doc := prepared . input . ( * typedTestInput ) . doc
installEvidencePlan ( prepared , 0 , [ ] string { "notes" } , func ( codecNotes ) ( [ ] source . SourceRef , error ) {
return [ ] source . SourceRef { { SourceID : doc . ID , StartUnitID : 1 , EndUnitID : 1 } } , nil
} )
artifact , err := prepared . artifactCodecs . Encode ( "test/notes" , codecNotes { Items : [ ] string { "valid" } } )
if err != nil {
t . Fatal ( err )
}
valid := contracts . SerializedOutput { LaneID : "notes" , SourceID : doc . ID , Artifact : artifact }
for _ , test := range [ ] struct {
name string
outputs [ ] contracts . SerializedOutput
mutate func ( * contracts . SerializedOutput )
want string
} {
{ name : "duplicate lane" , outputs : [ ] contracts . SerializedOutput { valid , valid } , want : "duplicate lane" } ,
{ name : "foreign source" , outputs : [ ] contracts . SerializedOutput { valid } , mutate : func ( output * contracts . SerializedOutput ) { output . SourceID = "other" } , want : "source is incompatible" } ,
{ name : "wrong artifact kind" , outputs : [ ] contracts . SerializedOutput { valid } , mutate : func ( output * contracts . SerializedOutput ) { output . Artifact . Kind = "test/score" } , want : "artifact kind is incompatible" } ,
{ name : "invalid artifact payload" , outputs : [ ] contracts . SerializedOutput { valid } , mutate : func ( output * contracts . SerializedOutput ) { output . Artifact . Content = [ ] byte ( "not JSON" ) } , want : "cannot be decoded" } ,
} {
t . Run ( test . name , func ( t * testing . T ) {
outputs := append ( [ ] contracts . SerializedOutput ( nil ) , test . outputs ... )
for index := range outputs {
outputs [ index ] = contracts . CloneSerializedOutput ( outputs [ index ] )
}
if test . mutate != nil {
test . mutate ( & outputs [ 0 ] )
}
_ , _ , err := buildOutputEvidenceContext ( prepared , doc , outputs )
if err == nil || ! strings . Contains ( err . Error ( ) , test . want ) {
t . Fatalf ( "buildOutputEvidenceContext() error = %v, want %q" , err , test . want )
}
if strings . Contains ( err . Error ( ) , "not JSON" ) {
t . Fatalf ( "buildOutputEvidenceContext() exposed artifact payload: %v" , err )
}
} )
}
}
func TestEvidenceContextOutputRequestOwnsArtifactBytes ( t * testing . T ) {
artifact := & contracts . SerializedArtifact { Content : [ ] byte ( "original" ) , Metadata : map [ string ] any { "source" : "original" } }
request := contracts . OutputRequest { EvidenceContext : contracts . CloneSerializedArtifactPointer ( artifact ) }
request . EvidenceContext . Content [ 0 ] = 'X'
request . EvidenceContext . Metadata [ "source" ] = "changed"
if string ( artifact . Content ) != "original" || artifact . Metadata [ "source" ] != "original" {
t . Fatalf ( "output request evidence context aliases source artifact: %#v" , artifact )
}
}
2026-11-22 02:34:40 +00:00
func TestRunnerEvidenceContextRebuildsFromCachedNormalizedArtifact ( t * testing . T ) {
2026-07-27 18:17:54 +00:00
prepared := preparedConcurrentPipeline ( t , 1 )
encoder := & capturingEvidenceOutput { }
prepared . output = encoder
doc := prepared . input . ( * typedTestInput ) . doc
2026-11-22 02:34:40 +00:00
calls , assessments := 0 , 0
engine := nativeProducerEngine ( t , producerClientFunc ( func ( context . Context , promptkit . GenerateRequest ) ( * promptkit . GenerateResponse , error ) {
calls ++
return & promptkit . GenerateResponse { Content : ` { "value":1} ` } , nil
} ) )
lane := & prepared . Steps [ 0 ] . lanes [ 0 ]
lane . resolved . NormalizeExecutionClass = contracts . ExecutionClassLLMBacked
lane . typed . normalize = func ( ctx context . Context , _ any , request contracts . TypedNormalizeRequest [ any ] ) ( erasedTypedResult , error ) {
operation := request . Operation
assess := operation . Assess
operation . Assess = func ( ctx context . Context , value any ) ( contracts . CandidateAssessment , error ) {
assessments ++
return assess ( ctx , value )
}
value , outcome , err := RunProducer ( ctx , engine , operation , nativeProducerInitial ( ) , func ( context . Context , * promptkit . RunResult ) ( * any , [ ] string , error ) {
var value any = codecNotes { Items : [ ] string { "stored" } }
return & value , nil , nil
} , nil , nil )
return erasedTypedResult { Value : value , Outcome : outcome } , err
2026-07-27 18:17:54 +00:00
}
installEvidencePlan ( prepared , 0 , [ ] string { "notes" } , func ( notes codecNotes ) ( [ ] source . SourceRef , error ) {
if ! reflect . DeepEqual ( notes . Items , [ ] string { "stored" } ) {
2026-11-22 02:34:40 +00:00
return nil , errors . New ( "cached artifact was not projected" )
2026-07-27 18:17:54 +00:00
}
return [ ] source . SourceRef { { SourceID : doc . ID , StartUnitID : 1 , EndUnitID : 1 } } , nil
} )
2026-11-22 02:34:40 +00:00
_ , store := filesystemProducerCache ( t )
for range 2 {
encoder . requests = nil
if _ , err := New ( ) . Run ( context . Background ( ) , RunInput { Prepared : prepared , RawInput : [ ] byte ( "input" ) , Cache : store , Workers : 1 } ) ; err != nil {
t . Fatal ( err )
}
value := decodeCapturedEvidence ( t , encoder )
if len ( value ) != 1 || value [ 0 ] . ID != 1 {
t . Fatalf ( "evidence=%#v" , value )
}
2026-07-27 18:17:54 +00:00
}
2026-11-22 02:34:40 +00:00
if calls != 1 || assessments != 1 {
t . Fatalf ( "cache reuse calls=%d assessments=%d" , calls , assessments )
2026-07-27 18:17:54 +00:00
}
}
func TestRunnerSkipsEvidenceContextWhenOutputDoesNotOptIn ( t * testing . T ) {
prepared := preparedConcurrentPipeline ( t , 1 )
encoder := & capturingEvidenceOutput { }
prepared . output = encoder
projected := 0
prepared . evidencePlan = nil
if _ , err := New ( ) . Run ( context . Background ( ) , RunInput { Prepared : prepared , RawInput : [ ] byte ( "input" ) } ) ; err != nil {
t . Fatalf ( "Run() error = %v" , err )
}
if projected != 0 || len ( encoder . requests ) != 1 || encoder . requests [ 0 ] . EvidenceContext != nil {
t . Fatalf ( "projected = %d requests = %#v, want no evidence work" , projected , encoder . requests )
}
}