2026-07-20 19:38:57 +00:00
package cli
import (
"encoding/json"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
2026-07-20 15:14:41 -05:00
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
2026-07-20 19:38:57 +00:00
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
spellcatalog "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/spells/catalog"
)
2026-11-22 05:07:00 +00:00
func TestMaterializedSpellCatalogOrderingPreservesSemanticContents ( t * testing . T ) {
2026-07-20 19:38:57 +00:00
components := productionTestComponents ( t )
2026-07-25 15:48:48 +00:00
configPath := writeProductionSpellCatalogContractConfig ( t )
2026-11-22 05:07:00 +00:00
catalog := catalogFromRegistries ( components . registries )
effective , err := loadMaintainedExample ( t , configPath ) . Resolve ( config . ResolveInput { PipelineID : "dnd-session" , Catalog : catalog } )
2026-07-20 19:38:57 +00:00
if err != nil {
2026-11-22 05:07:00 +00:00
t . Fatal ( err )
2026-07-20 19:38:57 +00:00
}
resolved := effective . ResolvedPipeline
2026-11-22 05:07:00 +00:00
overlayPath := filepath . Join ( t . TempDir ( ) , "catalog.json" )
for _ , target := range [ ] * pipeline . ResolvedReferenceTarget { & resolved . Steps [ 0 ] . ArtifactLanes [ 0 ] . ExtractReferences , & resolved . Steps [ 0 ] . ArtifactLanes [ 0 ] . NormalizeReferences } {
matched := false
for index := range target . Bindings {
if target . Bindings [ index ] . SlotName == spellcatalog . SpellCatalogReferenceSlot {
target . Bindings [ index ] . Source = overlayPath
matched = true
}
2026-07-20 19:38:57 +00:00
}
2026-11-22 05:07:00 +00:00
if ! matched {
t . Fatal ( "missing configured spell catalog binding" )
2026-07-20 21:16:08 +00:00
}
}
2026-11-22 05:07:00 +00:00
materialize := func ( contents string ) pipeline . ResolvedPipeline {
t . Helper ( )
if err := os . WriteFile ( overlayPath , [ ] byte ( contents ) , 0600 ) ; err != nil {
t . Fatal ( err )
}
value , _ , err := pipeline . MaterializeReferences ( resolved , catalog , pipeline . ReferenceMaterializationOptions { ConfigPath : configPath , WorkingDir : filepath . Dir ( configPath ) } )
if err != nil {
t . Fatal ( err )
}
return value
}
first , second := materialize ( reorderedOverlayA ) , materialize ( reorderedOverlayB )
laneA , laneB := first . Steps [ 0 ] . ArtifactLanes [ 0 ] , second . Steps [ 0 ] . ArtifactLanes [ 0 ]
for _ , pair := range [ ] [ 2 ] pipeline . ResolvedReferenceTarget {
{ laneA . ExtractReferences , laneB . ExtractReferences } ,
{ laneA . NormalizeReferences , laneB . NormalizeReferences } ,
} {
itemsA , itemsB := pair [ 0 ] . ReferenceSet . Slots [ spellcatalog . SpellCatalogReferenceSlot ] . Items , pair [ 1 ] . ReferenceSet . Slots [ spellcatalog . SpellCatalogReferenceSlot ] . Items
if len ( itemsA ) != 1 || len ( itemsB ) != 1 {
t . Fatal ( "materialized catalog missing" )
}
if itemsA [ 0 ] . Digest == itemsB [ 0 ] . Digest || itemsA [ 0 ] . Origin != itemsB [ 0 ] . Origin {
t . Fatalf ( "raw catalog provenance=%#v/%#v" , itemsA , itemsB )
}
catalogA , err := spellcatalog . ResolveEffectiveCatalog ( pair [ 0 ] . ReferenceSet )
if err != nil {
t . Fatal ( err )
}
catalogB , err := spellcatalog . ResolveEffectiveCatalog ( pair [ 1 ] . ReferenceSet )
if err != nil {
t . Fatal ( err )
}
if catalogA . Digest ( ) != catalogB . Digest ( ) || ! reflect . DeepEqual ( catalogA . OverlayIDs ( ) , [ ] string { "campaign.a" , "campaign.b" } ) || ! reflect . DeepEqual ( catalogB . OverlayIDs ( ) , catalogA . OverlayIDs ( ) ) || ! reflect . DeepEqual ( catalogA . PromptSpells ( ) , catalogB . PromptSpells ( ) ) {
t . Fatal ( "catalog ordering changed semantic recognition" )
}
2026-07-20 21:16:08 +00:00
}
2026-07-20 19:38:57 +00:00
}
func TestConfiguredSpellCatalogBindingChangesResolvedPipelineIdentity ( t * testing . T ) {
2026-07-25 15:48:48 +00:00
base := productionSpellCatalogContractConfig ( t )
changed := strings . Replace ( base , repositoryPath ( "examples" , "dnd-spell-catalog.json" ) , filepath . Join ( t . TempDir ( ) , "alternate-spell-catalog.json" ) , 1 )
2026-07-20 19:38:57 +00:00
if changed == base {
t . Fatal ( "production configuration did not contain the maintained catalog binding" )
}
root := t . TempDir ( )
firstPath := filepath . Join ( root , "first.yml" )
secondPath := filepath . Join ( root , "second.yml" )
if err := os . WriteFile ( firstPath , [ ] byte ( base ) , 0 o600 ) ; err != nil {
t . Fatal ( err )
}
if err := os . WriteFile ( secondPath , [ ] byte ( changed ) , 0 o600 ) ; err != nil {
t . Fatal ( err )
}
components := productionTestComponents ( t )
2026-11-22 05:07:00 +00:00
first , err := loadMaintainedExample ( t , firstPath ) . Resolve ( config . ResolveInput { PipelineID : "dnd-session" , Catalog : catalogFromRegistries ( components . registries ) } )
2026-07-20 19:38:57 +00:00
if err != nil {
t . Fatalf ( "resolve first configuration: %v" , err )
}
2026-11-22 05:07:00 +00:00
second , err := loadMaintainedExample ( t , secondPath ) . Resolve ( config . ResolveInput { PipelineID : "dnd-session" , Catalog : catalogFromRegistries ( components . registries ) } )
2026-07-20 19:38:57 +00:00
if err != nil {
t . Fatalf ( "resolve changed configuration: %v" , err )
}
if first . ResolvedPipeline . Digest == second . ResolvedPipeline . Digest {
t . Fatalf ( "resolved pipeline digest = %q for different catalog bindings, want change" , first . ResolvedPipeline . Digest )
}
}
2026-11-22 05:07:00 +00:00
func TestProductionOverlayRunAlignsGroundingAndValidation ( t * testing . T ) {
2026-07-20 19:38:57 +00:00
outputRoot := filepath . Join ( t . TempDir ( ) , "output" )
2026-11-22 05:07:00 +00:00
client := & productionFakeLLMClient { spellResponse : productionSpellResponse ( "Aegis of Emberfall" ) }
configPath := writeProductionContractConfig ( t , productionSpellCatalogContractConfig ( t ) + productionProviderConfiguration )
2026-07-20 19:38:57 +00:00
var stdout , stderr strings . Builder
2026-11-22 05:07:00 +00:00
code := RunWithOptions ( [ ] string { "run" , "dnd-session" , "--config" , configPath , "--input-file" , repositoryPath ( "examples" , "seriatim-minimal-transcript.json" ) , "--lane" , "spells" , "--cache" , "bypass" , "--output-dir" , outputRoot , "--format" , "json" } , & stdout , & stderr , productionRunOptions ( t , client ) )
2026-07-20 19:38:57 +00:00
if code != 0 {
2026-11-22 05:07:00 +00:00
t . Fatalf ( "status=%d stdout=%q stderr=%q" , code , stdout . String ( ) , stderr . String ( ) )
2026-07-20 19:38:57 +00:00
}
2026-11-22 05:07:00 +00:00
var report runResult
if err := json . Unmarshal ( [ ] byte ( stdout . String ( ) ) , & report ) ; err != nil {
2026-07-20 19:38:57 +00:00
t . Fatal ( err )
}
2026-11-22 05:07:00 +00:00
if report . ArtifactCount != 1 || report . FallbackCount != 0 || report . OutputDirectory != filepath . Join ( outputRoot , productionRunID ) {
t . Fatalf ( "completion=%#v" , report )
2026-07-20 19:38:57 +00:00
}
2026-11-22 05:07:00 +00:00
requests := client . requestsWithProperty ( "spell_casts" )
2026-07-20 19:38:57 +00:00
if len ( requests ) != 1 {
2026-11-22 05:07:00 +00:00
t . Fatalf ( "native spell requests=%d" , len ( requests ) )
2026-07-20 19:38:57 +00:00
}
2026-11-22 05:07:00 +00:00
var prompt strings . Builder
for _ , message := range requests [ 0 ] . Prompt . Messages {
prompt . WriteString ( message . Content )
2026-07-20 19:38:57 +00:00
}
2026-11-22 05:07:00 +00:00
for _ , fragment := range [ ] string { ` "canonical_name":"Aegis of Emberfall" ` , ` "aliases":["Emberfall Aegis"] ` } {
if ! strings . Contains ( prompt . String ( ) , fragment ) {
t . Fatalf ( "catalog prompt omitted overlay recognition %q" , fragment )
2026-07-20 19:38:57 +00:00
}
}
2026-11-22 05:07:00 +00:00
artifact := readProductionJSON [ dnd . SpellList ] ( t , filepath . Join ( report . OutputDirectory , "lanes" , "spells.json" ) )
if len ( artifact . SpellCasts ) != 1 || artifact . SpellCasts [ 0 ] . Spell != "Aegis of Emberfall" {
t . Fatalf ( "spell artifact=%#v" , artifact )
2026-07-20 19:38:57 +00:00
}
}
const reorderedOverlayA = ` {
"schema_version" : "notarius.dnd.spell-catalog-overlay.v1" ,
"catalogs" : [
{ "id" : "campaign.a" , "ruleset" : "dnd-5e-2014" , "source" : { "title" : "Campaign A" } , "spells" : [ { "name" : "Aegis of Emberfall" , "aliases" : [ "Emberfall Aegis" ] } ] } ,
{ "id" : "campaign.b" , "ruleset" : "dnd-5e-2014" , "source" : { "title" : "Campaign B" } , "spells" : [ { "name" : "Cinder Veil" , "aliases" : [ "Veil of Cinder" , "Cinder Shroud" ] } ] }
]
} `
const reorderedOverlayB = ` { "catalogs":[ { "spells":[ { "aliases":["Cinder Shroud","Veil of Cinder"],"name":"Cinder Veil"}],"source": { "title":"Campaign B"},"ruleset":"dnd-5e-2014","id":"campaign.b"}, { "spells":[ { "aliases":["Emberfall Aegis"],"name":"Aegis of Emberfall"}],"source": { "title":"Campaign A"},"ruleset":"dnd-5e-2014","id":"campaign.a"}],"schema_version":"notarius.dnd.spell-catalog-overlay.v1"} `