Files
notarius/internal/cli/dnd_enemy_events_contract_test.go

461 lines
20 KiB
Go
Raw Normal View History

2026-08-03 21:10:21 +00:00
package cli
import (
"context"
"encoding/json"
"fmt"
"path/filepath"
"reflect"
"strings"
"sync"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/framework/evidencecontext"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
enemyevents "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/enemyevents"
enemyeventnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/enemyevents"
"go.maximumdirect.net/promptkit"
2026-08-03 21:10:21 +00:00
)
func TestProductionEnemyEventConfigurationResolvesGeneratedHandoffs(t *testing.T) {
components := productionTestComponents(t)
configPath := writeEnemyEventContractConfig(t)
loaded, err := config.Load(configPath, config.LoadOptions{})
if err != nil {
t.Fatal(err)
}
cfg := loaded.RuntimeConfig()
effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalogFromRegistries(components.registries)})
2026-08-03 21:10:21 +00:00
if err != nil {
t.Fatalf("Resolve() error = %v, want nil", err)
}
materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{
ConfigPath: configPath,
WorkingDir: filepath.Dir(configPath),
2026-08-03 21:10:21 +00:00
})
if err != nil {
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
}
lane := referenceContractLane(t, materialized, "enemy-events")
if lane.ArtifactKind != dnd.EnemyEventListKind || lane.Extract.Module != enemyevents.Key || lane.Merge.Module != pipeline.DefaultMergeModule || lane.Normalize.Module != enemyeventnormalize.Key {
2026-08-03 21:10:21 +00:00
t.Fatalf("enemy event lane = %#v, want typed production composition", lane)
}
for slot, want := range map[string]struct{ step, lane string }{
2026-08-05 20:07:31 +00:00
"npc_registry": {step: "describe-session", lane: "npc-registry"},
2026-08-03 21:10:21 +00:00
"scene_descriptions": {step: "describe-session", lane: "scene-descriptions"},
"combat_turns": {step: "extract-events", lane: "combat-turns"},
"npc_occurrences": {step: "extract-events", lane: "npc-occurrences"},
2026-08-03 21:10:21 +00:00
} {
binding, found := generatedReferenceBinding(lane.ExtractReferences.Bindings, slot)
if !found || binding.Artifact.Step != want.step || binding.Artifact.Lane != want.lane {
t.Fatalf("enemy event %s reference = %#v, want generated %s/%s artifact", slot, binding, want.step, want.lane)
}
}
2026-08-05 20:07:31 +00:00
if binding, found := generatedReferenceBinding(lane.NormalizeReferences.Bindings, "npc_registry"); !found || binding.Artifact.Step != "describe-session" || binding.Artifact.Lane != "npc-registry" {
2026-08-03 21:10:21 +00:00
t.Fatalf("enemy event normalizer NPC reference = %#v, want generated NPC artifact", binding)
}
catalog := catalogFromRegistries(components.registries)
extractSpec, ok := catalog.Extractors.Spec(enemyevents.Key)
if !ok || !reflect.DeepEqual(extractSpec.Requires, []string{"chunks", "source.transcript"}) || !reflect.DeepEqual(extractSpec.Provides, []string{"dnd.enemy_events"}) {
t.Fatalf("enemy event extractor spec = %#v, want source and artifact capabilities", extractSpec)
}
normalizeSpec, ok := catalog.Normalizers.SpecForArtifact(enemyeventnormalize.Key, dnd.EnemyEventListKind)
if !ok || !reflect.DeepEqual(normalizeSpec.Requires, []string{"merged"}) || !reflect.DeepEqual(normalizeSpec.Provides, []string{"normalized"}) {
t.Fatalf("enemy event normalizer spec = %#v, want merged/normalized capabilities", normalizeSpec)
}
for _, slot := range []string{"npc_registry", "scene_descriptions", "combat_turns", "npc_occurrences"} {
2026-08-03 21:10:21 +00:00
if !hasReferenceSlot(extractSpec.ReferenceSlots, slot) {
t.Fatalf("enemy event extractor slots = %#v, want %q", extractSpec.ReferenceSlots, slot)
}
}
if !hasReferenceSlot(normalizeSpec.ReferenceSlots, "npc_registry") {
2026-08-03 21:10:21 +00:00
t.Fatalf("enemy event normalizer slots = %#v, want NPC registry", normalizeSpec.ReferenceSlots)
}
profile := cfg.Pipelines["dnd-session"]
profile.Steps[2].References["npc_registry"] = pipeline.GeneratedReference("track-enemies", "enemy-events")
2026-08-03 21:10:21 +00:00
cfg.Pipelines["dnd-session"] = profile
if _, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalogFromRegistries(components.registries)}); err == nil || !strings.Contains(err.Error(), "earlier step") {
2026-08-03 21:10:21 +00:00
t.Fatalf("Resolve() error = %v, want future generated-reference rejection", err)
}
}
func TestNativeOrderedDNDRunPublishesRegistryBackedEntityOccurrences(t *testing.T) {
configPath := writeEnemyEventContractConfig(t)
2026-08-03 21:10:21 +00:00
outputRoot := filepath.Join(t.TempDir(), "output")
client := &enemyEventLLMClient{}
options := productionRunOptions(t, client)
2026-08-03 21:10:21 +00:00
var stdout, stderr strings.Builder
code := RunWithOptions([]string{
"run", "dnd-session", "--config", configPath,
"--input-file", repositoryPath("examples", "dnd-complete-transcript.json"),
"--cache", "bypass", "--output-dir", outputRoot,
"--llm-session-id", "enemy-event-session", "--format", "json",
2026-08-03 21:10:21 +00:00
}, &stdout, &stderr, options)
if code != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
var report runResult
if err := json.Unmarshal([]byte(stdout.String()), &report); err != nil {
t.Fatal(err)
}
2026-08-03 21:10:21 +00:00
runRoot := filepath.Join(outputRoot, productionRunID)
if report.OutputDirectory != runRoot || report.ArtifactCount != 12 || report.FallbackCount != 0 {
t.Fatalf("completion=%#v, want ten domain lanes and two auxiliary artifacts without fallback", report)
}
if files := readTree(t, runRoot); len(files) != report.ArtifactCount {
t.Fatalf("published files=%v, report=%#v", files, report)
}
value := readProductionJSON[dnd.EnemyEventList](t, filepath.Join(runRoot, "lanes/enemy-events.json"))
2026-08-03 21:10:21 +00:00
if len(value.Events) != 1 || value.Events[0].Name != "Kesh" || value.Events[0].Kind != dnd.EnemyEventKindFled || len(value.Events[0].SourceRefs) != 1 || value.Events[0].SourceRefs[0].SourceID != "session-ravenfall" || value.Events[0].SourceRefs[0].StartUnitID != 10 {
t.Fatalf("enemy event artifact = %#v, want source-linked Kesh fleeing event", value)
}
locationsValue := readProductionJSON[dnd.LocationRegistry](t, filepath.Join(runRoot, "lanes/location-registry.json"))
if len(locationsValue.Locations) != 2 || locationsValue.Locations[0].Name != "Moon Gate" || locationsValue.Locations[1].Name != "Moon Gate" || locationsValue.Locations[0].ID == locationsValue.Locations[1].ID {
t.Fatalf("location registry = %#v, want distinct source-grounded identities for same-name locations", locationsValue)
}
occurrencesValue := readProductionJSON[dnd.LocationOccurrenceList](t, filepath.Join(runRoot, "lanes/location-occurrences.json"))
if len(occurrencesValue.Occurrences) != 2 || occurrencesValue.Occurrences[0].LocationID == occurrencesValue.Occurrences[1].LocationID || occurrencesValue.Occurrences[0].Name != "Moon Gate" || occurrencesValue.Occurrences[1].Name != "Moon Gate" {
t.Fatalf("location occurrences = %#v, want source-grounded references to distinct registry identities", occurrencesValue)
}
npcRegistry := readProductionJSON[dnd.NPCRegistry](t, filepath.Join(runRoot, "lanes/npc-registry.json"))
2026-08-05 20:07:31 +00:00
if len(npcRegistry.NPCs) != 1 || npcRegistry.NPCs[0].Name != "Kesh" {
t.Fatalf("NPC registry = %#v, want Kesh identity", npcRegistry)
}
npcOccurrences := readProductionJSON[dnd.NPCOccurrenceList](t, filepath.Join(runRoot, "lanes/npc-occurrences.json"))
2026-08-05 20:07:31 +00:00
if len(npcOccurrences.Occurrences) != 1 || npcOccurrences.Occurrences[0].NPCID != npcRegistry.NPCs[0].ID || npcOccurrences.Occurrences[0].Name != "Kesh" || len(npcOccurrences.Occurrences[0].SourceRefs) != 1 || npcOccurrences.Occurrences[0].SourceRefs[0].StartUnitID != 7 {
t.Fatalf("NPC occurrences = %#v, want independently evidenced Kesh registry grounding", npcOccurrences)
}
itemRegistry := readProductionJSON[dnd.ItemRegistry](t, filepath.Join(runRoot, "lanes/item-registry.json"))
2026-08-05 20:07:31 +00:00
if len(itemRegistry.Items) != 1 || itemRegistry.Items[0].Name != "Moonblade" {
t.Fatalf("item registry = %#v, want Moonblade identity", itemRegistry)
}
itemOccurrences := readProductionJSON[dnd.ItemOccurrenceList](t, filepath.Join(runRoot, "lanes/item-occurrences.json"))
2026-08-05 20:07:31 +00:00
if len(itemOccurrences.Occurrences) != 1 || itemOccurrences.Occurrences[0].ItemID != itemRegistry.Items[0].ID || itemOccurrences.Occurrences[0].Name != "Moonblade" || len(itemOccurrences.Occurrences[0].SourceRefs) != 1 || itemOccurrences.Occurrences[0].SourceRefs[0].StartUnitID != 5 {
t.Fatalf("item occurrences = %#v, want independently evidenced Moonblade registry grounding", itemOccurrences)
}
2026-08-03 21:10:21 +00:00
evidence := readProductionJSON[evidencecontext.Document](t, filepath.Join(runRoot, "evidence-context.json"))
if len(evidence) == 0 {
t.Fatalf("evidence context = %#v, want selected source-unit evidence", evidence)
}
seenEvidenceUnits := make(map[int]struct{}, len(evidence))
for _, unit := range evidence {
if unit.Ref.SourceID != "session-ravenfall" || unit.Ref.StartUnitID != unit.ID || unit.Ref.EndUnitID != unit.ID {
t.Fatalf("evidence unit = %#v, want unchanged source-unit self-reference", unit)
}
if _, exists := seenEvidenceUnits[unit.ID]; exists {
t.Fatalf("evidence context = %#v, want each source unit once", evidence)
}
seenEvidenceUnits[unit.ID] = struct{}{}
2026-08-03 21:10:21 +00:00
}
requests := client.requestsFor("events")
2026-08-03 21:10:21 +00:00
if len(requests) != 1 {
t.Fatalf("enemy event requests=%d, want only the combat scene", len(requests))
2026-08-03 21:10:21 +00:00
}
request := requests[0]
if request.Prompt.SessionID != "enemy-event-session" {
t.Fatalf("session=%q", request.Prompt.SessionID)
2026-08-03 21:10:21 +00:00
}
for _, field := range []string{"npcs", "combat_turns", "npc_occurrences"} {
input, err := productionJSONInput(request, field)
if err != nil || !strings.Contains(string(input), "Kesh") || strings.Contains(string(input), "source_refs") || strings.Contains(string(input), "start_unit_id") {
t.Fatalf("%s projection=%s error=%v, want compact source-free grounding", field, input, err)
2026-08-03 21:10:21 +00:00
}
}
for _, check := range []struct {
role, field, name string
contextual bool
2026-08-05 20:07:31 +00:00
}{
{"location-occurrences", "locations", "Moon Gate", true},
{"npc-occurrences", "npcs", "Kesh", false},
{"item-occurrences", "items", "Moonblade", false},
2026-08-05 20:07:31 +00:00
} {
requests := client.requestsFor(check.role)
2026-08-05 20:07:31 +00:00
if len(requests) != 2 {
t.Fatalf("%s requests=%d, want one per scene", check.role, len(requests))
2026-08-05 20:07:31 +00:00
}
for _, request := range requests {
input, err := productionJSONInput(request, check.field)
if err != nil || !strings.Contains(string(input), check.name) || strings.Contains(string(input), `"id"`) || strings.Contains(string(input), "source_refs") || strings.Contains(string(input), "registry_refs") != check.contextual {
t.Fatalf("%s projection=%s error=%v", check.role, input, err)
2026-08-05 20:07:31 +00:00
}
}
}
2026-08-03 21:10:21 +00:00
}
func writeEnemyEventContractConfig(t *testing.T) string {
2026-08-03 21:10:21 +00:00
t.Helper()
return writeProductionContractConfig(t, fmt.Sprintf(`version: 6
pipelines:
dnd-session:
cache: bypass
input: {module: seriatim}
chunk: {module: dnd/scenes}
references:
party: {path: %q}
glossary: {path: %q}
output:
module: json
options:
include_chunk_map: true
evidence_context:
enabled: true
window_units: 3
lanes: [item-occurrences, item-registry, location-registry, location-occurrences, npc-registry, spells, combat-turns, npc-occurrences, enemy-events]
steps:
- id: describe-session
artifacts:
item-registry:
extract: {module: dnd/item-registry}
normalize: {module: dnd/item-registry}
npc-registry:
extract: {module: dnd/npc-registry}
normalize: {module: dnd/npc-registry}
location-registry:
extract: {module: dnd/location-registry}
normalize: {module: dnd/location-registry}
scene-descriptions:
extract: {module: dnd/scene-descriptions}
normalize: {module: dnd/scene-descriptions}
- id: extract-events
references:
location_registry: {artifact: {step: describe-session, lane: location-registry}}
npc_registry: {artifact: {step: describe-session, lane: npc-registry}}
scene_descriptions: {artifact: {step: describe-session, lane: scene-descriptions}}
item_registry: {artifact: {step: describe-session, lane: item-registry}}
artifacts:
item-occurrences:
extract: {module: dnd/item-occurrences}
normalize: {module: dnd/item-occurrences}
spells:
extract:
module: dnd/spells
references:
spell_catalog: {path: %q}
normalize:
module: dnd/spells
references:
spell_catalog: {path: %q}
combat-turns:
extract: {module: dnd/combat-turns}
normalize: {module: dnd/combat-turns}
npc-occurrences:
extract: {module: dnd/npc-occurrences}
normalize: {module: dnd/npc-occurrences}
location-occurrences:
extract: {module: dnd/location-occurrences}
normalize: {module: dnd/location-occurrences}
- id: track-enemies
references:
npc_registry: {artifact: {step: describe-session, lane: npc-registry}}
scene_descriptions: {artifact: {step: describe-session, lane: scene-descriptions}}
combat_turns: {artifact: {step: extract-events, lane: combat-turns}}
npc_occurrences: {artifact: {step: extract-events, lane: npc-occurrences}}
artifacts:
enemy-events:
extract: {module: dnd/enemy-events}
normalize: {module: dnd/enemy-events}
`+productionProviderConfiguration,
repositoryPath("examples", "dnd-party.txt"), repositoryPath("examples", "dnd-glossary.txt"),
repositoryPath("examples", "dnd-spell-catalog.json"), repositoryPath("examples", "dnd-spell-catalog.json")))
2026-08-03 21:10:21 +00:00
}
type enemyEventLLMClient struct {
mu sync.Mutex
requests []promptkit.GenerateRequest
2026-08-03 21:10:21 +00:00
}
func (client *enemyEventLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
2026-08-03 21:10:21 +00:00
if err := ctx.Err(); err != nil {
return nil, err
}
role := enemyEventRequestRole(request)
var combatScene bool
if role != "duplicate_groups" && role != "scenes" {
transcript, err := productionJSONInput(request, "units")
if err != nil {
return nil, err
}
combatScene = strings.Contains(string(transcript), "Roll initiative")
2026-08-03 21:10:21 +00:00
}
var content []byte
switch role {
case "scenes":
2026-08-03 21:10:21 +00:00
content = []byte(`{"scenes":[{"start_unit_id":1,"end_unit_id":6},{"start_unit_id":7,"end_unit_id":11}]}`)
case "npcs":
2026-08-03 21:10:21 +00:00
if combatScene {
content = []byte(`{"npcs":[{"name":"Kesh","source_refs":[{"start_unit_id":7,"end_unit_id":7}]}]}`)
} else {
content = []byte(`{"npcs":[]}`)
}
case "duplicate_groups":
2026-08-03 21:10:21 +00:00
content = []byte(`{"duplicate_groups":[]}`)
case "kind":
2026-08-03 21:10:21 +00:00
kind, title := "narrative", "Arrival"
if combatScene {
kind, title = "combat", "Raiders attack"
}
content = []byte(fmt.Sprintf(`{"kind":%q,"title":%q,"summary":"session scene"}`, kind, title))
case "locations":
unitID := 1
if combatScene {
unitID = 7
}
content = []byte(fmt.Sprintf(`{"locations":[{"name":"Moon Gate","source_refs":[{"start_unit_id":%d,"end_unit_id":%d}]}]}`, unitID, unitID))
case "spell_casts":
2026-08-03 21:10:21 +00:00
content = []byte(`{"spell_casts":[]}`)
case "items":
2026-08-05 20:07:31 +00:00
if combatScene {
content = []byte(`{"items":[]}`)
} else {
content = []byte(`{"items":[{"name":"Moonblade","source_refs":[{"start_unit_id":5,"end_unit_id":5}]}]}`)
}
case "item-occurrences":
2026-08-05 20:07:31 +00:00
if combatScene {
content = []byte(`{"occurrences":[]}`)
} else {
var registry struct {
Items []struct {
Name string `json:"name"`
2026-08-05 20:07:31 +00:00
} `json:"items"`
}
data, err := productionJSONInput(request, "items")
if err != nil {
return nil, err
}
if err := json.Unmarshal(data, &registry); err != nil {
return nil, fmt.Errorf("decode generated item registry: %w", err)
2026-08-05 20:07:31 +00:00
}
if len(registry.Items) != 1 || registry.Items[0].Name != "Moonblade" {
return nil, fmt.Errorf("generated item registry has %d items, want 1", len(registry.Items))
2026-08-05 20:07:31 +00:00
}
2026-08-09 02:11:27 +00:00
content = []byte(`{"occurrences":[{"name":"Moonblade","kind":"discovered","quantity":null,"from":null,"to":null,"source_refs":[{"start_unit_id":5,"end_unit_id":5}]}]}`)
2026-08-05 20:07:31 +00:00
}
case "combat_turns":
2026-08-03 21:10:21 +00:00
content = []byte(`{"combat_turns":[{"actor":"Kesh","turn_kind":"turn","source_refs":[{"start_unit_id":8,"end_unit_id":8}]}]}`)
case "npc-occurrences":
2026-08-03 21:10:21 +00:00
if combatScene {
var registry struct {
NPCs []struct {
Name string `json:"name"`
} `json:"npcs"`
}
data, err := productionJSONInput(request, "npcs")
if err != nil {
return nil, err
}
if err := json.Unmarshal(data, &registry); err != nil {
return nil, fmt.Errorf("decode generated NPC registry: %w", err)
}
if len(registry.NPCs) == 0 || registry.NPCs[0].Name != "Kesh" {
return nil, fmt.Errorf("generated NPC registry has no NPCs")
}
content = []byte(`{"occurrences":[{"name":"Kesh","kind":"combat_opponent","source_refs":[{"start_unit_id":7,"end_unit_id":7}]}]}`)
2026-08-03 21:10:21 +00:00
} else {
content = []byte(`{"occurrences":[]}`)
2026-08-03 21:10:21 +00:00
}
case "location-occurrences":
var registry struct {
Locations []struct {
Name string `json:"name"`
RegistryRefs []struct {
StartUnitID int `json:"start_unit_id"`
EndUnitID int `json:"end_unit_id"`
} `json:"registry_refs"`
} `json:"locations"`
}
data, err := productionJSONInput(request, "locations")
if err != nil {
return nil, err
}
if err := json.Unmarshal(data, &registry); err != nil {
return nil, fmt.Errorf("decode generated location registry: %w", err)
}
if len(registry.Locations) == 0 {
return nil, fmt.Errorf("generated location registry has no locations")
}
unitID := 1
location := registry.Locations[0]
if combatScene {
unitID = 7
if len(registry.Locations) > 1 {
location = registry.Locations[1]
}
}
registryRefs, err := json.Marshal(location.RegistryRefs)
if err != nil {
return nil, fmt.Errorf("encode location selector: %w", err)
}
content = []byte(fmt.Sprintf(`{"occurrences":[{"name":%q,"registry_refs":%s,"kind":"visited","source_refs":[{"start_unit_id":%d,"end_unit_id":%d}]}]}`, location.Name, registryRefs, unitID, unitID))
case "events":
2026-08-03 21:10:21 +00:00
content = []byte(`{"events":[{"name":"Kesh","kind":"fled","source_refs":[{"start_unit_id":10,"end_unit_id":10}]}]}`)
default:
return nil, fmt.Errorf("unexpected native response contract %q", role)
}
2026-08-03 21:10:21 +00:00
client.mu.Lock()
client.requests = append(client.requests, request)
2026-08-03 21:10:21 +00:00
client.mu.Unlock()
return &promptkit.GenerateResponse{Content: string(content)}, nil
}
// Distinguish the real prepared output contracts and fixed registry projections,
// not module-private prompt identifiers or a synthetic completion interface.
func enemyEventRequestRole(request promptkit.GenerateRequest) string {
for _, property := range []string{"scenes", "npcs", "locations", "spell_casts", "items", "duplicate_groups", "kind", "combat_turns", "events"} {
if productionSchemaHasProperty(request, property) {
return property
}
}
if productionSchemaHasProperty(request, "occurrences") {
for _, field := range []string{"locations", "npcs", "items"} {
if _, err := productionJSONInput(request, field); err == nil {
switch field {
case "locations":
return "location-occurrences"
case "npcs":
return "npc-occurrences"
case "items":
return "item-occurrences"
}
}
}
}
return ""
2026-08-03 21:10:21 +00:00
}
func (client *enemyEventLLMClient) requestsFor(role string) []promptkit.GenerateRequest {
2026-08-03 21:10:21 +00:00
client.mu.Lock()
defer client.mu.Unlock()
var requests []promptkit.GenerateRequest
2026-08-03 21:10:21 +00:00
for _, request := range client.requests {
if enemyEventRequestRole(request) == role {
requests = append(requests, request)
2026-08-03 21:10:21 +00:00
}
}
return requests
}
func containsString(values []string, want string) bool {
for _, value := range values {
if value == want {
return true
}
}
return false
}
func generatedReferenceBinding(bindings []pipeline.ReferenceBinding, slotName string) (pipeline.ReferenceBinding, bool) {
for _, binding := range bindings {
if binding.SlotName == slotName && binding.Artifact != nil {
return binding, true
}
}
return pipeline.ReferenceBinding{}, false
}