Files
notarius/internal/modules/integration/dnd_npc_grounded_test.go

421 lines
17 KiB
Go

package integration_test
import (
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"reflect"
"strings"
"sync"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/evidencecontext"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
combatcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/combatturns"
npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcregistry"
scenecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/scenedescriptions"
combatsemantics "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/combat_semantics"
"go.maximumdirect.net/promptkit"
)
func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T) {
registries := productionNPCRegistries(t)
catalog := moduleCatalog(registries)
configValue := loadGroundedPipelineConfig(t)
campaignInputs := configureGroundedCampaignReferences(t, &configValue)
effective, err := configValue.Resolve(config.ResolveInput{PipelineID: "dnd-npc-grounded", Catalog: catalog})
if err != nil {
t.Fatal(err)
}
materialized, warnings, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{})
if err != nil || len(warnings) != 0 {
t.Fatalf("materialize references: %v warnings=%#v", err, warnings)
}
client := &groundedDNDLLMClient{sceneTitle: "eligibility-only-scene-sentinel"}
prepared, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{Engine: nativeTestEngine(t, client)})
if err != nil {
t.Fatal(err)
}
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{Prepared: prepared, RawInput: readNPCFixture(t), Workers: 1})
if err != nil {
t.Fatal(err)
}
if len(output.OutputFiles) != 4 || output.FallbackCount != 0 {
t.Fatalf("output=%#v, want four accepted domain artifacts", output)
}
npcs, err := npccodec.New().Decode(publishedLane(t, output, "npc_registry").Bytes)
if err != nil || len(npcs.NPCs) != 2 {
t.Fatalf("NPC registry=%#v error=%v", npcs, err)
}
scenes, err := scenecodec.New().Decode(publishedLane(t, output, "scene-descriptions").Bytes)
if err != nil || len(scenes.Scenes) != 1 || scenes.Scenes[0].Kind != dnd.SceneKindCombat {
t.Fatalf("scene descriptions=%#v error=%v", scenes, err)
}
projection := `{"npcs":[{"name":"Mira Thorn"},{"name":"Hooded Guard"}]}`
seenConsumers := map[string]bool{}
for _, request := range client.requestsSnapshot() {
kind := "spell_casts"
if nativeSchemaHasProperty(request, "combat_turns") {
kind = "combat_turns"
} else if !nativeSchemaHasProperty(request, "spell_casts") {
continue
}
input, err := nativeJSONInput(request, "npcs")
if err != nil || string(input) != projection {
t.Fatalf("%s NPC projection=%s error=%v", kind, input, err)
}
text := nativePromptText(request)
for _, npc := range npcs.NPCs {
if strings.Contains(text, npc.ID) {
t.Fatalf("%s prompt contains opaque registry identity", kind)
}
}
if kind == "combat_turns" {
transcript, err := nativeJSONInput(request, "units")
if err != nil || !strings.Contains(string(transcript), "Mira Thorn asks the party") {
t.Fatalf("combat transcript=%s error=%v", transcript, err)
}
for name, want := range campaignInputs {
if !strings.Contains(text, want) {
t.Fatalf("combat prompt omitted %s reference", name)
}
}
for _, excluded := range []string{client.sceneTitle, "The party faces an active encounter."} {
if strings.Contains(text, excluded) {
t.Fatalf("combat prompt included eligibility-only scene prose %q", excluded)
}
}
}
seenConsumers[kind] = true
}
if !seenConsumers["spell_casts"] || !seenConsumers["combat_turns"] || combatCompletionCount(client.requestsSnapshot()) != 1 {
t.Fatalf("consumer native requests=%#v", seenConsumers)
}
spellsValue := decodeRunnerSpellResponse(t, publishedLane(t, output, "spells").Bytes)
if len(spellsValue.SpellCasts) != 1 || spellsValue.SpellCasts[0].Caster != "Mira Thorn" {
t.Fatalf("spells=%#v", spellsValue)
}
assertCurrentEvidence(t, spellsValue.SpellCasts[0].SourceRefs)
combatValue := publishedCombatOutput(t, output)
if len(combatValue.CombatTurns) != 1 || combatValue.CombatTurns[0].Actor != "Mira Thorn" {
t.Fatalf("combat=%#v", combatValue)
}
assertCurrentEvidence(t, combatValue.CombatTurns[0].SourceRefs)
}
func TestSceneCombatSemanticsCorrectionRevalidatesRevisedScene(t *testing.T) {
registries := productionNPCRegistries(t)
configValue := loadGroundedPipelineConfig(t)
profile := configValue.Pipelines["dnd-npc-grounded"]
sceneLane := profile.Steps[0].Artifacts["scene-descriptions"]
turns := 1
sceneLane.Extract.MaxAdditionalTurns = &turns
sceneLane.Extract.Validators = pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{
pipeline.Binding("generic/valid_json"),
pipeline.Binding("extract/dnd/scene-descriptions/shape"),
pipeline.Binding("extract/dnd/scene-descriptions/source_refs"),
pipeline.Binding("generic/valid_json_schema"),
pipeline.Binding("extract/dnd/scene-descriptions/source_relatedness"),
pipeline.Binding(combatsemantics.Key),
}}
profile.Steps[0].Artifacts["scene-descriptions"] = sceneLane
configValue.Pipelines["dnd-npc-grounded"] = profile
client := &groundedDNDLLMClient{sceneKinds: []dnd.SceneKind{dnd.SceneKindNarrative, dnd.SceneKindCombat}, combatSemanticsClassifications: []string{"combat", "combat"}}
output := runGroundedPipeline(t, configValue, registries, client)
if output.FallbackCount != 0 {
t.Fatalf("unexpected fallback: %#v", output)
}
for _, group := range output.Diagnostics.Groups {
if group.Origin.ValidatorKey == combatsemantics.Key {
t.Fatalf("combat-semantics validator left a diagnostic after correction: %#v", group)
}
}
scenes, err := scenecodec.New().Decode(publishedLane(t, output, "scene-descriptions").Bytes)
if err != nil {
t.Fatal(err)
}
if len(scenes.Scenes) != 1 || scenes.Scenes[0].Kind != dnd.SceneKindCombat {
t.Fatalf("accepted scenes = %#v, want corrected combat scene", scenes)
}
requests := client.requestsSnapshot()
var sceneRequests, validatorRequests []promptkit.GenerateRequest
for _, request := range requests {
switch {
case nativeSchemaHasProperty(request, "kind"):
sceneRequests = append(sceneRequests, request)
case nativeSchemaHasProperty(request, "classification"):
validatorRequests = append(validatorRequests, request)
}
}
if len(sceneRequests) != 2 || len(validatorRequests) != 2 {
t.Fatalf("scene/validator calls=%d/%d, want correction and revalidation", len(sceneRequests), len(validatorRequests))
}
original := sceneRequests[0].Prompt.Messages
revised := sceneRequests[1].Prompt.Messages
if len(revised) != len(original)+2 || !reflect.DeepEqual(revised[:len(original)], original) {
t.Fatal("scene correction lost native conversation history")
}
assistant, guidance := revised[len(original)], revised[len(original)+1]
if assistant.Role != promptkit.RoleAssistant || !strings.Contains(assistant.Content, `"kind":"narrative"`) || guidance.Role != promptkit.RoleUser || !strings.Contains(guidance.Content, "Return kind: combat") || !strings.Contains(guidance.Content, "active encounter") {
t.Fatalf("native correction messages=%#v", revised[len(original):])
}
for _, forbidden := range []string{combatsemantics.Key, combatsemantics.ReasonCodeActiveCombatNotClassified, "session:chunk"} {
if strings.Contains(guidance.Content, forbidden) {
t.Fatalf("correction guidance contains diagnostic identifier %q", forbidden)
}
}
}
func TestProductionDNDOutputPublishesSelectedEvidenceContext(t *testing.T) {
registries := productionNPCRegistries(t)
configValue := loadGroundedPipelineConfig(t)
profile := configValue.Pipelines["dnd-npc-grounded"]
profile.Output.Options = map[string]any{"evidence_context": map[string]any{
"enabled": true, "window_units": 0, "lanes": []any{"npc_registry", "spells", "combat"},
}}
configValue.Pipelines["dnd-npc-grounded"] = profile
raw := strings.NewReplacer(
`"id": 1`, `"id": 10`,
`"id": 2`, `"id": 30`,
`"id": 3`, `"id": 20`,
`"id": 4`, `"id": 50`,
`"id": 5`, `"id": 40`,
).Replace(string(readNPCFixture(t)))
output := runGroundedPipelineWithRaw(t, configValue, registries, &groundedDNDLLMClient{firstUnitID: 10, thirdUnitID: 20}, []byte(raw))
value, err := evidencecontext.New().Decode(outputFileContent(t, output.OutputFiles, "evidence-context.json"))
if err != nil {
t.Fatalf("Decode(evidence context) error = %v", err)
}
if actual := evidenceUnitIDs(value); !reflect.DeepEqual(actual, []int{10, 20}) {
t.Fatalf("evidence units = %#v, want deduplicated source-position union without scene descriptions", actual)
}
}
func TestProductionDNDOutputCanExplicitlySelectSceneDescriptionEvidence(t *testing.T) {
registries := productionNPCRegistries(t)
configValue := loadGroundedPipelineConfig(t)
profile := configValue.Pipelines["dnd-npc-grounded"]
profile.Output.Options = map[string]any{"evidence_context": map[string]any{
"enabled": true, "lanes": []any{"scene-descriptions"},
}}
configValue.Pipelines["dnd-npc-grounded"] = profile
output := runGroundedPipeline(t, configValue, registries, &groundedDNDLLMClient{})
value, err := evidencecontext.New().Decode(outputFileContent(t, output.OutputFiles, "evidence-context.json"))
if err != nil {
t.Fatalf("Decode(evidence context) error = %v", err)
}
if len(value) == 0 {
t.Fatalf("evidence context = %#v, want explicitly selected scene-description source units", value)
}
}
func evidenceUnitIDs(value evidencecontext.Document) []int {
ids := make([]int, len(value))
for index, unit := range value {
ids[index] = unit.ID
}
return ids
}
func TestGroundedPipelineSkipsCombatForExactNarrativeScene(t *testing.T) {
registries := productionNPCRegistries(t)
configValue := loadGroundedPipelineConfig(t)
client := &groundedDNDLLMClient{sceneKind: dnd.SceneKindNarrative}
output := runGroundedPipeline(t, configValue, registries, client)
if combatCompletionCount(client.requestsSnapshot()) != 0 {
t.Fatalf("combat completion requests = %#v, want none for an exact narrative scene", client.requestsSnapshot())
}
if output.FallbackCount != 0 || hasCombatWarning(output.Diagnostics, "scene_classification_unavailable") {
t.Fatalf("narrative output=%#v, want accepted skip", output)
}
combatValue := publishedCombatOutput(t, output)
if combatValue.CombatTurns == nil || len(combatValue.CombatTurns) != 0 {
t.Fatalf("narrative combat output = %#v, want an accepted empty combat list", combatValue)
}
}
func assertCurrentEvidence(t *testing.T, references []source.SourceRef) {
t.Helper()
for _, reference := range references {
if reference.SourceID != "npc-session" {
t.Fatalf("evidence reference = %#v, want current source only", reference)
}
}
}
func loadGroundedPipelineConfig(t *testing.T) config.Config {
t.Helper()
return loadIntegrationConfig(t, repositoryPathForIntegration("internal", "modules", "integration", "testdata", "dnd_npc_grounded_pipeline.yml"))
}
func configureGroundedCampaignReferences(t *testing.T, cfg *config.Config) map[string]string {
t.Helper()
values := map[string]string{
"party": "Mira Thorn: ally\n",
"glossary": "Greencloak: Mira Thorn's title\n",
}
profile := cfg.Pipelines["dnd-npc-grounded"]
profile.References = make(map[string]pipeline.ReferenceSource, len(values))
for name, content := range values {
path := filepath.Join(t.TempDir(), name+".txt")
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
t.Fatalf("write %s reference: %v", name, err)
}
profile.References[name] = pipeline.ExternalReference(path)
}
cfg.Pipelines["dnd-npc-grounded"] = profile
return values
}
func runGroundedPipeline(t *testing.T, configValue config.Config, registries pipeline.Registries, client *groundedDNDLLMClient) pipeline.RunOutput {
return runGroundedPipelineWithRaw(t, configValue, registries, client, readNPCFixture(t))
}
func runGroundedPipelineWithRaw(t *testing.T, configValue config.Config, registries pipeline.Registries, client *groundedDNDLLMClient, raw []byte) pipeline.RunOutput {
t.Helper()
catalog := moduleCatalog(registries)
effective, err := configValue.Resolve(config.ResolveInput{PipelineID: "dnd-npc-grounded", Catalog: catalog})
if err != nil {
t.Fatalf("Resolve() error = %v", err)
}
materialized, warnings, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{})
if err != nil || len(warnings) != 0 {
t.Fatalf("MaterializeReferences() error = %v warnings = %#v", err, warnings)
}
prepared, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{Engine: nativeTestEngine(t, client)})
if err != nil {
t.Fatalf("Prepare() error = %v", err)
}
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
Prepared: prepared,
RawInput: append([]byte(nil), raw...),
Workers: 1,
})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
return output
}
func publishedCombatOutput(t *testing.T, output pipeline.RunOutput) dnd.CombatTurnList {
t.Helper()
value, err := combatcodec.New().Decode(publishedLane(t, output, "combat").Bytes)
if err != nil {
t.Fatal(err)
}
return value
}
func combatCompletionCount(requests []promptkit.GenerateRequest) int {
count := 0
for _, request := range requests {
if nativeSchemaHasProperty(request, "combat_turns") {
count++
}
}
return count
}
type groundedDNDLLMClient struct {
mu sync.Mutex
requests []promptkit.GenerateRequest
sceneKind dnd.SceneKind
sceneTitle string
firstUnitID int
thirdUnitID int
sceneKinds []dnd.SceneKind
combatSemanticsClassifications []string
sceneCalls int
combatSemanticsCalls int
}
func (client *groundedDNDLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
client.mu.Lock()
defer client.mu.Unlock()
client.requests = append(client.requests, request)
var payload any
firstUnitID := client.firstUnitID
if firstUnitID == 0 {
firstUnitID = 1
}
thirdUnitID := client.thirdUnitID
if thirdUnitID == 0 {
thirdUnitID = 3
}
switch {
case nativeSchemaHasProperty(request, "npcs"):
payload = map[string]any{"npcs": []any{
map[string]any{
"name": "Mira Thorn", "source_refs": []any{map[string]int{"start_unit_id": firstUnitID, "end_unit_id": firstUnitID}},
},
map[string]any{
"name": "Hooded Guard", "source_refs": []any{map[string]int{"start_unit_id": thirdUnitID, "end_unit_id": thirdUnitID}},
},
}}
case nativeSchemaHasProperty(request, "duplicate_groups"):
payload = map[string]any{"duplicate_groups": []any{}}
case nativeSchemaHasProperty(request, "kind"):
kind := client.sceneKind
if client.sceneCalls < len(client.sceneKinds) {
kind = client.sceneKinds[client.sceneCalls]
}
client.sceneCalls++
if kind == "" {
kind = dnd.SceneKindCombat
}
title := client.sceneTitle
if title == "" {
title = "A combat encounter"
}
payload = map[string]any{"kind": kind, "title": title, "summary": "The party faces an active encounter."}
case nativeSchemaHasProperty(request, "classification"):
classification := "combat"
if client.combatSemanticsCalls < len(client.combatSemanticsClassifications) {
classification = client.combatSemanticsClassifications[client.combatSemanticsCalls]
}
client.combatSemanticsCalls++
payload = map[string]any{"classification": classification, "explanation": "The transcript shows an active encounter with hostile action."}
case nativeSchemaHasProperty(request, "spell_casts"):
payload = map[string]any{"spell_casts": []any{map[string]any{
"caster": "Mira Thorn", "spell": "Cure Wounds",
"source_refs": []any{map[string]int{"start_unit_id": firstUnitID, "end_unit_id": firstUnitID}},
}}}
case nativeSchemaHasProperty(request, "combat_turns"):
payload = map[string]any{"combat_turns": []any{map[string]any{
"actor": "Mira Thorn",
"turn_kind": "turn",
"source_refs": []any{map[string]int{"start_unit_id": firstUnitID, "end_unit_id": firstUnitID}},
}}}
default:
return nil, fmt.Errorf("unexpected native grounded response contract")
}
content, err := json.Marshal(payload)
if err != nil {
return nil, err
}
return &promptkit.GenerateResponse{Content: string(content)}, nil
}
func (client *groundedDNDLLMClient) requestsSnapshot() []promptkit.GenerateRequest {
client.mu.Lock()
defer client.mu.Unlock()
return append([]promptkit.GenerateRequest(nil), client.requests...)
}
var _ promptkit.LLMClient = (*groundedDNDLLMClient)(nil)