Files
notarius/internal/modules/integration/dnd_combat_runner_test.go

368 lines
15 KiB
Go

package integration_test
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"runtime"
"strings"
"sync"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
combatcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/combatturns"
npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcregistry"
scenecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/scenedescriptions"
combatextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/combatturns"
combatnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/combatturns"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity"
npcregistry "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/registry"
"gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript"
"go.maximumdirect.net/promptkit"
)
func TestProductionCombatPipelineCorrectsMergesNormalizesAndWritesJSON(t *testing.T) {
registries := productionNPCRegistries(t)
catalog := moduleCatalog(registries)
npcPayload := combatTestNPCPayload(t)
configValue := combatOnlyConfig()
effective, err := configValue.Resolve(config.ResolveInput{
PipelineID: "dnd-combat-fixture",
Catalog: catalog,
ReferenceOverrides: []pipeline.ReferenceBinding{
{Stage: pipeline.StageExtract, LaneID: "combat", SlotName: "npc_registry", Source: npcPayload.path, BindingSource: contracts.ReferenceBindingSourceCLI},
{Stage: pipeline.StageNormalize, LaneID: "combat", SlotName: "npc_registry", Source: npcPayload.path, BindingSource: contracts.ReferenceBindingSourceCLI},
},
})
if err != nil {
t.Fatalf("Resolve() error = %v, want nil", err)
}
materialized, warnings, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{})
if err != nil || len(warnings) != 0 {
t.Fatalf("MaterializeReferences() error = %v warnings = %#v, want no warnings", err, warnings)
}
client := &fakeCombatLLMClient{responses: []string{
combatTestTurnResponse("", "turn", 1),
combatTestTurnResponse("mira thorn", "turn", 1),
combatTestTurnResponse("Mira Thorn", "reaction", 3),
combatTestTurnResponse("Hooded Guard", "turn", 5),
}}
prepared, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{Engine: nativeTestEngine(t, client)})
if err != nil {
t.Fatalf("Prepare() error = %v, want nil", err)
}
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
Prepared: prepared,
RawInput: readNPCFixture(t),
Workers: 1,
})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(client.requests) != 4 || len(output.OutputFiles) != 1 || output.FallbackCount != 0 {
t.Fatalf("native calls=%d output=%#v, want corrected accepted combat", len(client.requests), output)
}
value, err := combatcodec.New().Decode(publishedLane(t, output, "combat").Bytes)
if err != nil {
t.Fatal(err)
}
if len(value.CombatTurns) != 3 || value.CombatTurns[0].Actor != "Mira Thorn" || value.CombatTurns[1].Actor != "Mira Thorn" || value.CombatTurns[2].Actor != "Hooded Guard" {
t.Fatalf("normalized combat output = %#v, want ordered canonical actors", value)
}
for _, turn := range value.CombatTurns {
for _, ref := range turn.SourceRefs {
if ref.SourceID != "npc-session" {
t.Fatalf("combat evidence ref = %#v, want current transcript source", ref)
}
}
}
if !hasCombatWarning(output.Diagnostics, combatnormalize.ReasonCodeActorCanonicalized) {
t.Fatalf("diagnostics = %#v, want registry normalization advisory", output.Diagnostics)
}
var combatFile *contracts.OutputFile
for index := range output.OutputFiles {
if output.OutputFiles[index].Name == "lanes/combat.json" {
combatFile = &output.OutputFiles[index]
break
}
}
if combatFile == nil || combatFile.ContentType != combatcodec.MediaType {
t.Fatalf("output files = %#v, want lanes/combat.json JSON payload", output.OutputFiles)
}
}
func TestProductionCombatPipelineStopsAfterNativeStructuralFailure(t *testing.T) {
registries := productionNPCRegistries(t)
configValue := combatOnlyConfig()
profile := configValue.Pipelines["dnd-combat-fixture"]
profile.Chunk.Options["max_units"] = 100
profile.References["scene_descriptions"] = pipeline.ExternalReference(combatSceneDescriptionPath(t, []dnd.SceneDescription{{
ID: "chunk-000001",
SourceRef: source.SourceRef{SourceID: "npc-session", StartUnitID: 1, EndUnitID: 5},
Kind: dnd.SceneKindCombat,
Title: "Complete combat encounter",
Summary: "The accepted chunk contains active combat.",
}}))
configValue.Pipelines["dnd-combat-fixture"] = profile
effective, err := configValue.Resolve(config.ResolveInput{
PipelineID: "dnd-combat-fixture",
Catalog: moduleCatalog(registries),
})
if err != nil {
t.Fatalf("Resolve() error = %v, want nil", err)
}
client := &fakeCombatLLMClient{response: combatTestInvalidEnumResponse("unsupported")}
materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, moduleCatalog(registries), pipeline.ReferenceMaterializationOptions{})
if err != nil {
t.Fatalf("MaterializeReferences() error = %v", err)
}
prepared, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{Engine: nativeTestEngine(t, client)})
if err != nil {
t.Fatalf("Prepare() error = %v, want nil", err)
}
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
Prepared: prepared,
RawInput: readNPCFixture(t),
Workers: 1,
})
if !errors.Is(err, promptkit.ErrValidation) {
t.Fatalf("Run() error=%v, want native structural failure", err)
}
if len(client.requests) == 0 || len(output.OutputFiles) != 0 {
t.Fatalf("native calls=%d output=%#v, want exhausted rejection", len(client.requests), output)
}
}
func TestCombatPipelinePublishesUnavailableSceneWarningForMismatchedReference(t *testing.T) {
registries := productionNPCRegistries(t)
configValue := combatOnlyConfig()
profile := configValue.Pipelines["dnd-combat"]
profile.Chunk.Options["max_units"] = 100
profile.References["scene_descriptions"] = pipeline.ExternalReference(combatSceneDescriptionPath(t, []dnd.SceneDescription{{
ID: "chunk-000001",
SourceRef: source.SourceRef{SourceID: "npc-session", StartUnitID: 2, EndUnitID: 5},
Kind: dnd.SceneKindCombat,
Title: "Mismatched combat encounter",
Summary: "The classification range does not match the accepted chunk.",
}}))
configValue.Pipelines["dnd-combat"] = profile
catalog := moduleCatalog(registries)
effective, err := configValue.Resolve(config.ResolveInput{PipelineID: "dnd-combat", Catalog: catalog})
if err != nil {
t.Fatalf("Resolve() error = %v", err)
}
materialized, warnings, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{})
if err != nil || len(warnings) != 0 {
t.Fatalf("MaterializeReferences() error = %v warnings = %#v", err, warnings)
}
client := &fakeCombatLLMClient{}
prepared, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{Engine: nativeTestEngine(t, client)})
if err != nil {
t.Fatalf("Prepare() error = %v", err)
}
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
Prepared: prepared,
RawInput: readNPCFixture(t),
Workers: 1,
})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if len(client.requests) != 0 || !hasCombatWarning(output.Diagnostics, "scene_classification_unavailable") {
t.Fatalf("native calls=%d diagnostics=%#v, want unavailable-scene skip", len(client.requests), output.Diagnostics)
}
value, err := combatcodec.New().Decode(publishedLane(t, output, "combat").Bytes)
if err != nil {
t.Fatalf("Decode(combat output) error = %v", err)
}
if value.CombatTurns == nil || len(value.CombatTurns) != 0 {
t.Fatalf("mismatched scene combat output = %#v, want non-nil empty turns", value)
}
}
func TestCombatNormalizerRejectsCampaignReferenceBinding(t *testing.T) {
registries := productionNPCRegistries(t)
catalog := moduleCatalog(registries)
referencePath := filepath.Join(t.TempDir(), "party.yml")
if err := os.WriteFile(referencePath, []byte("Aria: cleric\n"), 0o600); err != nil {
t.Fatal(err)
}
effective, err := combatOnlyConfig().Resolve(config.ResolveInput{
PipelineID: "dnd-combat-fixture",
Catalog: catalog,
ReferenceOverrides: []pipeline.ReferenceBinding{{
Stage: pipeline.StageNormalize, LaneID: "combat", SlotName: "party", Source: referencePath,
BindingSource: contracts.ReferenceBindingSourceCLI,
}},
})
if err != nil {
if !strings.Contains(err.Error(), "party") {
t.Fatalf("Resolve() error = %v, want undeclared party context", err)
}
return
}
_, _, err = pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{})
if err == nil || !strings.Contains(err.Error(), "party") || !strings.Contains(err.Error(), "not declared") {
t.Fatalf("MaterializeReferences() error = %v, want undeclared normalize party binding", err)
}
}
func TestCombatPreparationRejectsMalformedOrOversizedNPCReferencesBeforeExecution(t *testing.T) {
registries := productionNPCRegistries(t)
catalog := moduleCatalog(registries)
for _, test := range []struct {
name string
content []byte
}{
{name: "malformed", content: []byte(`{"npcs":[`)},
{name: "oversized", content: append([]byte(`{"npcs":[]}`), make([]byte, npcregistry.MaxBytes+1)...)},
} {
t.Run(test.name, func(t *testing.T) {
path := filepath.Join(t.TempDir(), "npcs.json")
if err := os.WriteFile(path, test.content, 0o600); err != nil {
t.Fatal(err)
}
cfg := combatOnlyConfig()
effective, err := cfg.Resolve(config.ResolveInput{
PipelineID: "dnd-combat",
Catalog: catalog,
ReferenceOverrides: []pipeline.ReferenceBinding{
{Stage: pipeline.StageExtract, LaneID: "combat", SlotName: "npc_registry", Source: path, BindingSource: contracts.ReferenceBindingSourceCLI},
{Stage: pipeline.StageNormalize, LaneID: "combat", SlotName: "npc_registry", Source: path, BindingSource: contracts.ReferenceBindingSourceCLI},
},
})
if err != nil {
t.Fatalf("Resolve() error = %v, want path binding to resolve before content preparation", err)
}
materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{})
if test.name == "oversized" {
if err == nil || !strings.Contains(err.Error(), "limit") {
t.Fatalf("MaterializeReferences() error = %v, want bounded oversized-reference failure", err)
}
return
}
if err != nil {
t.Fatalf("MaterializeReferences() error = %v, want materialization before typed preparation", err)
}
client := &fakeCombatLLMClient{}
if _, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{Engine: nativeTestEngine(t, client)}); err == nil {
t.Fatal("Prepare() error = nil, want NPC registry preparation failure")
}
if len(client.requests) != 0 {
t.Fatalf("LLM requests = %d, want no pipeline execution after preparation failure", len(client.requests))
}
})
}
}
type combatNPCPayload struct {
path string
}
func combatTestNPCPayload(t *testing.T) combatNPCPayload {
t.Helper()
value := dnd.NPCRegistry{NPCs: []dnd.NPC{
{ID: identity.DeriveID("Mira Thorn"), Name: "Mira Thorn", SourceRefs: []source.SourceRef{{SourceID: "prior-npc-session", StartUnitID: 1, EndUnitID: 1}}},
{ID: identity.DeriveID("Hooded Guard"), Name: "Hooded Guard", SourceRefs: []source.SourceRef{{SourceID: "prior-npc-session", StartUnitID: 3, EndUnitID: 3}}},
}}
content, err := npccodec.New().Encode(value)
if err != nil {
t.Fatalf("Encode(NPC fixture) error = %v", err)
}
path := filepath.Join(t.TempDir(), "npc-run", "lanes", "npcs.json")
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, content, 0o600); err != nil {
t.Fatal(err)
}
return combatNPCPayload{path: path}
}
func combatSceneDescriptionPath(t *testing.T, scenes []dnd.SceneDescription) string {
t.Helper()
content, err := scenecodec.New().Encode(dnd.SceneDescriptionList{Scenes: scenes})
if err != nil {
t.Fatalf("Encode(scene descriptions) error = %v", err)
}
path := filepath.Join(t.TempDir(), "scene-descriptions.json")
if err := os.WriteFile(path, content, 0o600); err != nil {
t.Fatal(err)
}
return path
}
func combatOnlyConfig() config.Config {
cfg := config.Default()
cfg.Pipelines["dnd-combat"] = pipeline.PipelineProfile{
Input: pipeline.Binding(transcript.Key),
Chunk: pipeline.ModuleBinding{Module: pipeline.DefaultChunkModule, Options: map[string]any{"max_units": 2}},
References: map[string]pipeline.ReferenceSource{"scene_descriptions": pipeline.ExternalReference(repositoryPathForIntegration("internal", "modules", "integration", "testdata", "combat_scene_descriptions.json"))},
Artifacts: map[string]pipeline.ArtifactLaneProfile{
"combat": {
Extract: pipeline.ModuleBinding{Module: combatextract.Key},
Normalize: pipeline.Binding(combatnormalize.Key),
},
},
}
cfg.Pipelines["dnd-combat-fixture"] = cfg.Pipelines["dnd-combat"]
return cfg
}
func repositoryPathForIntegration(parts ...string) string {
_, file, _, _ := runtime.Caller(0)
return filepath.Join(append([]string{filepath.Dir(file), "..", "..", ".."}, parts...)...)
}
type fakeCombatLLMClient struct {
mu sync.Mutex
response string
responses []string
requests []promptkit.GenerateRequest
}
func (client *fakeCombatLLMClient) Generate(ctx context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
client.mu.Lock()
client.requests = append(client.requests, req)
index := len(client.requests) - 1
client.mu.Unlock()
if client.response != "" {
return &promptkit.GenerateResponse{Content: client.response}, nil
}
if index >= len(client.responses) {
return nil, fmt.Errorf("no combat fake response for call %d", index)
}
return &promptkit.GenerateResponse{Content: client.responses[index]}, nil
}
func combatTestInvalidEnumResponse(turnKind string) string {
return fmt.Sprintf(`{"combat_turns":[{"actor":"Aria","turn_kind":%q,"source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`, turnKind)
}
func combatTestTurnResponse(actor, turnKind string, unit int) string {
return fmt.Sprintf(`{"combat_turns":[{"actor":%q,"turn_kind":%q,"source_refs":[{"start_unit_id":%d,"end_unit_id":%d}]}]}`, actor, turnKind, unit, unit)
}
func hasCombatWarning(diagnostics contracts.DiagnosticCollection, reason string) bool {
for _, group := range diagnostics.Groups {
if group.ReasonCode == reason {
return true
}
}
return false
}
var _ promptkit.LLMClient = (*fakeCombatLLMClient)(nil)