Files
notarius/internal/modules/integration/dnd_spells_runner_test.go

287 lines
9.7 KiB
Go
Raw Normal View History

2026-07-17 05:05:48 +00:00
package integration_test
import (
"context"
"os"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
spellcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/spells"
2026-07-17 05:05:48 +00:00
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
2026-07-22 14:30:20 +00:00
spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells"
spellshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/shape"
2026-07-22 14:38:35 +00:00
spellsourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/source_refs"
"gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript"
"go.maximumdirect.net/promptkit"
)
func runPreparedPipeline(t *testing.T, registries pipeline.Registries, resolved pipeline.ResolvedPipeline, llmClient promptkit.LLMClient, input pipeline.RunInput) (pipeline.RunOutput, error) {
t.Helper()
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{Engine: nativeTestEngine(t, llmClient)})
if err != nil {
return pipeline.RunOutput{}, err
}
input.Prepared = prepared
return pipeline.New().Run(context.Background(), input)
}
func TestRunnerProcessesSeriatimInputWithDNDSpellsExtractor(t *testing.T) {
raw := readDNDSpellsFixture(t)
expectedDoc := parseDNDSpellsFixture(t, raw)
resolved := resolveDNDSpellsPipeline(t)
llmClient := &fakeSpellsLLMClient{
response: extractionResponse{
SpellCasts: []spellCastResponse{
{
Caster: "Aria",
Spell: "Cure Wounds",
SourceRefs: responseSourceRefs(1, 1),
},
{
Caster: "Borin",
Spell: "Fire Bolt",
SourceRefs: responseSourceRefs(3, 3),
},
},
},
}
output, err := runPreparedPipeline(t, dndSpellsRunnerRegistries(t), resolved.ResolvedPipeline, llmClient, pipeline.RunInput{
RawInput: raw,
})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.OutputFiles) != 1 {
t.Fatalf("len(OutputFiles) = %d, want 1", len(output.OutputFiles))
}
if output.OutputFiles[0].Name != "lanes/spells.json" {
t.Fatalf("spell output = %#v", output.OutputFiles)
}
response := decodeRunnerSpellResponse(t, output.OutputFiles[0].Bytes)
2026-07-07 18:58:23 +00:00
if len(response.SpellCasts) != 2 {
t.Fatalf("len(spell_casts) = %d, want 2", len(response.SpellCasts))
}
2026-07-07 18:58:23 +00:00
first, second := response.SpellCasts[0], response.SpellCasts[1]
if first.Spell != "Cure Wounds" || second.Spell != "Fire Bolt" {
t.Fatalf("spell order = %q, %q; want source-unit order", first.Spell, second.Spell)
}
if first.Caster != "Aria" || second.Caster != "Borin" {
2026-07-07 18:58:23 +00:00
t.Fatalf("casters = %q, %q; want spell data", first.Caster, second.Caster)
}
2026-07-07 18:58:23 +00:00
for _, spell := range response.SpellCasts {
if len(spell.SourceRefs) != 1 {
t.Fatalf("len(SourceRefs) = %d, want 1", len(spell.SourceRefs))
}
2026-07-07 18:58:23 +00:00
if spell.SourceRefs[0].SourceID != expectedDoc.ID {
t.Fatalf("SourceID = %q, want fixture document ID", spell.SourceRefs[0].SourceID)
}
}
2026-07-03 21:26:05 -05:00
if output.OutputFiles[0].ContentType != "application/json" {
t.Fatalf("ContentType = %q, want application/json", output.OutputFiles[0].ContentType)
}
}
func TestRunnerPassesPartyAndGlossaryReferencesToDNDSpellsPrompt(t *testing.T) {
2026-07-05 14:49:17 +00:00
raw := readDNDSpellsFixture(t)
resolved := resolveDNDSpellsPipeline(t)
resolved.ResolvedPipeline.Steps[0].ArtifactLanes[0].ExtractReferences.ReferenceSet = dndSpellsReferenceSet(
2026-07-05 14:49:17 +00:00
"Aria: party cleric\nBorin: fighter",
"Fire Bolt: evocation cantrip",
)
llmClient := &fakeSpellsLLMClient{
response: extractionResponse{
SpellCasts: []spellCastResponse{
{
Caster: "Borin",
Spell: "Fire Bolt",
SourceRefs: responseSourceRefs(3, 3),
2026-07-05 14:49:17 +00:00
},
},
},
}
output, err := runPreparedPipeline(t, dndSpellsRunnerRegistries(t), resolved.ResolvedPipeline, llmClient, pipeline.RunInput{
RawInput: raw,
2026-07-05 14:49:17 +00:00
})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.OutputFiles) != 1 {
t.Fatalf("len(OutputFiles) = %d, want 1", len(output.OutputFiles))
2026-07-05 14:49:17 +00:00
}
if len(llmClient.requests) != 1 {
t.Fatalf("LLM calls = %d, want 1", len(llmClient.requests))
}
request := llmClient.requests[0]
for _, value := range []string{"Aria: party cleric\nBorin: fighter", "Fire Bolt: evocation cantrip"} {
if !strings.Contains(nativePromptText(request), value) {
t.Fatalf("rendered prompt omitted %q", value)
}
2026-07-05 14:49:17 +00:00
}
}
2026-07-22 14:38:35 +00:00
func TestProductionSpellPipelineRoutesSemanticCandidatesToDeterministicValidators(t *testing.T) {
2026-07-22 14:30:20 +00:00
registries := productionNPCRegistries(t)
configValue := config.Default()
configValue.Pipelines["dnd-spells-shape"] = pipeline.PipelineProfile{
Input: pipeline.Binding(transcript.Key),
Chunk: pipeline.ModuleBinding{Module: pipeline.DefaultChunkModule, Options: map[string]any{"max_units": 100}},
Artifacts: map[string]pipeline.ArtifactLaneProfile{
"spells": {
Extract: pipeline.ModuleBinding{Module: spells.Key},
2026-07-22 14:30:20 +00:00
Normalize: pipeline.Binding(spellnormalize.Key),
},
},
}
effective, err := configValue.Resolve(config.ResolveInput{
PipelineID: "dnd-spells-shape",
Catalog: moduleCatalog(registries),
})
if err != nil {
t.Fatalf("Resolve() error = %v, want nil", err)
}
2026-07-22 14:38:35 +00:00
for _, test := range []struct {
name string
response []byte
validatorName string
}{
{
name: "blank string",
response: []byte(`{"spell_casts":[{"caster":"","spell":"Cure Wounds","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`),
2026-07-22 14:38:35 +00:00
validatorName: spellshape.Key,
},
{
name: "empty evidence",
response: []byte(`{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","source_refs":[]}]}`),
2026-07-22 14:38:35 +00:00
validatorName: spellshape.Key,
},
{
name: "nonpositive unit candidate",
response: []byte(`{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","source_refs":[{"start_unit_id":0,"end_unit_id":1}]}]}`),
2026-07-22 14:38:35 +00:00
validatorName: spellsourcerefs.Key,
},
{
name: "unknown unit candidate",
response: []byte(`{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","source_refs":[{"start_unit_id":99,"end_unit_id":99}]}]}`),
2026-07-22 14:38:35 +00:00
validatorName: spellsourcerefs.Key,
},
} {
t.Run(test.name, func(t *testing.T) {
client := &fakeSpellsLLMClient{rawResponses: [][]byte{test.response, test.response, test.response}}
resolved := effective.ResolvedPipeline
output, err := runPreparedPipeline(t, registries, resolved, client, pipeline.RunInput{RawInput: readDNDSpellsFixture(t)})
if err == nil || !strings.Contains(err.Error(), test.validatorName) {
t.Fatalf("Run() error = %v, want scoped %s rejection", err, test.validatorName)
2026-07-22 14:38:35 +00:00
}
if len(client.requests) != 3 || len(output.OutputFiles) != 0 {
t.Fatalf("requests=%d files=%#v, want exhausted rejection", len(client.requests), output.OutputFiles)
2026-07-22 14:38:35 +00:00
}
})
2026-07-22 14:30:20 +00:00
}
}
func dndSpellsReferenceSet(party string, glossary string) contracts.ReferenceSet {
2026-07-05 14:49:17 +00:00
slots := make(map[string]contracts.ResolvedReferenceSlot)
if strings.TrimSpace(party) != "" {
slots["party"] = contracts.ResolvedReferenceSlot{
Slot: contracts.ReferenceSlot{Name: "party"},
2026-07-05 14:49:17 +00:00
Items: []contracts.ReferenceItem{
{
SlotName: "party",
2026-07-05 14:49:17 +00:00
MediaType: "text/plain; charset=utf-8",
Content: []byte(party),
Digest: "sha256:party",
Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///tmp/party.txt"},
SizeBytes: int64(len(party)),
2026-07-05 14:49:17 +00:00
BindingSource: contracts.ReferenceBindingSourceConfig,
},
},
}
}
if strings.TrimSpace(glossary) != "" {
slots["glossary"] = contracts.ResolvedReferenceSlot{
Slot: contracts.ReferenceSlot{Name: "glossary"},
Items: []contracts.ReferenceItem{
{
SlotName: "glossary",
MediaType: "text/plain; charset=utf-8",
Content: []byte(glossary),
Digest: "sha256:glossary",
Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///tmp/glossary.txt"},
SizeBytes: int64(len(glossary)),
BindingSource: contracts.ReferenceBindingSourceConfig,
},
},
}
}
return contracts.ReferenceSet{Slots: slots}
}
func resolveDNDSpellsPipeline(t *testing.T) config.EffectiveConfig {
t.Helper()
resolved, err := loadDNDSpellsPipelineConfig(t).Resolve(config.ResolveInput{
PipelineID: "dnd-spells-fixture",
Catalog: dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{}),
})
if err != nil {
t.Fatalf("Resolve() error = %v, want nil", err)
}
return resolved
}
func dndSpellsRunnerRegistries(t *testing.T) pipeline.Registries {
t.Helper()
catalog := dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{})
return pipeline.Registries{
Inputs: catalog.Inputs,
Chunkers: catalog.Chunkers,
ArtifactCodecs: catalog.ArtifactCodecs,
ArtifactEvidence: catalog.ArtifactEvidence,
Extractors: catalog.Extractors,
Mergers: catalog.Mergers,
Normalizers: catalog.Normalizers,
Outputs: catalog.Outputs,
}
}
func readDNDSpellsFixture(t *testing.T) []byte {
t.Helper()
raw, err := os.ReadFile("testdata/seriatim_spell_session.json")
if err != nil {
t.Fatalf("ReadFile(seriatim_spell_session.json) error = %v, want nil", err)
}
return raw
}
func parseDNDSpellsFixture(t *testing.T, raw []byte) *source.SourceDocument {
t.Helper()
doc, err := transcript.New().Parse(context.Background(), contracts.ParseRequest{Raw: raw})
if err != nil {
t.Fatalf("Parse() error = %v, want nil", err)
}
return doc
}
2026-07-07 18:58:23 +00:00
func decodeRunnerSpellResponse(t *testing.T, raw []byte) dnd.SpellList {
2026-07-07 18:58:23 +00:00
t.Helper()
response, err := spellcodec.New().Decode(raw)
if err != nil {
t.Fatalf("decode durable spell output: %v", err)
2026-07-07 18:58:23 +00:00
}
return response
}