package integration_test import ( "context" "os" "strings" "testing" "gitea.maximumdirect.net/eric/notarius/internal/core/config" "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" spellcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/spells" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells" spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells" spellshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/shape" spellsourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/source_refs" "gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript" "go.maximumdirect.net/promptkit" ) func runPreparedPipeline(t *testing.T, registries pipeline.Registries, resolved pipeline.ResolvedPipeline, llmClient promptkit.LLMClient, input pipeline.RunInput) (pipeline.RunOutput, error) { t.Helper() prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{Engine: nativeTestEngine(t, llmClient)}) if err != nil { return pipeline.RunOutput{}, err } input.Prepared = prepared return pipeline.New().Run(context.Background(), input) } func TestRunnerProcessesSeriatimInputWithDNDSpellsExtractor(t *testing.T) { raw := readDNDSpellsFixture(t) expectedDoc := parseDNDSpellsFixture(t, raw) resolved := resolveDNDSpellsPipeline(t) llmClient := &fakeSpellsLLMClient{ response: extractionResponse{ SpellCasts: []spellCastResponse{ { Caster: "Aria", Spell: "Cure Wounds", SourceRefs: responseSourceRefs(1, 1), }, { Caster: "Borin", Spell: "Fire Bolt", SourceRefs: responseSourceRefs(3, 3), }, }, }, } output, err := runPreparedPipeline(t, dndSpellsRunnerRegistries(t), resolved.ResolvedPipeline, llmClient, pipeline.RunInput{ RawInput: raw, }) if err != nil { t.Fatalf("Run() error = %v, want nil", err) } if len(output.OutputFiles) != 1 { t.Fatalf("len(OutputFiles) = %d, want 1", len(output.OutputFiles)) } if output.OutputFiles[0].Name != "lanes/spells.json" { t.Fatalf("spell output = %#v", output.OutputFiles) } response := decodeRunnerSpellResponse(t, output.OutputFiles[0].Bytes) if len(response.SpellCasts) != 2 { t.Fatalf("len(spell_casts) = %d, want 2", len(response.SpellCasts)) } first, second := response.SpellCasts[0], response.SpellCasts[1] if first.Spell != "Cure Wounds" || second.Spell != "Fire Bolt" { t.Fatalf("spell order = %q, %q; want source-unit order", first.Spell, second.Spell) } if first.Caster != "Aria" || second.Caster != "Borin" { t.Fatalf("casters = %q, %q; want spell data", first.Caster, second.Caster) } for _, spell := range response.SpellCasts { if len(spell.SourceRefs) != 1 { t.Fatalf("len(SourceRefs) = %d, want 1", len(spell.SourceRefs)) } if spell.SourceRefs[0].SourceID != expectedDoc.ID { t.Fatalf("SourceID = %q, want fixture document ID", spell.SourceRefs[0].SourceID) } } if output.OutputFiles[0].ContentType != "application/json" { t.Fatalf("ContentType = %q, want application/json", output.OutputFiles[0].ContentType) } } func TestRunnerPassesPartyAndGlossaryReferencesToDNDSpellsPrompt(t *testing.T) { raw := readDNDSpellsFixture(t) resolved := resolveDNDSpellsPipeline(t) resolved.ResolvedPipeline.Steps[0].ArtifactLanes[0].ExtractReferences.ReferenceSet = dndSpellsReferenceSet( "Aria: party cleric\nBorin: fighter", "Fire Bolt: evocation cantrip", ) llmClient := &fakeSpellsLLMClient{ response: extractionResponse{ SpellCasts: []spellCastResponse{ { Caster: "Borin", Spell: "Fire Bolt", SourceRefs: responseSourceRefs(3, 3), }, }, }, } output, err := runPreparedPipeline(t, dndSpellsRunnerRegistries(t), resolved.ResolvedPipeline, llmClient, pipeline.RunInput{ RawInput: raw, }) if err != nil { t.Fatalf("Run() error = %v, want nil", err) } if len(output.OutputFiles) != 1 { t.Fatalf("len(OutputFiles) = %d, want 1", len(output.OutputFiles)) } if len(llmClient.requests) != 1 { t.Fatalf("LLM calls = %d, want 1", len(llmClient.requests)) } request := llmClient.requests[0] for _, value := range []string{"Aria: party cleric\nBorin: fighter", "Fire Bolt: evocation cantrip"} { if !strings.Contains(nativePromptText(request), value) { t.Fatalf("rendered prompt omitted %q", value) } } } func TestProductionSpellPipelineRoutesSemanticCandidatesToDeterministicValidators(t *testing.T) { registries := productionNPCRegistries(t) configValue := config.Default() configValue.Pipelines["dnd-spells-shape"] = pipeline.PipelineProfile{ Input: pipeline.Binding(transcript.Key), Chunk: pipeline.ModuleBinding{Module: pipeline.DefaultChunkModule, Options: map[string]any{"max_units": 100}}, Artifacts: map[string]pipeline.ArtifactLaneProfile{ "spells": { Extract: pipeline.ModuleBinding{Module: spells.Key}, Normalize: pipeline.Binding(spellnormalize.Key), }, }, } effective, err := configValue.Resolve(config.ResolveInput{ PipelineID: "dnd-spells-shape", Catalog: moduleCatalog(registries), }) if err != nil { t.Fatalf("Resolve() error = %v, want nil", err) } for _, test := range []struct { name string response []byte validatorName string }{ { name: "blank string", response: []byte(`{"spell_casts":[{"caster":"","spell":"Cure Wounds","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`), validatorName: spellshape.Key, }, { name: "empty evidence", response: []byte(`{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","source_refs":[]}]}`), validatorName: spellshape.Key, }, { name: "nonpositive unit candidate", response: []byte(`{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","source_refs":[{"start_unit_id":0,"end_unit_id":1}]}]}`), validatorName: spellsourcerefs.Key, }, { name: "unknown unit candidate", response: []byte(`{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","source_refs":[{"start_unit_id":99,"end_unit_id":99}]}]}`), validatorName: spellsourcerefs.Key, }, } { t.Run(test.name, func(t *testing.T) { client := &fakeSpellsLLMClient{rawResponses: [][]byte{test.response, test.response, test.response}} resolved := effective.ResolvedPipeline output, err := runPreparedPipeline(t, registries, resolved, client, pipeline.RunInput{RawInput: readDNDSpellsFixture(t)}) if err == nil || !strings.Contains(err.Error(), test.validatorName) { t.Fatalf("Run() error = %v, want scoped %s rejection", err, test.validatorName) } if len(client.requests) != 3 || len(output.OutputFiles) != 0 { t.Fatalf("requests=%d files=%#v, want exhausted rejection", len(client.requests), output.OutputFiles) } }) } } func dndSpellsReferenceSet(party string, glossary string) contracts.ReferenceSet { slots := make(map[string]contracts.ResolvedReferenceSlot) if strings.TrimSpace(party) != "" { slots["party"] = contracts.ResolvedReferenceSlot{ Slot: contracts.ReferenceSlot{Name: "party"}, Items: []contracts.ReferenceItem{ { SlotName: "party", MediaType: "text/plain; charset=utf-8", Content: []byte(party), Digest: "sha256:party", Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///tmp/party.txt"}, SizeBytes: int64(len(party)), BindingSource: contracts.ReferenceBindingSourceConfig, }, }, } } if strings.TrimSpace(glossary) != "" { slots["glossary"] = contracts.ResolvedReferenceSlot{ Slot: contracts.ReferenceSlot{Name: "glossary"}, Items: []contracts.ReferenceItem{ { SlotName: "glossary", MediaType: "text/plain; charset=utf-8", Content: []byte(glossary), Digest: "sha256:glossary", Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///tmp/glossary.txt"}, SizeBytes: int64(len(glossary)), BindingSource: contracts.ReferenceBindingSourceConfig, }, }, } } return contracts.ReferenceSet{Slots: slots} } func resolveDNDSpellsPipeline(t *testing.T) config.EffectiveConfig { t.Helper() resolved, err := loadDNDSpellsPipelineConfig(t).Resolve(config.ResolveInput{ PipelineID: "dnd-spells-fixture", Catalog: dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{}), }) if err != nil { t.Fatalf("Resolve() error = %v, want nil", err) } return resolved } func dndSpellsRunnerRegistries(t *testing.T) pipeline.Registries { t.Helper() catalog := dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{}) return pipeline.Registries{ Inputs: catalog.Inputs, Chunkers: catalog.Chunkers, ArtifactCodecs: catalog.ArtifactCodecs, ArtifactEvidence: catalog.ArtifactEvidence, Extractors: catalog.Extractors, Mergers: catalog.Mergers, Normalizers: catalog.Normalizers, Outputs: catalog.Outputs, } } func readDNDSpellsFixture(t *testing.T) []byte { t.Helper() raw, err := os.ReadFile("testdata/seriatim_spell_session.json") if err != nil { t.Fatalf("ReadFile(seriatim_spell_session.json) error = %v, want nil", err) } return raw } func parseDNDSpellsFixture(t *testing.T, raw []byte) *source.SourceDocument { t.Helper() doc, err := transcript.New().Parse(context.Background(), contracts.ParseRequest{Raw: raw}) if err != nil { t.Fatalf("Parse() error = %v, want nil", err) } return doc } func decodeRunnerSpellResponse(t *testing.T, raw []byte) dnd.SpellList { t.Helper() response, err := spellcodec.New().Decode(raw) if err != nil { t.Fatalf("decode durable spell output: %v", err) } return response }