package integration_test import ( "context" "encoding/json" "fmt" "os" "path/filepath" "reflect" "strings" "sync" "testing" "gitea.maximumdirect.net/eric/notarius/internal/core/config" "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/evidencecontext" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" combatcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/combatturns" npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcregistry" scenecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/scenedescriptions" combatsemantics "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/combat_semantics" "go.maximumdirect.net/promptkit" ) func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T) { registries := productionNPCRegistries(t) catalog := moduleCatalog(registries) configValue := loadGroundedPipelineConfig(t) campaignInputs := configureGroundedCampaignReferences(t, &configValue) effective, err := configValue.Resolve(config.ResolveInput{PipelineID: "dnd-npc-grounded", Catalog: catalog}) if err != nil { t.Fatal(err) } materialized, warnings, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{}) if err != nil || len(warnings) != 0 { t.Fatalf("materialize references: %v warnings=%#v", err, warnings) } client := &groundedDNDLLMClient{sceneTitle: "eligibility-only-scene-sentinel"} prepared, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{Engine: nativeTestEngine(t, client)}) if err != nil { t.Fatal(err) } output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{Prepared: prepared, RawInput: readNPCFixture(t), Workers: 1}) if err != nil { t.Fatal(err) } if len(output.OutputFiles) != 4 || output.FallbackCount != 0 { t.Fatalf("output=%#v, want four accepted domain artifacts", output) } npcs, err := npccodec.New().Decode(publishedLane(t, output, "npc_registry").Bytes) if err != nil || len(npcs.NPCs) != 2 { t.Fatalf("NPC registry=%#v error=%v", npcs, err) } scenes, err := scenecodec.New().Decode(publishedLane(t, output, "scene-descriptions").Bytes) if err != nil || len(scenes.Scenes) != 1 || scenes.Scenes[0].Kind != dnd.SceneKindCombat { t.Fatalf("scene descriptions=%#v error=%v", scenes, err) } projection := `{"npcs":[{"name":"Mira Thorn"},{"name":"Hooded Guard"}]}` seenConsumers := map[string]bool{} for _, request := range client.requestsSnapshot() { kind := "spell_casts" if nativeSchemaHasProperty(request, "combat_turns") { kind = "combat_turns" } else if !nativeSchemaHasProperty(request, "spell_casts") { continue } input, err := nativeJSONInput(request, "npcs") if err != nil || string(input) != projection { t.Fatalf("%s NPC projection=%s error=%v", kind, input, err) } text := nativePromptText(request) for _, npc := range npcs.NPCs { if strings.Contains(text, npc.ID) { t.Fatalf("%s prompt contains opaque registry identity", kind) } } if kind == "combat_turns" { transcript, err := nativeJSONInput(request, "units") if err != nil || !strings.Contains(string(transcript), "Mira Thorn asks the party") { t.Fatalf("combat transcript=%s error=%v", transcript, err) } for name, want := range campaignInputs { if !strings.Contains(text, want) { t.Fatalf("combat prompt omitted %s reference", name) } } for _, excluded := range []string{client.sceneTitle, "The party faces an active encounter."} { if strings.Contains(text, excluded) { t.Fatalf("combat prompt included eligibility-only scene prose %q", excluded) } } } seenConsumers[kind] = true } if !seenConsumers["spell_casts"] || !seenConsumers["combat_turns"] || combatCompletionCount(client.requestsSnapshot()) != 1 { t.Fatalf("consumer native requests=%#v", seenConsumers) } spellsValue := decodeRunnerSpellResponse(t, publishedLane(t, output, "spells").Bytes) if len(spellsValue.SpellCasts) != 1 || spellsValue.SpellCasts[0].Caster != "Mira Thorn" { t.Fatalf("spells=%#v", spellsValue) } assertCurrentEvidence(t, spellsValue.SpellCasts[0].SourceRefs) combatValue := publishedCombatOutput(t, output) if len(combatValue.CombatTurns) != 1 || combatValue.CombatTurns[0].Actor != "Mira Thorn" { t.Fatalf("combat=%#v", combatValue) } assertCurrentEvidence(t, combatValue.CombatTurns[0].SourceRefs) } func TestSceneCombatSemanticsCorrectionRevalidatesRevisedScene(t *testing.T) { registries := productionNPCRegistries(t) configValue := loadGroundedPipelineConfig(t) profile := configValue.Pipelines["dnd-npc-grounded"] sceneLane := profile.Steps[0].Artifacts["scene-descriptions"] turns := 1 sceneLane.Extract.MaxAdditionalTurns = &turns sceneLane.Extract.Validators = pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{ pipeline.Binding("generic/valid_json"), pipeline.Binding("extract/dnd/scene-descriptions/shape"), pipeline.Binding("extract/dnd/scene-descriptions/source_refs"), pipeline.Binding("generic/valid_json_schema"), pipeline.Binding("extract/dnd/scene-descriptions/source_relatedness"), pipeline.Binding(combatsemantics.Key), }} profile.Steps[0].Artifacts["scene-descriptions"] = sceneLane configValue.Pipelines["dnd-npc-grounded"] = profile client := &groundedDNDLLMClient{sceneKinds: []dnd.SceneKind{dnd.SceneKindNarrative, dnd.SceneKindCombat}, combatSemanticsClassifications: []string{"combat", "combat"}} output := runGroundedPipeline(t, configValue, registries, client) if output.FallbackCount != 0 { t.Fatalf("unexpected fallback: %#v", output) } for _, group := range output.Diagnostics.Groups { if group.Origin.ValidatorKey == combatsemantics.Key { t.Fatalf("combat-semantics validator left a diagnostic after correction: %#v", group) } } scenes, err := scenecodec.New().Decode(publishedLane(t, output, "scene-descriptions").Bytes) if err != nil { t.Fatal(err) } if len(scenes.Scenes) != 1 || scenes.Scenes[0].Kind != dnd.SceneKindCombat { t.Fatalf("accepted scenes = %#v, want corrected combat scene", scenes) } requests := client.requestsSnapshot() var sceneRequests, validatorRequests []promptkit.GenerateRequest for _, request := range requests { switch { case nativeSchemaHasProperty(request, "kind"): sceneRequests = append(sceneRequests, request) case nativeSchemaHasProperty(request, "classification"): validatorRequests = append(validatorRequests, request) } } if len(sceneRequests) != 2 || len(validatorRequests) != 2 { t.Fatalf("scene/validator calls=%d/%d, want correction and revalidation", len(sceneRequests), len(validatorRequests)) } original := sceneRequests[0].Prompt.Messages revised := sceneRequests[1].Prompt.Messages if len(revised) != len(original)+2 || !reflect.DeepEqual(revised[:len(original)], original) { t.Fatal("scene correction lost native conversation history") } assistant, guidance := revised[len(original)], revised[len(original)+1] if assistant.Role != promptkit.RoleAssistant || !strings.Contains(assistant.Content, `"kind":"narrative"`) || guidance.Role != promptkit.RoleUser || !strings.Contains(guidance.Content, "Return kind: combat") || !strings.Contains(guidance.Content, "active encounter") { t.Fatalf("native correction messages=%#v", revised[len(original):]) } for _, forbidden := range []string{combatsemantics.Key, combatsemantics.ReasonCodeActiveCombatNotClassified, "session:chunk"} { if strings.Contains(guidance.Content, forbidden) { t.Fatalf("correction guidance contains diagnostic identifier %q", forbidden) } } } func TestProductionDNDOutputPublishesSelectedEvidenceContext(t *testing.T) { registries := productionNPCRegistries(t) configValue := loadGroundedPipelineConfig(t) profile := configValue.Pipelines["dnd-npc-grounded"] profile.Output.Options = map[string]any{"evidence_context": map[string]any{ "enabled": true, "window_units": 0, "lanes": []any{"npc_registry", "spells", "combat"}, }} configValue.Pipelines["dnd-npc-grounded"] = profile raw := strings.NewReplacer( `"id": 1`, `"id": 10`, `"id": 2`, `"id": 30`, `"id": 3`, `"id": 20`, `"id": 4`, `"id": 50`, `"id": 5`, `"id": 40`, ).Replace(string(readNPCFixture(t))) output := runGroundedPipelineWithRaw(t, configValue, registries, &groundedDNDLLMClient{firstUnitID: 10, thirdUnitID: 20}, []byte(raw)) value, err := evidencecontext.New().Decode(outputFileContent(t, output.OutputFiles, "evidence-context.json")) if err != nil { t.Fatalf("Decode(evidence context) error = %v", err) } if actual := evidenceUnitIDs(value); !reflect.DeepEqual(actual, []int{10, 20}) { t.Fatalf("evidence units = %#v, want deduplicated source-position union without scene descriptions", actual) } } func TestProductionDNDOutputCanExplicitlySelectSceneDescriptionEvidence(t *testing.T) { registries := productionNPCRegistries(t) configValue := loadGroundedPipelineConfig(t) profile := configValue.Pipelines["dnd-npc-grounded"] profile.Output.Options = map[string]any{"evidence_context": map[string]any{ "enabled": true, "lanes": []any{"scene-descriptions"}, }} configValue.Pipelines["dnd-npc-grounded"] = profile output := runGroundedPipeline(t, configValue, registries, &groundedDNDLLMClient{}) value, err := evidencecontext.New().Decode(outputFileContent(t, output.OutputFiles, "evidence-context.json")) if err != nil { t.Fatalf("Decode(evidence context) error = %v", err) } if len(value) == 0 { t.Fatalf("evidence context = %#v, want explicitly selected scene-description source units", value) } } func evidenceUnitIDs(value evidencecontext.Document) []int { ids := make([]int, len(value)) for index, unit := range value { ids[index] = unit.ID } return ids } func TestGroundedPipelineSkipsCombatForExactNarrativeScene(t *testing.T) { registries := productionNPCRegistries(t) configValue := loadGroundedPipelineConfig(t) client := &groundedDNDLLMClient{sceneKind: dnd.SceneKindNarrative} output := runGroundedPipeline(t, configValue, registries, client) if combatCompletionCount(client.requestsSnapshot()) != 0 { t.Fatalf("combat completion requests = %#v, want none for an exact narrative scene", client.requestsSnapshot()) } if output.FallbackCount != 0 || hasCombatWarning(output.Diagnostics, "scene_classification_unavailable") { t.Fatalf("narrative output=%#v, want accepted skip", output) } combatValue := publishedCombatOutput(t, output) if combatValue.CombatTurns == nil || len(combatValue.CombatTurns) != 0 { t.Fatalf("narrative combat output = %#v, want an accepted empty combat list", combatValue) } } func assertCurrentEvidence(t *testing.T, references []source.SourceRef) { t.Helper() for _, reference := range references { if reference.SourceID != "npc-session" { t.Fatalf("evidence reference = %#v, want current source only", reference) } } } func loadGroundedPipelineConfig(t *testing.T) config.Config { t.Helper() return loadIntegrationConfig(t, repositoryPathForIntegration("internal", "modules", "integration", "testdata", "dnd_npc_grounded_pipeline.yml")) } func configureGroundedCampaignReferences(t *testing.T, cfg *config.Config) map[string]string { t.Helper() values := map[string]string{ "party": "Mira Thorn: ally\n", "glossary": "Greencloak: Mira Thorn's title\n", } profile := cfg.Pipelines["dnd-npc-grounded"] profile.References = make(map[string]pipeline.ReferenceSource, len(values)) for name, content := range values { path := filepath.Join(t.TempDir(), name+".txt") if err := os.WriteFile(path, []byte(content), 0o600); err != nil { t.Fatalf("write %s reference: %v", name, err) } profile.References[name] = pipeline.ExternalReference(path) } cfg.Pipelines["dnd-npc-grounded"] = profile return values } func runGroundedPipeline(t *testing.T, configValue config.Config, registries pipeline.Registries, client *groundedDNDLLMClient) pipeline.RunOutput { return runGroundedPipelineWithRaw(t, configValue, registries, client, readNPCFixture(t)) } func runGroundedPipelineWithRaw(t *testing.T, configValue config.Config, registries pipeline.Registries, client *groundedDNDLLMClient, raw []byte) pipeline.RunOutput { t.Helper() catalog := moduleCatalog(registries) effective, err := configValue.Resolve(config.ResolveInput{PipelineID: "dnd-npc-grounded", Catalog: catalog}) if err != nil { t.Fatalf("Resolve() error = %v", err) } materialized, warnings, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{}) if err != nil || len(warnings) != 0 { t.Fatalf("MaterializeReferences() error = %v warnings = %#v", err, warnings) } prepared, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{Engine: nativeTestEngine(t, client)}) if err != nil { t.Fatalf("Prepare() error = %v", err) } output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{ Prepared: prepared, RawInput: append([]byte(nil), raw...), Workers: 1, }) if err != nil { t.Fatalf("Run() error = %v", err) } return output } func publishedCombatOutput(t *testing.T, output pipeline.RunOutput) dnd.CombatTurnList { t.Helper() value, err := combatcodec.New().Decode(publishedLane(t, output, "combat").Bytes) if err != nil { t.Fatal(err) } return value } func combatCompletionCount(requests []promptkit.GenerateRequest) int { count := 0 for _, request := range requests { if nativeSchemaHasProperty(request, "combat_turns") { count++ } } return count } type groundedDNDLLMClient struct { mu sync.Mutex requests []promptkit.GenerateRequest sceneKind dnd.SceneKind sceneTitle string firstUnitID int thirdUnitID int sceneKinds []dnd.SceneKind combatSemanticsClassifications []string sceneCalls int combatSemanticsCalls int } func (client *groundedDNDLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { if err := ctx.Err(); err != nil { return nil, err } client.mu.Lock() defer client.mu.Unlock() client.requests = append(client.requests, request) var payload any firstUnitID := client.firstUnitID if firstUnitID == 0 { firstUnitID = 1 } thirdUnitID := client.thirdUnitID if thirdUnitID == 0 { thirdUnitID = 3 } switch { case nativeSchemaHasProperty(request, "npcs"): payload = map[string]any{"npcs": []any{ map[string]any{ "name": "Mira Thorn", "source_refs": []any{map[string]int{"start_unit_id": firstUnitID, "end_unit_id": firstUnitID}}, }, map[string]any{ "name": "Hooded Guard", "source_refs": []any{map[string]int{"start_unit_id": thirdUnitID, "end_unit_id": thirdUnitID}}, }, }} case nativeSchemaHasProperty(request, "duplicate_groups"): payload = map[string]any{"duplicate_groups": []any{}} case nativeSchemaHasProperty(request, "kind"): kind := client.sceneKind if client.sceneCalls < len(client.sceneKinds) { kind = client.sceneKinds[client.sceneCalls] } client.sceneCalls++ if kind == "" { kind = dnd.SceneKindCombat } title := client.sceneTitle if title == "" { title = "A combat encounter" } payload = map[string]any{"kind": kind, "title": title, "summary": "The party faces an active encounter."} case nativeSchemaHasProperty(request, "classification"): classification := "combat" if client.combatSemanticsCalls < len(client.combatSemanticsClassifications) { classification = client.combatSemanticsClassifications[client.combatSemanticsCalls] } client.combatSemanticsCalls++ payload = map[string]any{"classification": classification, "explanation": "The transcript shows an active encounter with hostile action."} case nativeSchemaHasProperty(request, "spell_casts"): payload = map[string]any{"spell_casts": []any{map[string]any{ "caster": "Mira Thorn", "spell": "Cure Wounds", "source_refs": []any{map[string]int{"start_unit_id": firstUnitID, "end_unit_id": firstUnitID}}, }}} case nativeSchemaHasProperty(request, "combat_turns"): payload = map[string]any{"combat_turns": []any{map[string]any{ "actor": "Mira Thorn", "turn_kind": "turn", "source_refs": []any{map[string]int{"start_unit_id": firstUnitID, "end_unit_id": firstUnitID}}, }}} default: return nil, fmt.Errorf("unexpected native grounded response contract") } content, err := json.Marshal(payload) if err != nil { return nil, err } return &promptkit.GenerateResponse{Content: string(content)}, nil } func (client *groundedDNDLLMClient) requestsSnapshot() []promptkit.GenerateRequest { client.mu.Lock() defer client.mu.Unlock() return append([]promptkit.GenerateRequest(nil), client.requests...) } var _ promptkit.LLMClient = (*groundedDNDLLMClient)(nil)