421 lines
17 KiB
Go
421 lines
17 KiB
Go
package integration_test
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"reflect"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/evidencecontext"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
|
combatcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/combatturns"
|
|
npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcregistry"
|
|
scenecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/scenedescriptions"
|
|
combatsemantics "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/combat_semantics"
|
|
"go.maximumdirect.net/promptkit"
|
|
)
|
|
|
|
func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T) {
|
|
registries := productionNPCRegistries(t)
|
|
catalog := moduleCatalog(registries)
|
|
configValue := loadGroundedPipelineConfig(t)
|
|
campaignInputs := configureGroundedCampaignReferences(t, &configValue)
|
|
effective, err := configValue.Resolve(config.ResolveInput{PipelineID: "dnd-npc-grounded", Catalog: catalog})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
materialized, warnings, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{})
|
|
if err != nil || len(warnings) != 0 {
|
|
t.Fatalf("materialize references: %v warnings=%#v", err, warnings)
|
|
}
|
|
client := &groundedDNDLLMClient{sceneTitle: "eligibility-only-scene-sentinel"}
|
|
prepared, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{Engine: nativeTestEngine(t, client)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{Prepared: prepared, RawInput: readNPCFixture(t), Workers: 1})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(output.OutputFiles) != 4 || output.FallbackCount != 0 {
|
|
t.Fatalf("output=%#v, want four accepted domain artifacts", output)
|
|
}
|
|
|
|
npcs, err := npccodec.New().Decode(publishedLane(t, output, "npc_registry").Bytes)
|
|
if err != nil || len(npcs.NPCs) != 2 {
|
|
t.Fatalf("NPC registry=%#v error=%v", npcs, err)
|
|
}
|
|
scenes, err := scenecodec.New().Decode(publishedLane(t, output, "scene-descriptions").Bytes)
|
|
if err != nil || len(scenes.Scenes) != 1 || scenes.Scenes[0].Kind != dnd.SceneKindCombat {
|
|
t.Fatalf("scene descriptions=%#v error=%v", scenes, err)
|
|
}
|
|
|
|
projection := `{"npcs":[{"name":"Mira Thorn"},{"name":"Hooded Guard"}]}`
|
|
seenConsumers := map[string]bool{}
|
|
for _, request := range client.requestsSnapshot() {
|
|
kind := "spell_casts"
|
|
if nativeSchemaHasProperty(request, "combat_turns") {
|
|
kind = "combat_turns"
|
|
} else if !nativeSchemaHasProperty(request, "spell_casts") {
|
|
continue
|
|
}
|
|
input, err := nativeJSONInput(request, "npcs")
|
|
if err != nil || string(input) != projection {
|
|
t.Fatalf("%s NPC projection=%s error=%v", kind, input, err)
|
|
}
|
|
text := nativePromptText(request)
|
|
for _, npc := range npcs.NPCs {
|
|
if strings.Contains(text, npc.ID) {
|
|
t.Fatalf("%s prompt contains opaque registry identity", kind)
|
|
}
|
|
}
|
|
if kind == "combat_turns" {
|
|
transcript, err := nativeJSONInput(request, "units")
|
|
if err != nil || !strings.Contains(string(transcript), "Mira Thorn asks the party") {
|
|
t.Fatalf("combat transcript=%s error=%v", transcript, err)
|
|
}
|
|
for name, want := range campaignInputs {
|
|
if !strings.Contains(text, want) {
|
|
t.Fatalf("combat prompt omitted %s reference", name)
|
|
}
|
|
}
|
|
for _, excluded := range []string{client.sceneTitle, "The party faces an active encounter."} {
|
|
if strings.Contains(text, excluded) {
|
|
t.Fatalf("combat prompt included eligibility-only scene prose %q", excluded)
|
|
}
|
|
}
|
|
}
|
|
seenConsumers[kind] = true
|
|
}
|
|
if !seenConsumers["spell_casts"] || !seenConsumers["combat_turns"] || combatCompletionCount(client.requestsSnapshot()) != 1 {
|
|
t.Fatalf("consumer native requests=%#v", seenConsumers)
|
|
}
|
|
spellsValue := decodeRunnerSpellResponse(t, publishedLane(t, output, "spells").Bytes)
|
|
if len(spellsValue.SpellCasts) != 1 || spellsValue.SpellCasts[0].Caster != "Mira Thorn" {
|
|
t.Fatalf("spells=%#v", spellsValue)
|
|
}
|
|
assertCurrentEvidence(t, spellsValue.SpellCasts[0].SourceRefs)
|
|
combatValue := publishedCombatOutput(t, output)
|
|
if len(combatValue.CombatTurns) != 1 || combatValue.CombatTurns[0].Actor != "Mira Thorn" {
|
|
t.Fatalf("combat=%#v", combatValue)
|
|
}
|
|
assertCurrentEvidence(t, combatValue.CombatTurns[0].SourceRefs)
|
|
}
|
|
|
|
func TestSceneCombatSemanticsCorrectionRevalidatesRevisedScene(t *testing.T) {
|
|
registries := productionNPCRegistries(t)
|
|
configValue := loadGroundedPipelineConfig(t)
|
|
profile := configValue.Pipelines["dnd-npc-grounded"]
|
|
sceneLane := profile.Steps[0].Artifacts["scene-descriptions"]
|
|
turns := 1
|
|
sceneLane.Extract.MaxAdditionalTurns = &turns
|
|
sceneLane.Extract.Validators = pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{
|
|
pipeline.Binding("generic/valid_json"),
|
|
pipeline.Binding("extract/dnd/scene-descriptions/shape"),
|
|
pipeline.Binding("extract/dnd/scene-descriptions/source_refs"),
|
|
pipeline.Binding("generic/valid_json_schema"),
|
|
pipeline.Binding("extract/dnd/scene-descriptions/source_relatedness"),
|
|
pipeline.Binding(combatsemantics.Key),
|
|
}}
|
|
profile.Steps[0].Artifacts["scene-descriptions"] = sceneLane
|
|
configValue.Pipelines["dnd-npc-grounded"] = profile
|
|
|
|
client := &groundedDNDLLMClient{sceneKinds: []dnd.SceneKind{dnd.SceneKindNarrative, dnd.SceneKindCombat}, combatSemanticsClassifications: []string{"combat", "combat"}}
|
|
output := runGroundedPipeline(t, configValue, registries, client)
|
|
if output.FallbackCount != 0 {
|
|
t.Fatalf("unexpected fallback: %#v", output)
|
|
}
|
|
for _, group := range output.Diagnostics.Groups {
|
|
if group.Origin.ValidatorKey == combatsemantics.Key {
|
|
t.Fatalf("combat-semantics validator left a diagnostic after correction: %#v", group)
|
|
}
|
|
}
|
|
scenes, err := scenecodec.New().Decode(publishedLane(t, output, "scene-descriptions").Bytes)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if len(scenes.Scenes) != 1 || scenes.Scenes[0].Kind != dnd.SceneKindCombat {
|
|
t.Fatalf("accepted scenes = %#v, want corrected combat scene", scenes)
|
|
}
|
|
requests := client.requestsSnapshot()
|
|
var sceneRequests, validatorRequests []promptkit.GenerateRequest
|
|
for _, request := range requests {
|
|
switch {
|
|
case nativeSchemaHasProperty(request, "kind"):
|
|
sceneRequests = append(sceneRequests, request)
|
|
case nativeSchemaHasProperty(request, "classification"):
|
|
validatorRequests = append(validatorRequests, request)
|
|
}
|
|
}
|
|
if len(sceneRequests) != 2 || len(validatorRequests) != 2 {
|
|
t.Fatalf("scene/validator calls=%d/%d, want correction and revalidation", len(sceneRequests), len(validatorRequests))
|
|
}
|
|
original := sceneRequests[0].Prompt.Messages
|
|
revised := sceneRequests[1].Prompt.Messages
|
|
if len(revised) != len(original)+2 || !reflect.DeepEqual(revised[:len(original)], original) {
|
|
t.Fatal("scene correction lost native conversation history")
|
|
}
|
|
assistant, guidance := revised[len(original)], revised[len(original)+1]
|
|
if assistant.Role != promptkit.RoleAssistant || !strings.Contains(assistant.Content, `"kind":"narrative"`) || guidance.Role != promptkit.RoleUser || !strings.Contains(guidance.Content, "Return kind: combat") || !strings.Contains(guidance.Content, "active encounter") {
|
|
t.Fatalf("native correction messages=%#v", revised[len(original):])
|
|
}
|
|
for _, forbidden := range []string{combatsemantics.Key, combatsemantics.ReasonCodeActiveCombatNotClassified, "session:chunk"} {
|
|
if strings.Contains(guidance.Content, forbidden) {
|
|
t.Fatalf("correction guidance contains diagnostic identifier %q", forbidden)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestProductionDNDOutputPublishesSelectedEvidenceContext(t *testing.T) {
|
|
registries := productionNPCRegistries(t)
|
|
configValue := loadGroundedPipelineConfig(t)
|
|
profile := configValue.Pipelines["dnd-npc-grounded"]
|
|
profile.Output.Options = map[string]any{"evidence_context": map[string]any{
|
|
"enabled": true, "window_units": 0, "lanes": []any{"npc_registry", "spells", "combat"},
|
|
}}
|
|
configValue.Pipelines["dnd-npc-grounded"] = profile
|
|
raw := strings.NewReplacer(
|
|
`"id": 1`, `"id": 10`,
|
|
`"id": 2`, `"id": 30`,
|
|
`"id": 3`, `"id": 20`,
|
|
`"id": 4`, `"id": 50`,
|
|
`"id": 5`, `"id": 40`,
|
|
).Replace(string(readNPCFixture(t)))
|
|
output := runGroundedPipelineWithRaw(t, configValue, registries, &groundedDNDLLMClient{firstUnitID: 10, thirdUnitID: 20}, []byte(raw))
|
|
|
|
value, err := evidencecontext.New().Decode(outputFileContent(t, output.OutputFiles, "evidence-context.json"))
|
|
if err != nil {
|
|
t.Fatalf("Decode(evidence context) error = %v", err)
|
|
}
|
|
if actual := evidenceUnitIDs(value); !reflect.DeepEqual(actual, []int{10, 20}) {
|
|
t.Fatalf("evidence units = %#v, want deduplicated source-position union without scene descriptions", actual)
|
|
}
|
|
}
|
|
|
|
func TestProductionDNDOutputCanExplicitlySelectSceneDescriptionEvidence(t *testing.T) {
|
|
registries := productionNPCRegistries(t)
|
|
configValue := loadGroundedPipelineConfig(t)
|
|
profile := configValue.Pipelines["dnd-npc-grounded"]
|
|
profile.Output.Options = map[string]any{"evidence_context": map[string]any{
|
|
"enabled": true, "lanes": []any{"scene-descriptions"},
|
|
}}
|
|
configValue.Pipelines["dnd-npc-grounded"] = profile
|
|
output := runGroundedPipeline(t, configValue, registries, &groundedDNDLLMClient{})
|
|
|
|
value, err := evidencecontext.New().Decode(outputFileContent(t, output.OutputFiles, "evidence-context.json"))
|
|
if err != nil {
|
|
t.Fatalf("Decode(evidence context) error = %v", err)
|
|
}
|
|
if len(value) == 0 {
|
|
t.Fatalf("evidence context = %#v, want explicitly selected scene-description source units", value)
|
|
}
|
|
}
|
|
|
|
func evidenceUnitIDs(value evidencecontext.Document) []int {
|
|
ids := make([]int, len(value))
|
|
for index, unit := range value {
|
|
ids[index] = unit.ID
|
|
}
|
|
return ids
|
|
}
|
|
|
|
func TestGroundedPipelineSkipsCombatForExactNarrativeScene(t *testing.T) {
|
|
registries := productionNPCRegistries(t)
|
|
configValue := loadGroundedPipelineConfig(t)
|
|
client := &groundedDNDLLMClient{sceneKind: dnd.SceneKindNarrative}
|
|
output := runGroundedPipeline(t, configValue, registries, client)
|
|
if combatCompletionCount(client.requestsSnapshot()) != 0 {
|
|
t.Fatalf("combat completion requests = %#v, want none for an exact narrative scene", client.requestsSnapshot())
|
|
}
|
|
if output.FallbackCount != 0 || hasCombatWarning(output.Diagnostics, "scene_classification_unavailable") {
|
|
t.Fatalf("narrative output=%#v, want accepted skip", output)
|
|
}
|
|
combatValue := publishedCombatOutput(t, output)
|
|
if combatValue.CombatTurns == nil || len(combatValue.CombatTurns) != 0 {
|
|
t.Fatalf("narrative combat output = %#v, want an accepted empty combat list", combatValue)
|
|
}
|
|
}
|
|
|
|
func assertCurrentEvidence(t *testing.T, references []source.SourceRef) {
|
|
t.Helper()
|
|
for _, reference := range references {
|
|
if reference.SourceID != "npc-session" {
|
|
t.Fatalf("evidence reference = %#v, want current source only", reference)
|
|
}
|
|
}
|
|
}
|
|
|
|
func loadGroundedPipelineConfig(t *testing.T) config.Config {
|
|
t.Helper()
|
|
return loadIntegrationConfig(t, repositoryPathForIntegration("internal", "modules", "integration", "testdata", "dnd_npc_grounded_pipeline.yml"))
|
|
}
|
|
|
|
func configureGroundedCampaignReferences(t *testing.T, cfg *config.Config) map[string]string {
|
|
t.Helper()
|
|
values := map[string]string{
|
|
"party": "Mira Thorn: ally\n",
|
|
"glossary": "Greencloak: Mira Thorn's title\n",
|
|
}
|
|
profile := cfg.Pipelines["dnd-npc-grounded"]
|
|
profile.References = make(map[string]pipeline.ReferenceSource, len(values))
|
|
for name, content := range values {
|
|
path := filepath.Join(t.TempDir(), name+".txt")
|
|
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
|
|
t.Fatalf("write %s reference: %v", name, err)
|
|
}
|
|
profile.References[name] = pipeline.ExternalReference(path)
|
|
}
|
|
cfg.Pipelines["dnd-npc-grounded"] = profile
|
|
return values
|
|
}
|
|
|
|
func runGroundedPipeline(t *testing.T, configValue config.Config, registries pipeline.Registries, client *groundedDNDLLMClient) pipeline.RunOutput {
|
|
return runGroundedPipelineWithRaw(t, configValue, registries, client, readNPCFixture(t))
|
|
}
|
|
|
|
func runGroundedPipelineWithRaw(t *testing.T, configValue config.Config, registries pipeline.Registries, client *groundedDNDLLMClient, raw []byte) pipeline.RunOutput {
|
|
t.Helper()
|
|
catalog := moduleCatalog(registries)
|
|
effective, err := configValue.Resolve(config.ResolveInput{PipelineID: "dnd-npc-grounded", Catalog: catalog})
|
|
if err != nil {
|
|
t.Fatalf("Resolve() error = %v", err)
|
|
}
|
|
materialized, warnings, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{})
|
|
if err != nil || len(warnings) != 0 {
|
|
t.Fatalf("MaterializeReferences() error = %v warnings = %#v", err, warnings)
|
|
}
|
|
prepared, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{Engine: nativeTestEngine(t, client)})
|
|
if err != nil {
|
|
t.Fatalf("Prepare() error = %v", err)
|
|
}
|
|
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
|
|
Prepared: prepared,
|
|
RawInput: append([]byte(nil), raw...),
|
|
Workers: 1,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("Run() error = %v", err)
|
|
}
|
|
return output
|
|
}
|
|
|
|
func publishedCombatOutput(t *testing.T, output pipeline.RunOutput) dnd.CombatTurnList {
|
|
t.Helper()
|
|
value, err := combatcodec.New().Decode(publishedLane(t, output, "combat").Bytes)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return value
|
|
}
|
|
|
|
func combatCompletionCount(requests []promptkit.GenerateRequest) int {
|
|
count := 0
|
|
for _, request := range requests {
|
|
if nativeSchemaHasProperty(request, "combat_turns") {
|
|
count++
|
|
}
|
|
}
|
|
return count
|
|
}
|
|
|
|
type groundedDNDLLMClient struct {
|
|
mu sync.Mutex
|
|
requests []promptkit.GenerateRequest
|
|
sceneKind dnd.SceneKind
|
|
sceneTitle string
|
|
firstUnitID int
|
|
thirdUnitID int
|
|
sceneKinds []dnd.SceneKind
|
|
combatSemanticsClassifications []string
|
|
sceneCalls int
|
|
combatSemanticsCalls int
|
|
}
|
|
|
|
func (client *groundedDNDLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
client.mu.Lock()
|
|
defer client.mu.Unlock()
|
|
client.requests = append(client.requests, request)
|
|
|
|
var payload any
|
|
firstUnitID := client.firstUnitID
|
|
if firstUnitID == 0 {
|
|
firstUnitID = 1
|
|
}
|
|
thirdUnitID := client.thirdUnitID
|
|
if thirdUnitID == 0 {
|
|
thirdUnitID = 3
|
|
}
|
|
switch {
|
|
case nativeSchemaHasProperty(request, "npcs"):
|
|
payload = map[string]any{"npcs": []any{
|
|
map[string]any{
|
|
"name": "Mira Thorn", "source_refs": []any{map[string]int{"start_unit_id": firstUnitID, "end_unit_id": firstUnitID}},
|
|
},
|
|
map[string]any{
|
|
"name": "Hooded Guard", "source_refs": []any{map[string]int{"start_unit_id": thirdUnitID, "end_unit_id": thirdUnitID}},
|
|
},
|
|
}}
|
|
case nativeSchemaHasProperty(request, "duplicate_groups"):
|
|
payload = map[string]any{"duplicate_groups": []any{}}
|
|
case nativeSchemaHasProperty(request, "kind"):
|
|
kind := client.sceneKind
|
|
if client.sceneCalls < len(client.sceneKinds) {
|
|
kind = client.sceneKinds[client.sceneCalls]
|
|
}
|
|
client.sceneCalls++
|
|
if kind == "" {
|
|
kind = dnd.SceneKindCombat
|
|
}
|
|
title := client.sceneTitle
|
|
if title == "" {
|
|
title = "A combat encounter"
|
|
}
|
|
payload = map[string]any{"kind": kind, "title": title, "summary": "The party faces an active encounter."}
|
|
case nativeSchemaHasProperty(request, "classification"):
|
|
classification := "combat"
|
|
if client.combatSemanticsCalls < len(client.combatSemanticsClassifications) {
|
|
classification = client.combatSemanticsClassifications[client.combatSemanticsCalls]
|
|
}
|
|
client.combatSemanticsCalls++
|
|
payload = map[string]any{"classification": classification, "explanation": "The transcript shows an active encounter with hostile action."}
|
|
case nativeSchemaHasProperty(request, "spell_casts"):
|
|
payload = map[string]any{"spell_casts": []any{map[string]any{
|
|
"caster": "Mira Thorn", "spell": "Cure Wounds",
|
|
"source_refs": []any{map[string]int{"start_unit_id": firstUnitID, "end_unit_id": firstUnitID}},
|
|
}}}
|
|
case nativeSchemaHasProperty(request, "combat_turns"):
|
|
payload = map[string]any{"combat_turns": []any{map[string]any{
|
|
"actor": "Mira Thorn",
|
|
"turn_kind": "turn",
|
|
"source_refs": []any{map[string]int{"start_unit_id": firstUnitID, "end_unit_id": firstUnitID}},
|
|
}}}
|
|
default:
|
|
return nil, fmt.Errorf("unexpected native grounded response contract")
|
|
}
|
|
content, err := json.Marshal(payload)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &promptkit.GenerateResponse{Content: string(content)}, nil
|
|
}
|
|
|
|
func (client *groundedDNDLLMClient) requestsSnapshot() []promptkit.GenerateRequest {
|
|
client.mu.Lock()
|
|
defer client.mu.Unlock()
|
|
return append([]promptkit.GenerateRequest(nil), client.requests...)
|
|
}
|
|
|
|
var _ promptkit.LLMClient = (*groundedDNDLLMClient)(nil)
|