Files
notarius/internal/modules/integration/dnd_npc_registry_runner_test.go

303 lines
11 KiB
Go
Raw Normal View History

package integration_test
import (
"context"
"encoding/json"
"fmt"
"os"
"strings"
"sync"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcregistry"
npcnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/npcregistry"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity"
npcshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/npcregistry/shape"
npcregistrysourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/npcregistry/source_refs"
"gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript"
"go.maximumdirect.net/promptkit"
)
func TestRunnerProcessesSeriatimInputWithProductionDNDNPCPipeline(t *testing.T) {
raw := readNPCFixture(t)
doc, err := transcript.New().Parse(context.Background(), contracts.ParseRequest{Raw: raw})
if err != nil {
t.Fatalf("Parse() error = %v, want nil", err)
}
registries := productionNPCRegistries(t)
configValue := loadNPCPipelineConfig(t)
effective, err := configValue.Resolve(config.ResolveInput{PipelineID: "dnd-npc-registry-fixture", Catalog: moduleCatalog(registries)})
if err != nil {
t.Fatalf("Resolve() error = %v, want nil", err)
}
client := &fakeNPCProductionLLMClient{response: npcProductionResponse{
NPCs: []npcProductionRecord{
{
2026-07-22 18:51:26 +00:00
Name: "Mira Thorn",
SourceRefs: []npcProductionSourceRef{{StartUnitID: 1, EndUnitID: 1}},
},
{
2026-07-22 18:51:26 +00:00
Name: " mira thorn ",
SourceRefs: []npcProductionSourceRef{{StartUnitID: 2, EndUnitID: 2}},
},
{
2026-07-22 18:51:26 +00:00
Name: "Hooded Guard",
SourceRefs: []npcProductionSourceRef{{StartUnitID: 3, EndUnitID: 3}},
},
}},
}
output, err := runPreparedPipeline(t, registries, effective.ResolvedPipeline, client, pipeline.RunInput{RawInput: raw})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.OutputFiles) != 1 {
t.Fatalf("output files = %#v, want NPC registry", output.OutputFiles)
}
file := publishedLane(t, output, "npc_registry")
value, err := npccodec.New().Decode(file.Bytes)
if err != nil {
t.Fatal(err)
}
if len(value.NPCs) != 2 {
t.Fatalf("NPC output = %#v, want repeated name consolidated and group/PC omitted", value.NPCs)
}
first, second := value.NPCs[0], value.NPCs[1]
2026-07-22 18:51:26 +00:00
if first.Name != "Mira Thorn" || len(first.SourceRefs) != 2 {
t.Fatalf("first NPC = %#v, want consolidated Mira identity", first)
}
if first.ID != identity.DeriveID(first.Name) || second.Name != "Hooded Guard" || second.ID != identity.DeriveID(second.Name) {
t.Fatalf("NPC IDs = %q/%q, want derived IDs", first.ID, second.ID)
}
for _, npc := range value.NPCs {
for _, ref := range npc.SourceRefs {
if ref.SourceID != doc.ID {
t.Fatalf("NPC source ref = %#v, want source document %q", ref, doc.ID)
}
}
}
if !hasNPCWarning(output.Diagnostics, "duplicate_npc_collapsed") {
t.Fatalf("diagnostics = %#v, want name-only consolidation advisory", output.Diagnostics)
}
if file.ContentType != npccodec.MediaType {
t.Fatalf("NPC content type = %q, want %q", file.ContentType, npccodec.MediaType)
}
if len(client.requests) != 2 || client.extractions != 1 || client.normalizations != 1 {
t.Fatalf("native requests=%d extraction=%d normalization=%d", len(client.requests), client.extractions, client.normalizations)
}
}
2026-07-22 14:38:35 +00:00
func TestProductionNPCPipelineRoutesSemanticCandidatesToDeterministicValidators(t *testing.T) {
2026-07-22 14:30:20 +00:00
registries := productionNPCRegistries(t)
effective, err := loadNPCPipelineConfig(t).Resolve(config.ResolveInput{
PipelineID: "dnd-npc-registry-fixture",
2026-07-22 14:30:20 +00:00
Catalog: moduleCatalog(registries),
})
if err != nil {
t.Fatalf("Resolve() error = %v, want nil", err)
}
2026-07-22 14:38:35 +00:00
for _, test := range []struct {
name string
response []byte
validatorName string
}{
{
name: "blank string",
2026-07-22 18:51:26 +00:00
response: []byte(`{"npcs":[{"name":"","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`),
2026-07-22 14:38:35 +00:00
validatorName: npcshape.Key,
},
{
name: "empty evidence",
2026-07-22 18:51:26 +00:00
response: []byte(`{"npcs":[{"name":"Mira Thorn","source_refs":[]}]}`),
2026-07-22 14:38:35 +00:00
validatorName: npcshape.Key,
},
{
name: "nonpositive unit candidate",
2026-07-22 18:51:26 +00:00
response: []byte(`{"npcs":[{"name":"Mira Thorn","source_refs":[{"start_unit_id":0,"end_unit_id":1}]}]}`),
validatorName: npcregistrysourcerefs.Key,
2026-07-22 14:38:35 +00:00
},
{
name: "unknown unit candidate",
2026-07-22 18:51:26 +00:00
response: []byte(`{"npcs":[{"name":"Mira Thorn","source_refs":[{"start_unit_id":99,"end_unit_id":99}]}]}`),
validatorName: npcregistrysourcerefs.Key,
2026-07-22 14:38:35 +00:00
},
} {
t.Run(test.name, func(t *testing.T) {
client := &fakeNPCProductionLLMClient{rawResponses: [][]byte{test.response, test.response, test.response}}
resolved := effective.ResolvedPipeline
output, err := runPreparedPipeline(t, registries, resolved, client, pipeline.RunInput{RawInput: readNPCFixture(t)})
if err == nil || !strings.Contains(err.Error(), test.validatorName) {
t.Fatalf("Run() error = %v, want scoped %s rejection", err, test.validatorName)
2026-07-22 14:38:35 +00:00
}
if client.extractions != 3 || len(output.OutputFiles) != 0 {
t.Fatalf("native extraction calls=%d files=%#v, want exhausted rejection", client.extractions, output.OutputFiles)
2026-07-22 14:38:35 +00:00
}
})
2026-07-22 14:30:20 +00:00
}
}
func TestProductionNPCNormalizationUsesFinalSafeProposal(t *testing.T) {
2026-07-26 01:41:35 +00:00
registries := productionNPCRegistries(t)
baseResponse := npcProductionResponse{NPCs: []npcProductionRecord{
{Name: "Mira", SourceRefs: []npcProductionSourceRef{{StartUnitID: 1, EndUnitID: 1}}},
{Name: "Mira Thorn", SourceRefs: []npcProductionSourceRef{{StartUnitID: 2, EndUnitID: 2}}},
{Name: "Hooded Guard", SourceRefs: []npcProductionSourceRef{{StartUnitID: 3, EndUnitID: 3}}},
}}
partial := []byte(`{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":2},{"candidate_numbers":[3,99],"canonical_candidate_number":3}]}`)
safe := []byte(`{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":2}]}`)
2026-07-26 01:41:35 +00:00
for _, test := range []struct {
name string
turns int
mode contracts.ExecutionMode
responses [][]byte
wantCalls int
wantFallback bool
wantError bool
2026-07-26 01:41:35 +00:00
}{
{name: "permissive final safe groups", mode: contracts.ModePermissive, responses: [][]byte{partial}, wantCalls: 1, wantFallback: true},
{name: "later complete proposal", mode: contracts.ModeStrict, turns: 1, responses: [][]byte{partial, safe}, wantCalls: 2},
{name: "strict final rejection", mode: contracts.ModeStrict, responses: [][]byte{partial}, wantCalls: 1, wantError: true},
2026-07-26 01:41:35 +00:00
} {
t.Run(test.name, func(t *testing.T) {
cfg := loadNPCPipelineConfig(t)
profile := cfg.Pipelines["dnd-npc-registry-fixture"]
lane := profile.Artifacts["npc_registry"]
lane.Normalize.MaxAdditionalTurns = &test.turns
lane.Normalize.Mode = &test.mode
profile.Artifacts["npc_registry"] = lane
cfg.Pipelines["dnd-npc-registry-fixture"] = profile
effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-npc-registry-fixture", Catalog: moduleCatalog(registries)})
2026-07-26 01:41:35 +00:00
if err != nil {
t.Fatal(err)
}
client := &fakeNPCProductionLLMClient{response: baseResponse, normalizeResponses: test.responses}
output, err := runPreparedPipeline(t, registries, effective.ResolvedPipeline, client, pipeline.RunInput{RawInput: readNPCFixture(t)})
if (err != nil) != test.wantError {
t.Fatalf("Run() error=%v, want error %t", err, test.wantError)
}
if client.normalizations != test.wantCalls {
t.Fatalf("normalization calls=%d, want %d", client.normalizations, test.wantCalls)
}
if test.wantError {
if len(output.OutputFiles) != 0 {
t.Fatal("strict failure published files")
}
return
2026-07-26 01:41:35 +00:00
}
wantFallbacks := 0
if test.wantFallback {
wantFallbacks = 1
2026-07-26 01:41:35 +00:00
}
if output.FallbackCount != wantFallbacks || hasWarningReason(output.Diagnostics, npcnormalize.ReasonCodeNPCSemanticReconciliationExhausted) != test.wantFallback {
t.Fatalf("outcome=%#v, want fallback %t", output, test.wantFallback)
}
value, err := npccodec.New().Decode(publishedLane(t, output, "npc_registry").Bytes)
if err != nil || len(value.NPCs) != 2 || value.NPCs[0].Name != "Mira Thorn" || len(value.NPCs[0].SourceRefs) != 2 || value.NPCs[1].Name != "Hooded Guard" {
t.Fatalf("final safe registry=%#v error=%v", value, err)
2026-07-26 01:41:35 +00:00
}
})
}
}
type npcProductionResponse struct {
NPCs []npcProductionRecord `json:"npcs"`
}
type npcProductionRecord struct {
2026-07-22 18:51:26 +00:00
Name string `json:"name"`
SourceRefs []npcProductionSourceRef `json:"source_refs"`
}
type npcProductionSourceRef struct {
StartUnitID int `json:"start_unit_id"`
EndUnitID int `json:"end_unit_id"`
}
type fakeNPCProductionLLMClient struct {
2026-07-26 01:41:35 +00:00
response npcProductionResponse
rawResponses [][]byte
normalizeResponses [][]byte
requests []promptkit.GenerateRequest
mu sync.Mutex
extractions int
normalizations int
}
func (client *fakeNPCProductionLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
client.mu.Lock()
defer client.mu.Unlock()
if err := ctx.Err(); err != nil {
return nil, err
}
client.requests = append(client.requests, request)
2026-07-22 14:38:35 +00:00
var content []byte
if nativeSchemaHasProperty(request, "duplicate_groups") {
index := client.normalizations
client.normalizations++
content = []byte(`{"duplicate_groups":[]}`)
if client.normalizeResponses != nil {
if index >= len(client.normalizeResponses) {
return nil, fmt.Errorf("missing NPC normalization response %d", index)
}
content = client.normalizeResponses[index]
}
} else if nativeSchemaHasProperty(request, "npcs") {
index := client.extractions
client.extractions++
2026-07-26 01:41:35 +00:00
if client.rawResponses != nil {
if index >= len(client.rawResponses) {
return nil, fmt.Errorf("missing NPC extraction response %d", index)
2026-07-26 01:41:35 +00:00
}
content = client.rawResponses[index]
2026-07-26 01:41:35 +00:00
} else {
var err error
content, err = json.Marshal(client.response)
if err != nil {
return nil, err
2026-07-26 01:41:35 +00:00
}
2026-07-22 14:38:35 +00:00
}
} else {
return nil, fmt.Errorf("unexpected NPC provider output contract")
}
return &promptkit.GenerateResponse{Content: string(content)}, nil
}
func loadNPCPipelineConfig(t *testing.T) config.Config {
t.Helper()
2026-09-13 17:23:00 +00:00
return loadIntegrationConfig(t, "testdata/dnd_npc_registry_pipeline.yml")
}
func readNPCFixture(t *testing.T) []byte {
t.Helper()
raw, err := os.ReadFile("testdata/seriatim_npc_session.json")
if err != nil {
t.Fatalf("ReadFile(seriatim_npc_session.json) error = %v", err)
}
return raw
}
func hasNPCWarning(diagnostics contracts.DiagnosticCollection, reason string) bool {
for _, group := range diagnostics.Groups {
if group.ReasonCode == reason && len(group.Samples) > 0 && strings.HasPrefix(group.Samples[0].Scope, "npcs[") {
return true
}
}
return false
}
2026-07-26 01:41:35 +00:00
func hasWarningReason(diagnostics contracts.DiagnosticCollection, reason string) bool {
for _, group := range diagnostics.Groups {
if group.ReasonCode == reason {
2026-07-26 01:41:35 +00:00
return true
}
}
return false
}