2026-07-21 02:50:37 +00:00
|
|
|
package integration_test
|
|
|
|
|
|
|
|
|
|
import (
|
|
|
|
|
"context"
|
|
|
|
|
"encoding/json"
|
|
|
|
|
"fmt"
|
|
|
|
|
"os"
|
|
|
|
|
"strings"
|
2026-11-22 04:39:55 +00:00
|
|
|
"sync"
|
2026-07-21 02:50:37 +00:00
|
|
|
"testing"
|
|
|
|
|
|
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
2026-08-05 18:38:22 +00:00
|
|
|
npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcregistry"
|
|
|
|
|
npcnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/npcregistry"
|
2026-07-21 02:50:37 +00:00
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity"
|
2026-08-05 18:38:22 +00:00
|
|
|
npcshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/npcregistry/shape"
|
|
|
|
|
npcregistrysourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/npcregistry/source_refs"
|
2026-07-21 02:50:37 +00:00
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript"
|
2026-11-22 04:39:55 +00:00
|
|
|
"go.maximumdirect.net/promptkit"
|
2026-07-21 02:50:37 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
|
|
func TestRunnerProcessesSeriatimInputWithProductionDNDNPCPipeline(t *testing.T) {
|
|
|
|
|
raw := readNPCFixture(t)
|
|
|
|
|
doc, err := transcript.New().Parse(context.Background(), contracts.ParseRequest{Raw: raw})
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatalf("Parse() error = %v, want nil", err)
|
|
|
|
|
}
|
|
|
|
|
registries := productionNPCRegistries(t)
|
|
|
|
|
configValue := loadNPCPipelineConfig(t)
|
2026-08-05 18:38:22 +00:00
|
|
|
effective, err := configValue.Resolve(config.ResolveInput{PipelineID: "dnd-npc-registry-fixture", Catalog: moduleCatalog(registries)})
|
2026-07-21 02:50:37 +00:00
|
|
|
if err != nil {
|
|
|
|
|
t.Fatalf("Resolve() error = %v, want nil", err)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
client := &fakeNPCProductionLLMClient{response: npcProductionResponse{
|
|
|
|
|
NPCs: []npcProductionRecord{
|
|
|
|
|
{
|
2026-07-22 18:51:26 +00:00
|
|
|
Name: "Mira Thorn",
|
2026-07-21 02:50:37 +00:00
|
|
|
SourceRefs: []npcProductionSourceRef{{StartUnitID: 1, EndUnitID: 1}},
|
|
|
|
|
},
|
|
|
|
|
{
|
2026-07-22 18:51:26 +00:00
|
|
|
Name: " mira thorn ",
|
2026-07-21 02:50:37 +00:00
|
|
|
SourceRefs: []npcProductionSourceRef{{StartUnitID: 2, EndUnitID: 2}},
|
|
|
|
|
},
|
|
|
|
|
{
|
2026-07-22 18:51:26 +00:00
|
|
|
Name: "Hooded Guard",
|
2026-07-21 02:50:37 +00:00
|
|
|
SourceRefs: []npcProductionSourceRef{{StartUnitID: 3, EndUnitID: 3}},
|
|
|
|
|
},
|
|
|
|
|
}},
|
|
|
|
|
}
|
|
|
|
|
output, err := runPreparedPipeline(t, registries, effective.ResolvedPipeline, client, pipeline.RunInput{RawInput: raw})
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatalf("Run() error = %v, want nil", err)
|
|
|
|
|
}
|
2026-11-22 04:39:55 +00:00
|
|
|
if len(output.OutputFiles) != 1 {
|
|
|
|
|
t.Fatalf("output files = %#v, want NPC registry", output.OutputFiles)
|
2026-07-21 02:50:37 +00:00
|
|
|
}
|
2026-11-22 04:39:55 +00:00
|
|
|
file := publishedLane(t, output, "npc_registry")
|
|
|
|
|
value, err := npccodec.New().Decode(file.Bytes)
|
2026-07-21 02:50:37 +00:00
|
|
|
if err != nil {
|
2026-11-22 04:39:55 +00:00
|
|
|
t.Fatal(err)
|
2026-07-21 02:50:37 +00:00
|
|
|
}
|
2026-11-22 04:39:55 +00:00
|
|
|
|
2026-07-21 02:50:37 +00:00
|
|
|
if len(value.NPCs) != 2 {
|
|
|
|
|
t.Fatalf("NPC output = %#v, want repeated name consolidated and group/PC omitted", value.NPCs)
|
|
|
|
|
}
|
|
|
|
|
first, second := value.NPCs[0], value.NPCs[1]
|
2026-07-22 18:51:26 +00:00
|
|
|
if first.Name != "Mira Thorn" || len(first.SourceRefs) != 2 {
|
2026-07-21 02:50:37 +00:00
|
|
|
t.Fatalf("first NPC = %#v, want consolidated Mira identity", first)
|
|
|
|
|
}
|
|
|
|
|
if first.ID != identity.DeriveID(first.Name) || second.Name != "Hooded Guard" || second.ID != identity.DeriveID(second.Name) {
|
|
|
|
|
t.Fatalf("NPC IDs = %q/%q, want derived IDs", first.ID, second.ID)
|
|
|
|
|
}
|
|
|
|
|
for _, npc := range value.NPCs {
|
|
|
|
|
for _, ref := range npc.SourceRefs {
|
|
|
|
|
if ref.SourceID != doc.ID {
|
|
|
|
|
t.Fatalf("NPC source ref = %#v, want source document %q", ref, doc.ID)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
2026-08-27 16:40:17 +00:00
|
|
|
if !hasNPCWarning(output.Diagnostics, "duplicate_npc_collapsed") {
|
|
|
|
|
t.Fatalf("diagnostics = %#v, want name-only consolidation advisory", output.Diagnostics)
|
2026-07-21 02:50:37 +00:00
|
|
|
}
|
2026-11-22 04:39:55 +00:00
|
|
|
if file.ContentType != npccodec.MediaType {
|
|
|
|
|
t.Fatalf("NPC content type = %q, want %q", file.ContentType, npccodec.MediaType)
|
2026-07-21 02:50:37 +00:00
|
|
|
}
|
2026-11-22 04:39:55 +00:00
|
|
|
|
|
|
|
|
if len(client.requests) != 2 || client.extractions != 1 || client.normalizations != 1 {
|
|
|
|
|
t.Fatalf("native requests=%d extraction=%d normalization=%d", len(client.requests), client.extractions, client.normalizations)
|
2026-07-21 02:50:37 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-07-22 14:38:35 +00:00
|
|
|
func TestProductionNPCPipelineRoutesSemanticCandidatesToDeterministicValidators(t *testing.T) {
|
2026-07-22 14:30:20 +00:00
|
|
|
registries := productionNPCRegistries(t)
|
|
|
|
|
effective, err := loadNPCPipelineConfig(t).Resolve(config.ResolveInput{
|
2026-08-05 18:38:22 +00:00
|
|
|
PipelineID: "dnd-npc-registry-fixture",
|
2026-07-22 14:30:20 +00:00
|
|
|
Catalog: moduleCatalog(registries),
|
|
|
|
|
})
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatalf("Resolve() error = %v, want nil", err)
|
|
|
|
|
}
|
2026-07-22 14:38:35 +00:00
|
|
|
for _, test := range []struct {
|
|
|
|
|
name string
|
|
|
|
|
response []byte
|
|
|
|
|
validatorName string
|
|
|
|
|
}{
|
|
|
|
|
{
|
|
|
|
|
name: "blank string",
|
2026-07-22 18:51:26 +00:00
|
|
|
response: []byte(`{"npcs":[{"name":"","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`),
|
2026-07-22 14:38:35 +00:00
|
|
|
validatorName: npcshape.Key,
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
name: "empty evidence",
|
2026-07-22 18:51:26 +00:00
|
|
|
response: []byte(`{"npcs":[{"name":"Mira Thorn","source_refs":[]}]}`),
|
2026-07-22 14:38:35 +00:00
|
|
|
validatorName: npcshape.Key,
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
name: "nonpositive unit candidate",
|
2026-07-22 18:51:26 +00:00
|
|
|
response: []byte(`{"npcs":[{"name":"Mira Thorn","source_refs":[{"start_unit_id":0,"end_unit_id":1}]}]}`),
|
2026-08-05 18:38:22 +00:00
|
|
|
validatorName: npcregistrysourcerefs.Key,
|
2026-07-22 14:38:35 +00:00
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
name: "unknown unit candidate",
|
2026-07-22 18:51:26 +00:00
|
|
|
response: []byte(`{"npcs":[{"name":"Mira Thorn","source_refs":[{"start_unit_id":99,"end_unit_id":99}]}]}`),
|
2026-08-05 18:38:22 +00:00
|
|
|
validatorName: npcregistrysourcerefs.Key,
|
2026-07-22 14:38:35 +00:00
|
|
|
},
|
|
|
|
|
} {
|
|
|
|
|
t.Run(test.name, func(t *testing.T) {
|
|
|
|
|
client := &fakeNPCProductionLLMClient{rawResponses: [][]byte{test.response, test.response, test.response}}
|
2026-08-27 01:06:17 +00:00
|
|
|
resolved := effective.ResolvedPipeline
|
|
|
|
|
output, err := runPreparedPipeline(t, registries, resolved, client, pipeline.RunInput{RawInput: readNPCFixture(t)})
|
2026-11-22 04:39:55 +00:00
|
|
|
|
|
|
|
|
if err == nil || !strings.Contains(err.Error(), test.validatorName) {
|
|
|
|
|
t.Fatalf("Run() error = %v, want scoped %s rejection", err, test.validatorName)
|
2026-07-22 14:38:35 +00:00
|
|
|
}
|
2026-11-22 04:39:55 +00:00
|
|
|
if client.extractions != 3 || len(output.OutputFiles) != 0 {
|
|
|
|
|
t.Fatalf("native extraction calls=%d files=%#v, want exhausted rejection", client.extractions, output.OutputFiles)
|
2026-07-22 14:38:35 +00:00
|
|
|
}
|
|
|
|
|
})
|
2026-07-22 14:30:20 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-22 04:39:55 +00:00
|
|
|
func TestProductionNPCNormalizationUsesFinalSafeProposal(t *testing.T) {
|
2026-07-26 01:41:35 +00:00
|
|
|
registries := productionNPCRegistries(t)
|
|
|
|
|
baseResponse := npcProductionResponse{NPCs: []npcProductionRecord{
|
|
|
|
|
{Name: "Mira", SourceRefs: []npcProductionSourceRef{{StartUnitID: 1, EndUnitID: 1}}},
|
|
|
|
|
{Name: "Mira Thorn", SourceRefs: []npcProductionSourceRef{{StartUnitID: 2, EndUnitID: 2}}},
|
|
|
|
|
{Name: "Hooded Guard", SourceRefs: []npcProductionSourceRef{{StartUnitID: 3, EndUnitID: 3}}},
|
|
|
|
|
}}
|
2026-08-29 03:16:24 +00:00
|
|
|
partial := []byte(`{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":2},{"candidate_numbers":[3,99],"canonical_candidate_number":3}]}`)
|
|
|
|
|
safe := []byte(`{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":2}]}`)
|
2026-07-26 01:41:35 +00:00
|
|
|
|
|
|
|
|
for _, test := range []struct {
|
2026-11-22 04:39:55 +00:00
|
|
|
name string
|
|
|
|
|
turns int
|
|
|
|
|
mode contracts.ExecutionMode
|
|
|
|
|
responses [][]byte
|
|
|
|
|
wantCalls int
|
|
|
|
|
wantFallback bool
|
|
|
|
|
wantError bool
|
2026-07-26 01:41:35 +00:00
|
|
|
}{
|
2026-11-22 04:39:55 +00:00
|
|
|
{name: "permissive final safe groups", mode: contracts.ModePermissive, responses: [][]byte{partial}, wantCalls: 1, wantFallback: true},
|
|
|
|
|
{name: "later complete proposal", mode: contracts.ModeStrict, turns: 1, responses: [][]byte{partial, safe}, wantCalls: 2},
|
|
|
|
|
{name: "strict final rejection", mode: contracts.ModeStrict, responses: [][]byte{partial}, wantCalls: 1, wantError: true},
|
2026-07-26 01:41:35 +00:00
|
|
|
} {
|
|
|
|
|
t.Run(test.name, func(t *testing.T) {
|
|
|
|
|
cfg := loadNPCPipelineConfig(t)
|
2026-08-05 18:38:22 +00:00
|
|
|
profile := cfg.Pipelines["dnd-npc-registry-fixture"]
|
|
|
|
|
lane := profile.Artifacts["npc_registry"]
|
2026-11-22 04:39:55 +00:00
|
|
|
lane.Normalize.MaxAdditionalTurns = &test.turns
|
|
|
|
|
lane.Normalize.Mode = &test.mode
|
2026-08-05 18:38:22 +00:00
|
|
|
profile.Artifacts["npc_registry"] = lane
|
|
|
|
|
cfg.Pipelines["dnd-npc-registry-fixture"] = profile
|
|
|
|
|
effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-npc-registry-fixture", Catalog: moduleCatalog(registries)})
|
2026-07-26 01:41:35 +00:00
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
client := &fakeNPCProductionLLMClient{response: baseResponse, normalizeResponses: test.responses}
|
|
|
|
|
output, err := runPreparedPipeline(t, registries, effective.ResolvedPipeline, client, pipeline.RunInput{RawInput: readNPCFixture(t)})
|
2026-11-22 04:39:55 +00:00
|
|
|
if (err != nil) != test.wantError {
|
|
|
|
|
t.Fatalf("Run() error=%v, want error %t", err, test.wantError)
|
|
|
|
|
}
|
|
|
|
|
if client.normalizations != test.wantCalls {
|
|
|
|
|
t.Fatalf("normalization calls=%d, want %d", client.normalizations, test.wantCalls)
|
|
|
|
|
}
|
|
|
|
|
if test.wantError {
|
|
|
|
|
if len(output.OutputFiles) != 0 {
|
|
|
|
|
t.Fatal("strict failure published files")
|
|
|
|
|
}
|
|
|
|
|
return
|
2026-07-26 01:41:35 +00:00
|
|
|
}
|
2026-11-22 04:39:55 +00:00
|
|
|
wantFallbacks := 0
|
|
|
|
|
if test.wantFallback {
|
|
|
|
|
wantFallbacks = 1
|
2026-07-26 01:41:35 +00:00
|
|
|
}
|
2026-11-22 04:39:55 +00:00
|
|
|
if output.FallbackCount != wantFallbacks || hasWarningReason(output.Diagnostics, npcnormalize.ReasonCodeNPCSemanticReconciliationExhausted) != test.wantFallback {
|
|
|
|
|
t.Fatalf("outcome=%#v, want fallback %t", output, test.wantFallback)
|
|
|
|
|
}
|
|
|
|
|
value, err := npccodec.New().Decode(publishedLane(t, output, "npc_registry").Bytes)
|
|
|
|
|
if err != nil || len(value.NPCs) != 2 || value.NPCs[0].Name != "Mira Thorn" || len(value.NPCs[0].SourceRefs) != 2 || value.NPCs[1].Name != "Hooded Guard" {
|
|
|
|
|
t.Fatalf("final safe registry=%#v error=%v", value, err)
|
2026-07-26 01:41:35 +00:00
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-07-21 02:50:37 +00:00
|
|
|
type npcProductionResponse struct {
|
|
|
|
|
NPCs []npcProductionRecord `json:"npcs"`
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
type npcProductionRecord struct {
|
2026-07-22 18:51:26 +00:00
|
|
|
Name string `json:"name"`
|
|
|
|
|
SourceRefs []npcProductionSourceRef `json:"source_refs"`
|
2026-07-21 02:50:37 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
type npcProductionSourceRef struct {
|
|
|
|
|
StartUnitID int `json:"start_unit_id"`
|
|
|
|
|
EndUnitID int `json:"end_unit_id"`
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
type fakeNPCProductionLLMClient struct {
|
2026-07-26 01:41:35 +00:00
|
|
|
response npcProductionResponse
|
|
|
|
|
rawResponses [][]byte
|
|
|
|
|
normalizeResponses [][]byte
|
2026-11-22 04:39:55 +00:00
|
|
|
requests []promptkit.GenerateRequest
|
|
|
|
|
mu sync.Mutex
|
|
|
|
|
extractions int
|
|
|
|
|
normalizations int
|
2026-07-21 02:50:37 +00:00
|
|
|
}
|
|
|
|
|
|
2026-11-22 04:39:55 +00:00
|
|
|
func (client *fakeNPCProductionLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
|
|
|
client.mu.Lock()
|
|
|
|
|
defer client.mu.Unlock()
|
|
|
|
|
if err := ctx.Err(); err != nil {
|
|
|
|
|
return nil, err
|
2026-08-26 23:24:20 +00:00
|
|
|
}
|
2026-11-22 04:39:55 +00:00
|
|
|
client.requests = append(client.requests, request)
|
2026-07-22 14:38:35 +00:00
|
|
|
var content []byte
|
2026-11-22 04:39:55 +00:00
|
|
|
if nativeSchemaHasProperty(request, "duplicate_groups") {
|
|
|
|
|
index := client.normalizations
|
|
|
|
|
client.normalizations++
|
|
|
|
|
content = []byte(`{"duplicate_groups":[]}`)
|
|
|
|
|
if client.normalizeResponses != nil {
|
|
|
|
|
if index >= len(client.normalizeResponses) {
|
|
|
|
|
return nil, fmt.Errorf("missing NPC normalization response %d", index)
|
|
|
|
|
}
|
|
|
|
|
content = client.normalizeResponses[index]
|
|
|
|
|
}
|
|
|
|
|
} else if nativeSchemaHasProperty(request, "npcs") {
|
|
|
|
|
index := client.extractions
|
|
|
|
|
client.extractions++
|
2026-07-26 01:41:35 +00:00
|
|
|
if client.rawResponses != nil {
|
|
|
|
|
if index >= len(client.rawResponses) {
|
2026-11-22 04:39:55 +00:00
|
|
|
return nil, fmt.Errorf("missing NPC extraction response %d", index)
|
2026-07-26 01:41:35 +00:00
|
|
|
}
|
2026-11-22 04:39:55 +00:00
|
|
|
content = client.rawResponses[index]
|
2026-07-26 01:41:35 +00:00
|
|
|
} else {
|
|
|
|
|
var err error
|
|
|
|
|
content, err = json.Marshal(client.response)
|
|
|
|
|
if err != nil {
|
2026-11-22 04:39:55 +00:00
|
|
|
return nil, err
|
2026-07-26 01:41:35 +00:00
|
|
|
}
|
2026-07-22 14:38:35 +00:00
|
|
|
}
|
2026-11-22 04:39:55 +00:00
|
|
|
} else {
|
|
|
|
|
return nil, fmt.Errorf("unexpected NPC provider output contract")
|
2026-07-21 02:50:37 +00:00
|
|
|
}
|
2026-11-22 04:39:55 +00:00
|
|
|
return &promptkit.GenerateResponse{Content: string(content)}, nil
|
2026-07-21 02:50:37 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func loadNPCPipelineConfig(t *testing.T) config.Config {
|
|
|
|
|
t.Helper()
|
2026-09-13 17:23:00 +00:00
|
|
|
return loadIntegrationConfig(t, "testdata/dnd_npc_registry_pipeline.yml")
|
2026-07-21 02:50:37 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func readNPCFixture(t *testing.T) []byte {
|
|
|
|
|
t.Helper()
|
|
|
|
|
raw, err := os.ReadFile("testdata/seriatim_npc_session.json")
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatalf("ReadFile(seriatim_npc_session.json) error = %v", err)
|
|
|
|
|
}
|
|
|
|
|
return raw
|
|
|
|
|
}
|
|
|
|
|
|
2026-08-27 16:40:17 +00:00
|
|
|
func hasNPCWarning(diagnostics contracts.DiagnosticCollection, reason string) bool {
|
|
|
|
|
for _, group := range diagnostics.Groups {
|
|
|
|
|
if group.ReasonCode == reason && len(group.Samples) > 0 && strings.HasPrefix(group.Samples[0].Scope, "npcs[") {
|
2026-07-21 02:50:37 +00:00
|
|
|
return true
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return false
|
|
|
|
|
}
|
2026-07-26 01:41:35 +00:00
|
|
|
|
2026-08-27 16:40:17 +00:00
|
|
|
func hasWarningReason(diagnostics contracts.DiagnosticCollection, reason string) bool {
|
|
|
|
|
for _, group := range diagnostics.Groups {
|
|
|
|
|
if group.ReasonCode == reason {
|
2026-07-26 01:41:35 +00:00
|
|
|
return true
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return false
|
|
|
|
|
}
|