Files
notarius/internal/modules/dnd/extract/combatturns/extractor_test.go

383 lines
17 KiB
Go

package combatturns
import (
"context"
"errors"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcregistry"
scenecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/scenedescriptions"
)
func TestExtractMapsAndOrdersCombatTurnsBySourcePosition(t *testing.T) {
client := &fakeCombatTurnsLLMClient{response: extractionResponse{CombatTurns: []combatTurnResponse{
{
Actor: "Borin", TurnKind: "turn",
SourceRefs: []combatSourceRefResponse{{StartUnitID: 2, EndUnitID: 2}},
},
{
Actor: "Aria", TurnKind: "reaction", SourceRefs: []combatSourceRefResponse{
{StartUnitID: 2, EndUnitID: 10},
{StartUnitID: 2, EndUnitID: 10},
},
},
{
Actor: "Unknown", TurnKind: "other",
SourceRefs: []combatSourceRefResponse{{StartUnitID: 0, EndUnitID: 0}},
},
}}}
result, err := newExtractor(t, client).Extract(context.Background(), extractionRequest(t))
if err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
}
if got := []string{result.Value.CombatTurns[0].Actor, result.Value.CombatTurns[1].Actor, result.Value.CombatTurns[2].Actor}; !reflect.DeepEqual(got, []string{"Aria", "Borin", "Unknown"}) {
t.Fatalf("actor order = %#v, want source-position order with invalid evidence last", got)
}
wantRefs := []source.SourceRef{{SourceID: "session-alpha", StartUnitID: 10, EndUnitID: 2}}
if !reflect.DeepEqual(result.Value.CombatTurns[0].SourceRefs, wantRefs) {
t.Fatalf("canonical refs = %#v, want %#v", result.Value.CombatTurns[0].SourceRefs, wantRefs)
}
if ref := result.Value.CombatTurns[2].SourceRefs[0]; ref != (source.SourceRef{SourceID: "session-alpha"}) {
t.Fatalf("invalid evidence = %#v, want source identity and invalid range preserved", ref)
}
if len(client.requests) != 1 {
t.Fatalf("LLM calls = %d, want one call per chunk", len(client.requests))
}
request := client.requests[0]
if request.Prompt.SessionID != "session-123" {
t.Fatal("session selection lost")
}
if !reflect.DeepEqual(renderedInput(t, request, "transcript"), extractionRequest(t).Chunk.Content) {
t.Fatal("chunk-scoped transcript changed")
}
}
func TestExtractPreservesInvalidCandidatesForValidators(t *testing.T) {
client := &fakeCombatTurnsLLMClient{response: extractionResponse{CombatTurns: []combatTurnResponse{
{
Actor: " ", TurnKind: "turn",
SourceRefs: []combatSourceRefResponse{{StartUnitID: 99, EndUnitID: 0}},
},
}}}
result, err := newExtractor(t, client).Extract(context.Background(), extractionRequest(t))
if err != nil {
t.Fatalf("Extract() error = %v, want nil for candidate values", err)
}
turn := result.Value.CombatTurns[0]
if turn.Actor != " " || turn.TurnKind != "turn" {
t.Fatalf("invalid turn fields = %#v, want preserved candidate values", turn)
}
if turn.SourceRefs[0] != (source.SourceRef{SourceID: "session-alpha", StartUnitID: 99}) {
t.Fatalf("invalid source ref = %#v, want invalid range preserved", turn.SourceRefs[0])
}
}
func TestExtractUsesDocumentOrderForReferencesAndTurns(t *testing.T) {
client := &fakeCombatTurnsLLMClient{response: extractionResponse{CombatTurns: []combatTurnResponse{
{Actor: "Later", TurnKind: "turn", SourceRefs: []combatSourceRefResponse{{StartUnitID: 10, EndUnitID: 10}}},
{Actor: "First", TurnKind: "reaction", SourceRefs: []combatSourceRefResponse{
{StartUnitID: 10, EndUnitID: 10},
{StartUnitID: 30, EndUnitID: 30},
{StartUnitID: 30, EndUnitID: 30},
{StartUnitID: 999, EndUnitID: 0},
}},
{Actor: "Second", TurnKind: "other", SourceRefs: []combatSourceRefResponse{{StartUnitID: 30, EndUnitID: 30}}},
}}}
req := extractionRequest(t)
req.Source.Units = []source.SourceUnit{{ID: 30}, {ID: 10}}
req.Chunk.Units = append([]source.SourceUnit(nil), req.Source.Units...)
req.Chunk.Ref = source.SourceRef{SourceID: req.Source.ID, StartUnitID: 10, EndUnitID: 30}
req.References = sceneReferences(t, req.Chunk, dnd.SceneKindCombat)
result, err := newExtractor(t, client).Extract(context.Background(), req)
if err != nil {
t.Fatalf("Extract() error = %v", err)
}
if got := []string{result.Value.CombatTurns[0].Actor, result.Value.CombatTurns[1].Actor, result.Value.CombatTurns[2].Actor}; !reflect.DeepEqual(got, []string{"First", "Second", "Later"}) {
t.Fatalf("turn order = %#v, want document chronology with stable equal-evidence ties", got)
}
refs := result.Value.CombatTurns[0].SourceRefs
if got := []int{refs[0].StartUnitID, refs[1].StartUnitID, refs[2].StartUnitID}; !reflect.DeepEqual(got, []int{30, 10, 999}) {
t.Fatalf("source refs = %#v, want document order with exact duplicate removed", refs)
}
refs[0].StartUnitID = 777
for _, turn := range client.response.CombatTurns {
for _, ref := range turn.SourceRefs {
if ref.StartUnitID == 777 {
t.Fatal("result source references alias the model response")
}
}
}
}
func TestExtractPassesReferencesAndNPCGroundingWithoutUsingItAsEvidence(t *testing.T) {
client := &fakeCombatTurnsLLMClient{response: extractionResponse{CombatTurns: []combatTurnResponse{}}}
references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
"players": {Slot: contracts.ReferenceSlot{Name: "players"}, Items: []contracts.ReferenceItem{{SlotName: "players", Content: []byte("Alice: Aria")}}},
"party": {Slot: contracts.ReferenceSlot{Name: "party"}, Items: []contracts.ReferenceItem{{SlotName: "party", Content: []byte("Aria: cleric")}}},
"glossary": {Slot: contracts.ReferenceSlot{Name: "glossary"}, Items: []contracts.ReferenceItem{{SlotName: "glossary", Content: []byte("ogre: a large foe")}}},
NPCRegistryReferenceSlot: {Slot: contracts.ReferenceSlot{Name: NPCRegistryReferenceSlot}, Items: []contracts.ReferenceItem{{SlotName: NPCRegistryReferenceSlot, MediaType: "application/json", Content: npcRegistryJSON(t)}}},
}}
req := extractionRequest(t)
for name, slot := range req.References.Slots {
references.Slots[name] = slot
}
req.References = references
if _, err := newExtractor(t, client, references).Extract(context.Background(), req); err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
}
messages := client.requests[0].Prompt.Messages
var all string
for _, message := range messages {
all += message.Content
}
for _, material := range []string{"Alice: Aria", "Aria: cleric", "ogre: a large foe"} {
if strings.Count(all, material) != 1 {
t.Fatalf("campaign input not rendered once: %q", material)
}
}
input := renderedInput(t, client.requests[0], NPCRegistryReferenceSlot)
if string(input) != `{"npcs":[{"name":"Mira Thorn"}]}` {
t.Fatalf("NPC grounding=%s", input)
}
if strings.Contains(string(renderedInput(t, client.requests[0], "transcript")), "Aria: cleric") {
t.Fatal("campaign context became evidence")
}
if strings.Contains(all, "Combat eligibility") || strings.Contains(all, "A valid scene description") {
t.Fatal("eligibility prose became model input")
}
}
func TestExtractUnboundRegistryUsesExactEmptyPromptAndOmitsIdentity(t *testing.T) {
client := &fakeCombatTurnsLLMClient{response: extractionResponse{CombatTurns: []combatTurnResponse{}}}
extractor := newExtractor(t, client)
if _, err := extractor.Extract(context.Background(), extractionRequest(t)); err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
}
if input := renderedInput(t, client.requests[0], NPCRegistryReferenceSlot); string(input) != `{"npcs":[]}` {
t.Fatalf("unbound registry=%s", input)
}
}
func TestExtractorResolvesOperationNPCOverrideWithoutSingletonMetadata(t *testing.T) {
content := npcRegistryJSON(t)
client := &fakeCombatTurnsLLMClient{response: extractionResponse{CombatTurns: []combatTurnResponse{}}}
extractor := newExtractor(t, client)
request := extractionRequest(t)
request.References.Slots[NPCRegistryReferenceSlot] = contracts.ResolvedReferenceSlot{
Slot: contracts.ReferenceSlot{Name: NPCRegistryReferenceSlot},
Items: []contracts.ReferenceItem{{SlotName: NPCRegistryReferenceSlot, MediaType: npccodec.MediaType, Content: append([]byte(nil), content...), Origin: contracts.ReferenceOrigin{Type: "generated"}}},
}
if _, err := extractor.Extract(context.Background(), request); err != nil {
t.Fatalf("Extract() error = %v", err)
}
if input := renderedInput(t, client.requests[0], NPCRegistryReferenceSlot); string(input) != `{"npcs":[{"name":"Mira Thorn"}]}` {
t.Fatalf("generated grounding=%s", input)
}
}
func TestExtractAppliesSceneEligibilityBeforePromptConstruction(t *testing.T) {
for _, test := range []struct {
name string
scenes []dnd.SceneDescription
wantWarning string
wantError string
wantCalls int
wantEmptyList bool
}{
{
name: "exact combat",
scenes: []dnd.SceneDescription{sceneDescription(extractionRequest(t).Chunk, dnd.SceneKindCombat)},
wantCalls: 1,
},
{
name: "exact narrative",
scenes: []dnd.SceneDescription{sceneDescription(extractionRequest(t).Chunk, dnd.SceneKindNarrative)},
wantEmptyList: true,
},
{
name: "exact recap",
scenes: []dnd.SceneDescription{sceneDescription(extractionRequest(t).Chunk, dnd.SceneKindRecap)},
wantEmptyList: true,
},
{
name: "exact meta",
scenes: []dnd.SceneDescription{sceneDescription(extractionRequest(t).Chunk, dnd.SceneKindMeta)},
wantEmptyList: true,
},
{
name: "missing ID",
scenes: []dnd.SceneDescription{func() dnd.SceneDescription {
value := sceneDescription(extractionRequest(t).Chunk, dnd.SceneKindCombat)
value.ID = "other-chunk"
return value
}()},
wantWarning: "scene_classification_unavailable",
wantEmptyList: true,
},
{
name: "source mismatch",
scenes: []dnd.SceneDescription{func() dnd.SceneDescription {
value := sceneDescription(extractionRequest(t).Chunk, dnd.SceneKindCombat)
value.SourceRef.SourceID = "other-source"
return value
}()},
wantWarning: "scene_classification_unavailable",
wantEmptyList: true,
},
{
name: "start mismatch",
scenes: []dnd.SceneDescription{func() dnd.SceneDescription {
value := sceneDescription(extractionRequest(t).Chunk, dnd.SceneKindCombat)
value.SourceRef.StartUnitID = 3
return value
}()},
wantWarning: "scene_classification_unavailable",
wantEmptyList: true,
},
{
name: "end mismatch",
scenes: []dnd.SceneDescription{func() dnd.SceneDescription {
value := sceneDescription(extractionRequest(t).Chunk, dnd.SceneKindCombat)
value.SourceRef.EndUnitID = 9
return value
}()},
wantWarning: "scene_classification_unavailable",
wantEmptyList: true,
},
{
name: "missing required reference",
wantError: "required scene descriptions reference is missing",
},
} {
t.Run(test.name, func(t *testing.T) {
client := &fakeCombatTurnsLLMClient{response: extractionResponse{CombatTurns: []combatTurnResponse{}}}
req := extractionRequest(t)
if test.wantError != "" {
req.References = contracts.ReferenceSet{}
} else {
req.References = sceneListReferences(t, test.scenes...)
}
result, err := newExtractor(t, client).Extract(context.Background(), req)
if test.wantError != "" {
if err == nil || !strings.Contains(err.Error(), test.wantError) {
t.Fatalf("Extract() error = %v, want %q", err, test.wantError)
}
} else if err != nil {
t.Fatalf("Extract() error = %v", err)
}
if len(client.requests) != test.wantCalls {
t.Fatalf("LLM calls = %d, want %d", len(client.requests), test.wantCalls)
}
if test.wantEmptyList && result.Value.CombatTurns == nil {
t.Fatal("CombatTurns = nil, want accepted non-nil empty list")
}
if test.wantWarning != "" {
if len(result.Diagnostics) != 1 || result.Diagnostics[0].Samples[0].Scope != SceneDescriptionReferenceSlot || result.Diagnostics[0].ReasonCode != test.wantWarning || result.Diagnostics[0].Disposition != contracts.DiagnosticDispositionWarning || result.Diagnostics[0].Category != contracts.DiagnosticCategoryDegradation {
t.Fatalf("diagnostics = %#v", result.Diagnostics)
}
} else if len(result.Diagnostics) != 0 {
t.Fatalf("diagnostics = %#v, want none", result.Diagnostics)
}
})
}
}
func TestNewRejectsMalformedNPCRegistryBeforeLLMCallWithoutContent(t *testing.T) {
client := &fakeCombatTurnsLLMClient{}
references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
NPCRegistryReferenceSlot: {
Slot: contracts.ReferenceSlot{Name: NPCRegistryReferenceSlot},
Items: []contracts.ReferenceItem{{SlotName: NPCRegistryReferenceSlot, MediaType: "application/json", Content: []byte(`{"secret":"private transcript detail"}`)}},
},
}}
_, err := New(newExtractor(t, client).engine, Options{}, references)
if err == nil || !strings.Contains(err.Error(), "prepare NPC registry") || strings.Contains(err.Error(), "private transcript detail") {
t.Fatalf("New() error = %v, want bounded content-free registry failure", err)
}
if len(client.requests) != 0 {
t.Fatalf("LLM calls = %d, want none during failed construction", len(client.requests))
}
}
func TestNewRejectsInvalidSceneReferenceBeforeLLMCallWithoutContent(t *testing.T) {
client := &fakeCombatTurnsLLMClient{}
references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
SceneDescriptionReferenceSlot: {
Slot: contracts.ReferenceSlot{Name: SceneDescriptionReferenceSlot},
Items: []contracts.ReferenceItem{{SlotName: SceneDescriptionReferenceSlot, MediaType: scenecodec.MediaType, Content: []byte(`{"scenes":[]}`)}},
},
}}
_, err := New(newExtractor(t, client).engine, Options{}, references)
if err == nil || !strings.Contains(err.Error(), "prepare scene eligibility") || strings.Contains(err.Error(), `{"scenes":[]}`) {
t.Fatalf("New() error = %v, want bounded content-free scene failure", err)
}
if len(client.requests) != 0 {
t.Fatalf("LLM calls = %d, want none during failed construction", len(client.requests))
}
}
func TestExtractRejectsInvalidRequestsAndWrapsProviderFailures(t *testing.T) {
validReq := extractionRequest(t)
validExtractor := newExtractor(t, &fakeCombatTurnsLLMClient{response: extractionResponse{CombatTurns: []combatTurnResponse{}}})
var nilExtractor *Extractor
tests := []struct {
name string
extractor *Extractor
ctx context.Context
req contracts.TypedExtractionRequest[dnd.CombatTurnList]
want string
}{
{name: "nil extractor", extractor: nilExtractor, ctx: context.Background(), req: validReq, want: "extractor"},
{name: "nil LLM client", extractor: &Extractor{}, ctx: context.Background(), req: validReq, want: "native engine"},
{name: "wrapped preflight failure", extractor: validExtractor, ctx: context.Background(), req: mismatchedSourceInputRequest(validReq), want: "must match chunk"},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
_, err := test.extractor.Extract(test.ctx, test.req)
if err == nil || !strings.Contains(err.Error(), "dnd combat turns") || !strings.Contains(err.Error(), test.want) {
t.Fatalf("Extract() error = %v, want %q context", err, test.want)
}
})
}
_, err := newExtractor(t, &fakeCombatTurnsLLMClient{err: errors.New("provider unavailable")}).Extract(context.Background(), extractionRequest(t))
if err == nil || !strings.Contains(err.Error(), "dnd combat turns") || !strings.Contains(err.Error(), "provider unavailable") {
t.Fatalf("provider error = %v, want contextual wrapped error", err)
}
}
func TestModuleSpecAndRegistration(t *testing.T) {
wantSlots := referenceSlots()
got := ModuleSpec()
if got.Key != Key || got.Stage != pipeline.StageExtract || !reflect.DeepEqual(got.Prompts, []string{PromptID, CorrectionPromptID}) || got.ArtifactKind != dnd.CombatTurnListKind || !reflect.DeepEqual(got.Requires, []string{"chunks", "source.transcript"}) || !reflect.DeepEqual(got.Provides, []string{"dnd.combat_turns"}) || !reflect.DeepEqual(got.ReferenceSlots, wantSlots) {
t.Fatalf("ModuleSpec() = %#v, want combat extractor contract", got)
}
got.Requires[0] = "changed"
got.Provides[0] = "changed"
got.ReferenceSlots[0].AcceptedMediaTypes[0] = "changed"
again := ModuleSpec()
if again.Requires[0] == "changed" || again.Provides[0] == "changed" || again.ReferenceSlots[0].AcceptedMediaTypes[0] == "changed" {
t.Fatalf("ModuleSpec() returned shared mutable values: %#v", again)
}
registry := pipeline.NewExtractorRegistry()
if err := Register(registry); err != nil {
t.Fatalf("Register() error = %v", err)
}
if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil {
t.Fatal("DecodeOptions() accepted unknown option")
}
if _, ok := registry.Spec(Key); !ok {
t.Fatalf("registry missing spec for %q", Key)
}
}