318 lines
14 KiB
Go
318 lines
14 KiB
Go
package npcoccurrences
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
|
npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcregistry"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity"
|
|
)
|
|
|
|
func TestExtractMapsEveryKindAndOrdersBySourcePosition(t *testing.T) {
|
|
client := &fakeOccurrencesLLMClient{response: extractionResponse{Occurrences: []occurrenceResponse{
|
|
{Name: "Other", Kind: "other", SourceRefs: occurrenceRefs(30, 30)},
|
|
{Name: "Opponent", Kind: "combat_opponent", SourceRefs: occurrenceRefs(20, 20)},
|
|
{Name: "Ally", Kind: "combat_ally", SourceRefs: occurrenceRefs(5, 5)},
|
|
{Name: "Speaker", Kind: "dialogue", SourceRefs: append(occurrenceRefs(2, 2), occurrenceRefs(2, 2)...)},
|
|
{Name: "Present", Kind: "noncombat_presence", SourceRefs: occurrenceRefs(7, 2)},
|
|
{Name: "Mentioned", Kind: "mentioned", SourceRefs: occurrenceRefs(10, 10)},
|
|
{Name: "Invalid", Kind: "other", SourceRefs: occurrenceRefs(0, 0)},
|
|
}}}
|
|
references := requiredRegistryReferences(t, "Mentioned", "Speaker", "Present", "Ally", "Opponent", "Other", "Invalid")
|
|
req := extractionRequest()
|
|
req.References = references
|
|
|
|
result, err := newExtractor(t, client, references).Extract(context.Background(), req)
|
|
if err != nil {
|
|
t.Fatalf("Extract() error = %v", err)
|
|
}
|
|
if got := occurrenceNames(result.Value); !reflect.DeepEqual(got, []string{"Mentioned", "Speaker", "Present", "Ally", "Opponent", "Other", "Invalid"}) {
|
|
t.Fatalf("occurrence order = %#v", got)
|
|
}
|
|
if got := occurrenceKinds(result.Value); !reflect.DeepEqual(got, []dnd.NPCOccurrenceKind{
|
|
dnd.NPCOccurrenceKindMentioned,
|
|
dnd.NPCOccurrenceKindDialogue,
|
|
dnd.NPCOccurrenceKindNoncombatPresence,
|
|
dnd.NPCOccurrenceKindCombatAlly,
|
|
dnd.NPCOccurrenceKindCombatOpponent,
|
|
dnd.NPCOccurrenceKindOther,
|
|
dnd.NPCOccurrenceKindOther,
|
|
}) {
|
|
t.Fatalf("occurrence kinds = %#v", got)
|
|
}
|
|
if refs := result.Value.Occurrences[1].SourceRefs; !reflect.DeepEqual(refs, []source.SourceRef{{SourceID: "session-alpha", StartUnitID: 2, EndUnitID: 2}}) {
|
|
t.Fatalf("canonical source refs = %#v", refs)
|
|
}
|
|
if refs := result.Value.Occurrences[2].SourceRefs; !reflect.DeepEqual(refs, []source.SourceRef{{SourceID: "session-alpha", StartUnitID: 2, EndUnitID: 7}}) {
|
|
t.Fatalf("reversed source refs = %#v", refs)
|
|
}
|
|
if id := result.Value.Occurrences[0].NPCID; id != identity.DeriveID("Mentioned") {
|
|
t.Fatalf("durable NPC ID = %q, want registry identity", id)
|
|
}
|
|
if invalid := result.Value.Occurrences[6]; invalid.Name != "Invalid" || invalid.Kind != dnd.NPCOccurrenceKindOther || !reflect.DeepEqual(invalid.SourceRefs, []source.SourceRef{{SourceID: "session-alpha"}}) {
|
|
t.Fatalf("invalid candidate = %#v, want preserved values with current source identity", invalid)
|
|
}
|
|
if len(client.requests) != 1 {
|
|
t.Fatalf("LLM calls = %d, want 1", len(client.requests))
|
|
}
|
|
}
|
|
|
|
func TestExtractUsesDocumentOrderForReferencesAndOccurrences(t *testing.T) {
|
|
client := &fakeOccurrencesLLMClient{response: extractionResponse{Occurrences: []occurrenceResponse{
|
|
{Name: "Later", Kind: "dialogue", SourceRefs: occurrenceRefs(10, 10)},
|
|
{Name: "First", Kind: "mentioned", SourceRefs: []occurrenceSourceRefResponse{
|
|
{StartUnitID: 10, EndUnitID: 10},
|
|
{StartUnitID: 30, EndUnitID: 30},
|
|
{StartUnitID: 30, EndUnitID: 30},
|
|
{StartUnitID: 999, EndUnitID: 0},
|
|
}},
|
|
{Name: "Second", Kind: "other", SourceRefs: occurrenceRefs(30, 30)},
|
|
}}}
|
|
references := requiredRegistryReferences(t, "Later", "First", "Second")
|
|
req := extractionRequest()
|
|
req.References = references
|
|
req.Source.Units = []source.SourceUnit{{ID: 30}, {ID: 10}}
|
|
req.Chunk.Units = append([]source.SourceUnit(nil), req.Source.Units...)
|
|
req.Chunk.Ref = source.SourceRef{SourceID: req.Source.ID, StartUnitID: 30, EndUnitID: 10}
|
|
|
|
result, err := newExtractor(t, client, references).Extract(context.Background(), req)
|
|
if err != nil {
|
|
t.Fatalf("Extract() error = %v", err)
|
|
}
|
|
if got := occurrenceNames(result.Value); !reflect.DeepEqual(got, []string{"First", "Second", "Later"}) {
|
|
t.Fatalf("occurrence order = %#v, want document chronology with stable equal-evidence ties", got)
|
|
}
|
|
refs := result.Value.Occurrences[0].SourceRefs
|
|
if got := []int{refs[0].StartUnitID, refs[1].StartUnitID, refs[2].StartUnitID}; !reflect.DeepEqual(got, []int{30, 10, 999}) {
|
|
t.Fatalf("source refs = %#v, want document order with exact duplicate removed", refs)
|
|
}
|
|
refs[0].StartUnitID = 777
|
|
for _, occurrence := range client.response.Occurrences {
|
|
for _, ref := range occurrence.SourceRefs {
|
|
if ref.StartUnitID == 777 {
|
|
t.Fatal("result source references alias the model response")
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestNewRequiresLLMAndRejectsAmbiguousReferenceSets(t *testing.T) {
|
|
if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "native engine") {
|
|
t.Fatalf("New(nil) error = %v", err)
|
|
}
|
|
if _, err := New(newExtractor(t, &fakeOccurrencesLLMClient{}).engine, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one reference set") {
|
|
t.Fatalf("New() error = %v", err)
|
|
}
|
|
}
|
|
|
|
func TestExtractUsesNamesOnlyRegistryAndCurrentTranscriptEvidence(t *testing.T) {
|
|
client := &fakeOccurrencesLLMClient{response: extractionResponse{Occurrences: []occurrenceResponse{{
|
|
Name: "Mira Thorn", Kind: "dialogue", SourceRefs: occurrenceRefs(10, 10),
|
|
}}}}
|
|
references := requiredRegistryReferences(t, "Mira Thorn", "Hooded Guard")
|
|
req := extractionRequest()
|
|
req.References = references
|
|
if _, err := newExtractor(t, client, references).Extract(context.Background(), req); err != nil {
|
|
t.Fatalf("Extract() error = %v", err)
|
|
}
|
|
request := client.requests[0]
|
|
registry := renderedInput(t, request, NPCRegistryReferenceSlot)
|
|
if string(registry) != `{"npcs":[{"name":"Mira Thorn"},{"name":"Hooded Guard"}]}` {
|
|
t.Fatalf("names-only registry=%s", registry)
|
|
}
|
|
for _, forbidden := range []string{"npc:sha256:", "other-session", "start_unit_id"} {
|
|
if strings.Contains(string(registry), forbidden) {
|
|
t.Fatalf("registry projection exposed %q", forbidden)
|
|
}
|
|
}
|
|
if strings.Contains(string(renderedInput(t, request, "transcript")), "other-session") {
|
|
t.Fatal("registry provenance became occurrence evidence")
|
|
}
|
|
|
|
}
|
|
|
|
func TestExtractRequiresBoundRegistryBeforeLLMCall(t *testing.T) {
|
|
client := &fakeOccurrencesLLMClient{}
|
|
if _, err := newExtractor(t, client).Extract(context.Background(), extractionRequest()); err == nil || !strings.Contains(err.Error(), "NPC registry reference is required") {
|
|
t.Fatalf("Extract() error = %v, want required registry failure", err)
|
|
}
|
|
if len(client.requests) != 0 {
|
|
t.Fatalf("LLM calls = %d, want no call", len(client.requests))
|
|
}
|
|
|
|
malformed := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
|
|
NPCRegistryReferenceSlot: {
|
|
Slot: contracts.ReferenceSlot{Name: NPCRegistryReferenceSlot},
|
|
Items: []contracts.ReferenceItem{{SlotName: NPCRegistryReferenceSlot, MediaType: "application/json", Content: []byte(`{"secret":"private source"}`)}},
|
|
},
|
|
}}
|
|
if _, err := New(newExtractor(t, client).engine, Options{}, malformed); err == nil || !strings.Contains(err.Error(), "prepare NPC registry") || strings.Contains(err.Error(), "private source") {
|
|
t.Fatalf("New() error = %v, want content-safe malformed registry error", err)
|
|
}
|
|
if len(client.requests) != 0 {
|
|
t.Fatalf("LLM calls = %d, want no call", len(client.requests))
|
|
}
|
|
}
|
|
|
|
func TestExtractRejectsUnknownNamesWithoutPartialResult(t *testing.T) {
|
|
references := requiredRegistryReferences(t, "Mira Thorn")
|
|
client := &fakeOccurrencesLLMClient{response: extractionResponse{Occurrences: []occurrenceResponse{
|
|
{Name: "Mira Thorn", Kind: "dialogue", SourceRefs: occurrenceRefs(10, 10)},
|
|
{Name: "Unknown NPC", Kind: "mentioned", SourceRefs: occurrenceRefs(20, 20)},
|
|
}}}
|
|
req := extractionRequest()
|
|
req.References = references
|
|
result, err := newExtractor(t, client, references).Extract(context.Background(), req)
|
|
if err == nil || result.Outcome.Disposition != "" {
|
|
t.Fatalf("Extract() error = %v, want unknown name failure", err)
|
|
}
|
|
if len(result.Value.Occurrences) != 0 {
|
|
t.Fatalf("Extract() returned partial result = %#v", result.Value)
|
|
}
|
|
}
|
|
|
|
func TestExtractCanonicalizesComparisonEquivalentRegistryNames(t *testing.T) {
|
|
references := requiredRegistryReferences(t, "Mira Thorn")
|
|
client := &fakeOccurrencesLLMClient{response: extractionResponse{Occurrences: []occurrenceResponse{{
|
|
Name: " mIRA\u2003thorn ", Kind: "dialogue", SourceRefs: occurrenceRefs(10, 10),
|
|
}}}}
|
|
req := extractionRequest()
|
|
req.References = references
|
|
result, err := newExtractor(t, client, references).Extract(context.Background(), req)
|
|
if err != nil {
|
|
t.Fatalf("Extract() error = %v", err)
|
|
}
|
|
occurrence := result.Value.Occurrences[0]
|
|
if occurrence.Name != "Mira Thorn" || occurrence.NPCID != identity.DeriveID("Mira Thorn") {
|
|
t.Fatalf("canonical occurrence = %#v", occurrence)
|
|
}
|
|
}
|
|
|
|
func TestExtractResolvesGeneratedRegistryAtOperationTime(t *testing.T) {
|
|
client := &fakeOccurrencesLLMClient{response: extractionResponse{Occurrences: []occurrenceResponse{}}}
|
|
references := requiredRegistryReferences(t, "Mira Thorn")
|
|
req := extractionRequest()
|
|
req.References = references
|
|
extractor := newExtractor(t, client)
|
|
if _, err := extractor.Extract(context.Background(), req); err != nil {
|
|
t.Fatalf("Extract() error = %v", err)
|
|
}
|
|
if input := renderedInput(t, client.requests[0], NPCRegistryReferenceSlot); string(input) != `{"npcs":[{"name":"Mira Thorn"}]}` {
|
|
t.Fatalf("generated registry=%s", input)
|
|
}
|
|
|
|
}
|
|
|
|
func TestExtractAcceptsEmptyBoundRegistryAndEmptyResponse(t *testing.T) {
|
|
content, err := npccodec.New().Encode(dnd.NPCRegistry{NPCs: []dnd.NPC{}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
|
|
NPCRegistryReferenceSlot: {
|
|
Slot: contracts.ReferenceSlot{Name: NPCRegistryReferenceSlot},
|
|
Items: []contracts.ReferenceItem{{SlotName: NPCRegistryReferenceSlot, MediaType: npccodec.MediaType, Content: content}},
|
|
},
|
|
}}
|
|
client := &fakeOccurrencesLLMClient{response: extractionResponse{Occurrences: []occurrenceResponse{}}}
|
|
req := extractionRequest()
|
|
req.References = references
|
|
result, err := newExtractor(t, client, references).Extract(context.Background(), req)
|
|
if err != nil || result.Value.Occurrences == nil || len(result.Value.Occurrences) != 0 {
|
|
t.Fatalf("Extract() = %#v, %v; want present empty occurrences", result, err)
|
|
}
|
|
}
|
|
|
|
func TestExtractRejectsNonemptyResponseForEmptyBoundRegistry(t *testing.T) {
|
|
content, err := npccodec.New().Encode(dnd.NPCRegistry{NPCs: []dnd.NPC{}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{NPCRegistryReferenceSlot: {
|
|
Slot: contracts.ReferenceSlot{Name: NPCRegistryReferenceSlot},
|
|
Items: []contracts.ReferenceItem{{SlotName: NPCRegistryReferenceSlot, MediaType: npccodec.MediaType, Content: content}},
|
|
}}}
|
|
client := &fakeOccurrencesLLMClient{response: extractionResponse{Occurrences: []occurrenceResponse{{Name: "Mira Thorn", Kind: "dialogue", SourceRefs: occurrenceRefs(10, 10)}}}}
|
|
req := extractionRequest()
|
|
req.References = references
|
|
result, err := newExtractor(t, client, references).Extract(context.Background(), req)
|
|
if err == nil || result.Outcome.Disposition != "" || len(result.Value.Occurrences) != 0 {
|
|
t.Fatalf("Extract() = %#v, %v; want no accepted occurrences", result, err)
|
|
}
|
|
}
|
|
|
|
func TestExtractRejectsInvalidRequestsAndProviderFailures(t *testing.T) {
|
|
references := requiredRegistryReferences(t, "Mira Thorn")
|
|
valid := extractionRequest()
|
|
valid.References = references
|
|
extractor := newExtractor(t, &fakeOccurrencesLLMClient{}, references)
|
|
var nilExtractor *Extractor
|
|
for _, test := range []struct {
|
|
name string
|
|
extractor *Extractor
|
|
ctx context.Context
|
|
req contracts.TypedExtractionRequest[dnd.NPCOccurrenceList]
|
|
want string
|
|
}{
|
|
{"nil extractor", nilExtractor, context.Background(), valid, "extractor"},
|
|
{"nil native engine", &Extractor{}, context.Background(), valid, "native engine"},
|
|
{"wrapped preflight failure", extractor, context.Background(), mismatchedSourceInputRequest(valid), "must match chunk"},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
if _, err := test.extractor.Extract(test.ctx, test.req); err == nil || !strings.Contains(err.Error(), test.want) {
|
|
t.Fatalf("Extract() error = %v, want %q", err, test.want)
|
|
}
|
|
})
|
|
}
|
|
if _, err := newExtractor(t, &fakeOccurrencesLLMClient{err: errors.New("provider unavailable")}, references).Extract(context.Background(), valid); err == nil || !strings.Contains(err.Error(), "provider unavailable") {
|
|
t.Fatalf("provider error = %v", err)
|
|
}
|
|
}
|
|
|
|
func TestModuleSpecRegistrationAndNativeDependencies(t *testing.T) {
|
|
got := ModuleSpec()
|
|
if got.Key != Key || got.Stage != pipeline.StageExtract || !reflect.DeepEqual(got.Prompts, []string{PromptID, CorrectionPromptID}) || got.ArtifactKind != dnd.NPCOccurrenceListKind || !reflect.DeepEqual(got.Requires, []string{"chunks", "source.transcript"}) || !reflect.DeepEqual(got.Provides, []string{"dnd.npc_occurrences"}) {
|
|
t.Fatalf("ModuleSpec() = %#v", got)
|
|
}
|
|
var registrySlot contracts.ReferenceSlot
|
|
for _, slot := range got.ReferenceSlots {
|
|
if slot.Name == NPCRegistryReferenceSlot {
|
|
registrySlot = slot
|
|
}
|
|
}
|
|
if !registrySlot.Required || !reflect.DeepEqual(registrySlot.AcceptedArtifactKinds, []contracts.ArtifactKind{dnd.NPCRegistryKind}) || registrySlot.MaxBytes != NPCRegistryMaxBytes {
|
|
t.Fatalf("NPC registry slot = %#v", registrySlot)
|
|
}
|
|
got.Prompts[0] = "changed"
|
|
if ModuleSpec().Prompts[0] == "changed" {
|
|
t.Fatal("prompt dependencies alias caller memory")
|
|
}
|
|
got.ReferenceSlots[0].AcceptedMediaTypes[0] = "changed"
|
|
if ModuleSpec().ReferenceSlots[0].AcceptedMediaTypes[0] == "changed" {
|
|
t.Fatal("ModuleSpec() returned mutable reference slots")
|
|
}
|
|
extractorSlots := newExtractor(t, &fakeOccurrencesLLMClient{}).ReferenceSlots()
|
|
extractorSlots[0].AcceptedMediaTypes[0] = "changed"
|
|
if newExtractor(t, &fakeOccurrencesLLMClient{}).ReferenceSlots()[0].AcceptedMediaTypes[0] == "changed" {
|
|
t.Fatal("ReferenceSlots() returned mutable reference slots")
|
|
}
|
|
registry := pipeline.NewExtractorRegistry()
|
|
if err := Register(registry); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, ok := registry.Spec(Key); !ok {
|
|
t.Fatalf("registry missing %q", Key)
|
|
}
|
|
if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil {
|
|
t.Fatal("DecodeOptions() accepted unknown option")
|
|
}
|
|
|
|
}
|