251 lines
10 KiB
Go
251 lines
10 KiB
Go
package npcregistry
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/assets"
|
|
"go.maximumdirect.net/configkit"
|
|
"go.maximumdirect.net/promptkit"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity"
|
|
)
|
|
|
|
func TestModuleContractAndIdentity(t *testing.T) {
|
|
if _, err := DecodeOptions(nil); err != nil {
|
|
t.Fatalf("DecodeOptions(nil) error = %v, want nil", err)
|
|
}
|
|
if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil {
|
|
t.Fatal("DecodeOptions() accepted unknown option")
|
|
}
|
|
want := pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{PromptID, CorrectionPromptID}, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: dnd.NPCRegistryKind}
|
|
if got := ModuleSpec(); !reflect.DeepEqual(got, want) {
|
|
t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
|
|
}
|
|
registry := pipeline.NewNormalizerRegistry()
|
|
if err := Register(registry); err != nil {
|
|
t.Fatalf("Register() error = %v", err)
|
|
}
|
|
if _, err := New(nil, Options{}); err == nil {
|
|
t.Fatal("New(nil, Options{}) error = nil, want nil native engine rejection")
|
|
}
|
|
spec := ModuleSpec()
|
|
spec.Prompts[0] = "mutated"
|
|
if !reflect.DeepEqual(ModuleSpec(), want) {
|
|
t.Fatal("static dependencies are not owned")
|
|
}
|
|
|
|
}
|
|
|
|
func TestNormalizeNamesEvidenceAndIDs(t *testing.T) {
|
|
input := dnd.NPCRegistry{NPCs: []dnd.NPC{{
|
|
ID: "wrong", Name: " Lady\tAsh ", SourceRefs: []source.SourceRef{
|
|
{SourceID: "b", StartUnitID: 2, EndUnitID: 3},
|
|
{SourceID: "a", StartUnitID: 4, EndUnitID: 4},
|
|
{SourceID: "b", StartUnitID: 2, EndUnitID: 3},
|
|
},
|
|
}}}
|
|
result, err := newNormalizer(t, &recordingNPCNormalizerClient{}).Normalize(context.Background(), normalizeRequest(input))
|
|
if err != nil {
|
|
t.Fatalf("Normalize() error = %v", err)
|
|
}
|
|
want := dnd.NPC{ID: identity.DeriveID("Lady Ash"), Name: "Lady Ash", SourceRefs: []source.SourceRef{{SourceID: "a", StartUnitID: 4, EndUnitID: 4}, {SourceID: "b", StartUnitID: 2, EndUnitID: 3}}}
|
|
if !reflect.DeepEqual(result.Value.NPCs[0], want) {
|
|
t.Fatalf("NPC = %#v, want %#v", result.Value.NPCs[0], want)
|
|
}
|
|
for _, reason := range []string{ReasonCodeNPCFieldsNormalized, ReasonCodeSourceReferencesNormalized, ReasonCodeNPCIDRecomputed} {
|
|
if !hasDiagnostic(result.Diagnostics, reason, "npcs[0]", contracts.DiagnosticDispositionObservation) {
|
|
t.Fatalf("diagnostics = %#v, want %s", result.Diagnostics, reason)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestNormalizeOrdersEvidenceBySourceDocumentPosition(t *testing.T) {
|
|
doc := &source.SourceDocument{ID: "source", Units: []source.SourceUnit{{ID: 30}, {ID: 10}}}
|
|
input := dnd.NPCRegistry{NPCs: []dnd.NPC{{Name: "Lady Ash", SourceRefs: []source.SourceRef{
|
|
{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10},
|
|
{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30},
|
|
{SourceID: doc.ID, StartUnitID: 999, EndUnitID: 999},
|
|
}}}}
|
|
result, err := newNormalizer(t, &recordingNPCNormalizerClient{}).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
|
if err != nil {
|
|
t.Fatalf("Normalize() error = %v", err)
|
|
}
|
|
got := result.Value.NPCs[0].SourceRefs
|
|
if got[0].StartUnitID != 30 || got[1].StartUnitID != 10 || got[2].StartUnitID != 999 {
|
|
t.Fatalf("source refs = %#v, want source-document order followed by invalid reference", got)
|
|
}
|
|
}
|
|
|
|
func TestNormalizeConsolidatesCanonicalNamesOnlyAndUnionsEvidence(t *testing.T) {
|
|
input := dnd.NPCRegistry{NPCs: []dnd.NPC{
|
|
{Name: " Captain Vale ", SourceRefs: []source.SourceRef{{SourceID: "a", StartUnitID: 1, EndUnitID: 1}}},
|
|
{Name: "captain vale", SourceRefs: []source.SourceRef{{SourceID: "b", StartUnitID: 2, EndUnitID: 2}}},
|
|
{Name: "The Captain", SourceRefs: []source.SourceRef{{SourceID: "c", StartUnitID: 3, EndUnitID: 3}}},
|
|
}}
|
|
result, err := newNormalizer(t, &recordingNPCNormalizerClient{}).Normalize(context.Background(), normalizeRequest(input))
|
|
if err != nil {
|
|
t.Fatalf("Normalize() error = %v", err)
|
|
}
|
|
if len(result.Value.NPCs) != 2 || result.Value.NPCs[0].Name != "Captain Vale" || result.Value.NPCs[1].Name != "The Captain" {
|
|
t.Fatalf("NPCs = %#v, want name-only stable consolidation", result.Value.NPCs)
|
|
}
|
|
if refs := result.Value.NPCs[0].SourceRefs; len(refs) != 2 || refs[0].SourceID != "a" || refs[1].SourceID != "b" {
|
|
t.Fatalf("source refs = %#v, want evidence union", refs)
|
|
}
|
|
if !hasDiagnostic(result.Diagnostics, ReasonCodeDuplicateNPCCollapsed, "npcs[0]", contracts.DiagnosticDispositionObservation) {
|
|
t.Fatalf("diagnostics = %#v, want duplicate collapse", result.Diagnostics)
|
|
}
|
|
}
|
|
|
|
func TestNormalizePreservesInvalidCandidatesAndDoesNotAliasInput(t *testing.T) {
|
|
input := dnd.NPCRegistry{NPCs: []dnd.NPC{{Name: " ", SourceRefs: []source.SourceRef{{SourceID: "source", StartUnitID: 0, EndUnitID: -1}}}}}
|
|
before := dnd.NPCRegistry{NPCs: []dnd.NPC{{Name: " ", SourceRefs: []source.SourceRef{{SourceID: "source", StartUnitID: 0, EndUnitID: -1}}}}}
|
|
result, err := newNormalizer(t, &recordingNPCNormalizerClient{}).Normalize(context.Background(), normalizeRequest(input))
|
|
if err != nil || !reflect.DeepEqual(input, before) {
|
|
t.Fatalf("Normalize() = %#v, %v; input mutated to %#v", result, err, input)
|
|
}
|
|
if result.Value.NPCs[0].Name != "" || result.Value.NPCs[0].SourceRefs[0].EndUnitID != -1 {
|
|
t.Fatalf("candidate = %#v, want invalid semantics preserved", result.Value.NPCs[0])
|
|
}
|
|
result.Value.NPCs[0].SourceRefs[0].SourceID = "changed"
|
|
if input.NPCs[0].SourceRefs[0].SourceID != "source" {
|
|
t.Fatal("output aliases input evidence")
|
|
}
|
|
}
|
|
|
|
func TestNormalizeHandlesNilAndCanceledCalls(t *testing.T) {
|
|
normalizer := newNormalizer(t, &recordingNPCNormalizerClient{})
|
|
if _, err := normalizer.Normalize(context.Background(), contracts.TypedNormalizeRequest[dnd.NPCRegistry]{}); err == nil || !strings.Contains(err.Error(), "source document must not be nil") {
|
|
t.Fatalf("nil source Normalize() error = %v", err)
|
|
}
|
|
result, err := normalizer.Normalize(context.Background(), normalizeRequest(dnd.NPCRegistry{NPCs: nil}))
|
|
if err != nil || result.Value.NPCs != nil {
|
|
t.Fatalf("nil list result = %#v, error = %v", result.Value, err)
|
|
}
|
|
canceled, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
if _, err := normalizer.Normalize(canceled, normalizeRequest(dnd.NPCRegistry{})); err == nil || !strings.Contains(err.Error(), "context error") {
|
|
t.Fatalf("canceled Normalize() error = %v", err)
|
|
}
|
|
}
|
|
|
|
type recordingNPCNormalizerClient struct {
|
|
response string
|
|
responses []string
|
|
err error
|
|
errors []error
|
|
requests []promptkit.GenerateRequest
|
|
onGenerate func(context.Context, promptkit.GenerateRequest)
|
|
}
|
|
|
|
func (c *recordingNPCNormalizerClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
index := len(c.requests)
|
|
c.requests = append(c.requests, request)
|
|
if c.onGenerate != nil {
|
|
c.onGenerate(ctx, request)
|
|
}
|
|
if c.err != nil {
|
|
return nil, c.err
|
|
}
|
|
if index < len(c.errors) && c.errors[index] != nil {
|
|
return nil, c.errors[index]
|
|
}
|
|
response := c.response
|
|
if index < len(c.responses) {
|
|
response = c.responses[index]
|
|
}
|
|
if response == "" {
|
|
response = `{"duplicate_groups":[]}`
|
|
}
|
|
return &promptkit.GenerateResponse{Content: response}, nil
|
|
}
|
|
|
|
func newNormalizer(t *testing.T, client promptkit.LLMClient) *Normalizer {
|
|
t.Helper()
|
|
type settings struct {
|
|
PromptKit promptkit.Settings `yaml:"promptkit"`
|
|
}
|
|
definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit })
|
|
filesystem := configkit.NewFilesystem("Notarius assets", assets.FS())
|
|
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("normalizer fixture", []byte(`promptkit:
|
|
profiles:
|
|
- id: normalizer-profile
|
|
endpoint: https://example.invalid/v1
|
|
model: normalization-test-model
|
|
api_key_required: false
|
|
`))}, configkit.LayeredOptions[settings]{Definition: definition, Underlays: []configkit.Underlay{
|
|
promptkit.BundledOpenRouterUnderlay(),
|
|
{Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}},
|
|
}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
normalizer, err := New(engine, Options{})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return normalizer
|
|
}
|
|
|
|
func renderedInput(t *testing.T, request promptkit.GenerateRequest, name string) []byte {
|
|
t.Helper()
|
|
key := "candidate_number_range"
|
|
if name == "transcript" {
|
|
key = "windows"
|
|
}
|
|
for _, message := range request.Prompt.Messages {
|
|
start := strings.Index(message.Content, "{")
|
|
if start < 0 {
|
|
continue
|
|
}
|
|
content := []byte(message.Content[start:])
|
|
var fields map[string]json.RawMessage
|
|
if json.Unmarshal(content, &fields) == nil && fields[key] != nil {
|
|
return content
|
|
}
|
|
}
|
|
t.Fatalf("missing rendered %s input", name)
|
|
return nil
|
|
}
|
|
func normalizeRequest(value dnd.NPCRegistry) contracts.TypedNormalizeRequest[dnd.NPCRegistry] {
|
|
return contracts.TypedNormalizeRequest[dnd.NPCRegistry]{
|
|
Source: &source.SourceDocument{}, LLMProfile: "normalizer-profile",
|
|
Operation: contracts.ProducerOperation[dnd.NPCRegistry]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, Cache: contracts.CacheBypass}, ReuseEligible: true, Assess: func(context.Context, dnd.NPCRegistry) (contracts.CandidateAssessment, error) {
|
|
return contracts.CandidateAssessment{}, nil
|
|
}},
|
|
MergeOutput: contracts.MergeArtifact[dnd.NPCRegistry]{Value: value},
|
|
}
|
|
}
|
|
|
|
func normalizeRequestWithSource(value dnd.NPCRegistry, doc *source.SourceDocument) contracts.TypedNormalizeRequest[dnd.NPCRegistry] {
|
|
request := normalizeRequest(value)
|
|
request.Source = doc
|
|
return request
|
|
}
|
|
|
|
func hasDiagnostic(diagnostics []contracts.ProducerDiagnostic, reason, scope string, disposition contracts.DiagnosticDisposition) bool {
|
|
for _, diagnostic := range diagnostics {
|
|
if diagnostic.ReasonCode != reason || diagnostic.Disposition != disposition {
|
|
continue
|
|
}
|
|
for _, sample := range diagnostic.Samples {
|
|
if sample.Scope == scope {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|