521 lines
24 KiB
Go
521 lines
24 KiB
Go
package itemregistry
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"reflect"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/assets"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity"
|
|
identityvalidator "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemregistry/identity"
|
|
"go.maximumdirect.net/configkit"
|
|
"go.maximumdirect.net/promptkit"
|
|
)
|
|
|
|
func TestModuleContractAndNativeDependencies(t *testing.T) {
|
|
want := pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{PromptID, CorrectionPromptID}, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: dnd.ItemRegistryKind}
|
|
if got := ModuleSpec(); !reflect.DeepEqual(got, want) {
|
|
t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
|
|
}
|
|
registry := pipeline.NewNormalizerRegistry()
|
|
if err := Register(registry); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := New(nil, Options{}); err == nil {
|
|
t.Fatal("New() accepted nil native engine")
|
|
}
|
|
got := ModuleSpec()
|
|
got.Prompts[0] = "caller mutation"
|
|
if !reflect.DeepEqual(ModuleSpec(), want) {
|
|
t.Fatal("static dependency declaration is not owned")
|
|
}
|
|
}
|
|
|
|
func TestNormalizeRejectsNilSourceDocument(t *testing.T) {
|
|
_, err := newNormalizer(t, &recordingNormalizerClient{}).Normalize(
|
|
context.Background(),
|
|
contracts.TypedNormalizeRequest[dnd.ItemRegistry]{},
|
|
)
|
|
if err == nil || !strings.Contains(err.Error(), "source document must not be nil") {
|
|
t.Fatalf("Normalize() error = %v, want nil source rejection", err)
|
|
}
|
|
}
|
|
|
|
func TestNormalizeConsolidatesEqualNamesAcrossEvidenceWithoutMutation(t *testing.T) {
|
|
doc := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{
|
|
{ID: 1, Text: "The rope is secured."},
|
|
{ID: 2, Text: "The party takes the rope."},
|
|
{ID: 3, Text: "The rope is packed away."},
|
|
{ID: 4, Text: "A lantern lights the path."},
|
|
}}
|
|
input := dnd.ItemRegistry{Items: []dnd.Item{
|
|
{Name: " Rope ", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 3, EndUnitID: 3}, {SourceID: "session", StartUnitID: 1, EndUnitID: 1}}},
|
|
{Name: "rope", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 2, EndUnitID: 2}, {SourceID: "session", StartUnitID: 3, EndUnitID: 3}}},
|
|
{Name: "Lantern", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 4, EndUnitID: 4}}},
|
|
}}
|
|
before := dnd.ItemRegistry{Items: []dnd.Item{
|
|
{Name: " Rope ", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 3, EndUnitID: 3}, {SourceID: "session", StartUnitID: 1, EndUnitID: 1}}},
|
|
{Name: "rope", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 2, EndUnitID: 2}, {SourceID: "session", StartUnitID: 3, EndUnitID: 3}}},
|
|
{Name: "Lantern", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 4, EndUnitID: 4}}},
|
|
}}
|
|
client := &recordingNormalizerClient{}
|
|
result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
|
if err != nil || len(result.Value.Items) != 2 || !reflect.DeepEqual(input, before) || len(client.requests) != 1 {
|
|
t.Fatalf("Normalize() = %#v, %v; want deterministic non-mutating consolidation", result, err)
|
|
}
|
|
rope := result.Value.Items[0]
|
|
wantRefs := []source.SourceRef{{SourceID: "session", StartUnitID: 1, EndUnitID: 1}, {SourceID: "session", StartUnitID: 2, EndUnitID: 2}, {SourceID: "session", StartUnitID: 3, EndUnitID: 3}}
|
|
if rope.Name != "Rope" || rope.ID != identity.DeriveID("Rope") || !reflect.DeepEqual(rope.SourceRefs, wantRefs) || result.Value.Items[1].Name != "Lantern" || !hasDiagnostic(result.Diagnostics, ReasonCodeDuplicateItemCollapsed, contracts.DiagnosticDispositionObservation) {
|
|
t.Fatalf("items = %#v, diagnostics = %#v; want earliest display name, canonical evidence union, and stable placement", result.Value.Items, result.Diagnostics)
|
|
}
|
|
validation, validationErr := identityvalidator.New(identityvalidator.Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.ItemRegistry]{Value: result.Value})
|
|
if validationErr != nil || !validation.Approved {
|
|
t.Fatalf("default item-registry identity validation = %#v, %v; want normalized cross-chunk candidates accepted", validation, validationErr)
|
|
}
|
|
var candidates struct {
|
|
Candidates []struct {
|
|
Label string `json:"label"`
|
|
} `json:"candidates"`
|
|
}
|
|
if err := json.Unmarshal(renderedInput(t, client.requests[0], "candidates"), &candidates); err != nil || len(candidates.Candidates) != 2 || candidates.Candidates[0].Label != "Rope" || candidates.Candidates[1].Label != "Lantern" {
|
|
t.Fatalf("semantic candidates = %#v, %v; want one candidate per comparison name", candidates, err)
|
|
}
|
|
repeated, repeatErr := newNormalizer(t, &recordingNormalizerClient{}).Normalize(context.Background(), normalizeRequestWithSource(result.Value, doc))
|
|
if repeatErr != nil || !reflect.DeepEqual(repeated.Value, result.Value) {
|
|
t.Fatalf("repeated normalization = %#v, %v; want stable output", repeated, repeatErr)
|
|
}
|
|
}
|
|
|
|
func BenchmarkComparisonNameGroupsManyDistinct(b *testing.B) {
|
|
records := make([]normalizedRecord, 1_000)
|
|
for index := range records {
|
|
records[index] = normalizedRecord{item: dnd.Item{Name: "Item " + strconv.Itoa(index)}}
|
|
}
|
|
b.ReportAllocs()
|
|
b.ResetTimer()
|
|
for iteration := 0; iteration < b.N; iteration++ {
|
|
if groups := comparisonNameGroups(records); len(groups) != len(records) {
|
|
b.Fatalf("group count = %d, want %d", len(groups), len(records))
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestNormalizeAppliesSafeAliasProposal(t *testing.T) {
|
|
doc := semanticDocument()
|
|
input := dnd.ItemRegistry{Items: []dnd.Item{
|
|
{Name: "Star Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}},
|
|
{Name: "Compass of the Stars", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}},
|
|
{Name: "Gold Pieces", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}},
|
|
}}
|
|
client := &recordingNormalizerClient{response: `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":2}]}`}
|
|
result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
|
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || len(result.Value.Items) != 2 {
|
|
t.Fatalf("Normalize() = %#v, %v", result, err)
|
|
}
|
|
merged := result.Value.Items[0]
|
|
if merged.Name != "Compass of the Stars" || merged.ID != identity.DeriveID(merged.Name) || len(merged.SourceRefs) != 2 || !hasDiagnostic(result.Diagnostics, ReasonCodeDuplicateItemCollapsed, contracts.DiagnosticDispositionObservation) {
|
|
t.Fatalf("merged item = %#v, diagnostics = %#v", merged, result.Diagnostics)
|
|
}
|
|
encoded := string(renderedInput(t, client.requests[0], "candidates")) + string(renderedInput(t, client.requests[0], "transcript"))
|
|
if strings.Contains(encoded, doc.ID) || strings.Contains(encoded, "candidate-") || strings.Contains(encoded, merged.ID) || !strings.Contains(encoded, `"source_refs"`) {
|
|
t.Fatalf("private inputs = %s", encoded)
|
|
}
|
|
}
|
|
|
|
func TestNormalizeAppliesCurrencyReconciliationSafely(t *testing.T) {
|
|
doc := semanticDocument()
|
|
ref := func(unitID int) []source.SourceRef {
|
|
return []source.SourceRef{{SourceID: doc.ID, StartUnitID: unitID, EndUnitID: unitID}}
|
|
}
|
|
for _, test := range []struct {
|
|
name string
|
|
items []dnd.Item
|
|
response string
|
|
wantNames []string
|
|
wantRefCounts []int
|
|
wantFallback bool
|
|
reasonCode string
|
|
disposition contracts.DiagnosticDisposition
|
|
}{
|
|
{
|
|
name: "same denomination aliases",
|
|
items: []dnd.Item{
|
|
{Name: "GP", SourceRefs: ref(10)},
|
|
{Name: "Gold Piece", SourceRefs: ref(20)},
|
|
{Name: "Gold Pieces", SourceRefs: ref(30)},
|
|
},
|
|
response: `{"duplicate_groups":[{"candidate_numbers":[1,2,3],"canonical_candidate_number":2}]}`,
|
|
wantNames: []string{"Gold Piece"},
|
|
wantRefCounts: []int{3},
|
|
reasonCode: ReasonCodeDuplicateItemCollapsed,
|
|
disposition: contracts.DiagnosticDispositionObservation,
|
|
},
|
|
{
|
|
name: "different denominations",
|
|
items: []dnd.Item{
|
|
{Name: "Gold Pieces", SourceRefs: ref(10)},
|
|
{Name: "Silver Pieces", SourceRefs: ref(20)},
|
|
},
|
|
response: `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":1}]}`,
|
|
wantNames: []string{"Gold Pieces", "Silver Pieces"},
|
|
wantRefCounts: []int{1, 1},
|
|
wantFallback: true,
|
|
reasonCode: ReasonCodeItemSemanticProposalInvalid,
|
|
disposition: contracts.DiagnosticDispositionAdvisory,
|
|
},
|
|
{
|
|
name: "currency plus ordinary item",
|
|
items: []dnd.Item{
|
|
{Name: "Gold Pieces", SourceRefs: ref(10)},
|
|
{Name: "Longsword", SourceRefs: ref(20)},
|
|
},
|
|
response: `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":1}]}`,
|
|
wantNames: []string{"Gold Pieces", "Longsword"},
|
|
wantRefCounts: []int{1, 1},
|
|
wantFallback: true,
|
|
reasonCode: ReasonCodeItemSemanticProposalInvalid,
|
|
disposition: contracts.DiagnosticDispositionAdvisory,
|
|
},
|
|
{
|
|
name: "ordinary items",
|
|
items: []dnd.Item{
|
|
{Name: "Star Compass", SourceRefs: ref(10)},
|
|
{Name: "Compass of the Stars", SourceRefs: ref(20)},
|
|
},
|
|
response: `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":2}]}`,
|
|
wantNames: []string{"Compass of the Stars"},
|
|
wantRefCounts: []int{2},
|
|
reasonCode: ReasonCodeDuplicateItemCollapsed,
|
|
disposition: contracts.DiagnosticDispositionObservation,
|
|
},
|
|
{
|
|
name: "currency plus ordinary canonical item",
|
|
items: []dnd.Item{
|
|
{Name: "Gold Pieces", SourceRefs: ref(10)},
|
|
{Name: "Longsword", SourceRefs: ref(20)},
|
|
},
|
|
response: `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":2}]}`,
|
|
wantNames: []string{"Gold Pieces", "Longsword"},
|
|
wantRefCounts: []int{1, 1},
|
|
wantFallback: true,
|
|
reasonCode: ReasonCodeItemSemanticProposalInvalid,
|
|
disposition: contracts.DiagnosticDispositionAdvisory,
|
|
},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
input := dnd.ItemRegistry{Items: make([]dnd.Item, len(test.items))}
|
|
for index, item := range test.items {
|
|
input.Items[index] = cloneItem(item)
|
|
}
|
|
before := dnd.ItemRegistry{Items: make([]dnd.Item, len(input.Items))}
|
|
for index, item := range input.Items {
|
|
before.Items[index] = cloneItem(item)
|
|
}
|
|
|
|
result, err := newNormalizer(t, &recordingNormalizerClient{response: test.response}).Normalize(context.Background(), permissiveRequestWithSource(input, doc))
|
|
if err != nil || (result.Outcome.Disposition == contracts.ProducerFallback) != test.wantFallback || !reflect.DeepEqual(input, before) || !hasDiagnostic(result.Diagnostics, test.reasonCode, test.disposition) {
|
|
t.Fatalf("Normalize() = %#v, %v", result, err)
|
|
}
|
|
if len(result.Value.Items) != len(test.wantNames) {
|
|
t.Fatalf("items = %#v, want names %#v", result.Value.Items, test.wantNames)
|
|
}
|
|
for index, item := range result.Value.Items {
|
|
if item.Name != test.wantNames[index] || len(item.SourceRefs) != test.wantRefCounts[index] {
|
|
t.Fatalf("item %d = %#v, want name %q with %d source refs", index, item, test.wantNames[index], test.wantRefCounts[index])
|
|
}
|
|
}
|
|
if test.wantFallback && (result.Outcome.ReuseEligible || !hasDiagnostic(result.Diagnostics, ReasonCodeItemSemanticReconciliationExhausted, contracts.DiagnosticDispositionWarning)) {
|
|
t.Fatalf("fallback lacks qualification: %#v", result)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestNormalizePreservesCandidatesForUnsafeProposalGroups(t *testing.T) {
|
|
doc := semanticDocument()
|
|
input := dnd.ItemRegistry{Items: []dnd.Item{
|
|
{Name: "Star Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}},
|
|
{Name: "Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}},
|
|
{Name: "Rope", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}},
|
|
}}
|
|
client := &recordingNormalizerClient{response: `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":1},{"candidate_numbers":[2,3],"canonical_candidate_number":3},{"candidate_numbers":[1,3],"canonical_candidate_number":99}]}`}
|
|
result, err := newNormalizer(t, client).Normalize(context.Background(), permissiveRequestWithSource(input, doc))
|
|
if err != nil || result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || len(result.Value.Items) != 3 || !hasDiagnostic(result.Diagnostics, ReasonCodeItemSemanticReconciliationExhausted, contracts.DiagnosticDispositionWarning) {
|
|
t.Fatalf("Normalize() = %#v, %v; want qualified safe fallback", result, err)
|
|
}
|
|
}
|
|
|
|
func TestNormalizeAppliesIndependentGroupAndCountsAllOmissions(t *testing.T) {
|
|
doc := semanticDocument()
|
|
ref := func(unitID int) []source.SourceRef {
|
|
return []source.SourceRef{{SourceID: doc.ID, StartUnitID: unitID, EndUnitID: unitID}}
|
|
}
|
|
input := dnd.ItemRegistry{Items: []dnd.Item{
|
|
{Name: "Star Compass", SourceRefs: ref(10)},
|
|
{Name: "Compass of the Stars", SourceRefs: ref(20)},
|
|
{Name: "Gold Pieces", SourceRefs: ref(30)},
|
|
{Name: "Silver Pieces", SourceRefs: ref(30)},
|
|
{Name: "Rope", SourceRefs: ref(10)},
|
|
}}
|
|
client := &recordingNormalizerClient{response: `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":2},{"candidate_numbers":[3,4],"canonical_candidate_number":3},{"candidate_numbers":[5,99],"canonical_candidate_number":5}]}`}
|
|
request := permissiveRequestWithSource(input, doc)
|
|
request.Operation.Controls.MaxAdditionalTurns = 1
|
|
result, err := newNormalizer(t, client).Normalize(context.Background(), request)
|
|
if err != nil || result.Outcome.Disposition != contracts.ProducerFallback {
|
|
t.Fatalf("Normalize() = %#v, %v; want qualified safe partial output", result, err)
|
|
}
|
|
wantNames := []string{"Compass of the Stars", "Gold Pieces", "Silver Pieces", "Rope"}
|
|
if len(result.Value.Items) != len(wantNames) {
|
|
t.Fatalf("items = %#v, want %v", result.Value.Items, wantNames)
|
|
}
|
|
for index, name := range wantNames {
|
|
if result.Value.Items[index].Name != name {
|
|
t.Fatalf("item %d = %#v, want %q", index, result.Value.Items[index], name)
|
|
}
|
|
}
|
|
if !hasDiagnostic(result.Diagnostics, ReasonCodeDuplicateItemCollapsed, contracts.DiagnosticDispositionObservation) || !hasDiagnostic(result.Diagnostics, ReasonCodeItemSemanticProposalInvalid, contracts.DiagnosticDispositionAdvisory) {
|
|
t.Fatalf("diagnostics = %#v, want accepted and guarded-group diagnostics", result.Diagnostics)
|
|
}
|
|
|
|
if len(client.requests) != 2 {
|
|
t.Fatalf("native calls = %d, want initial and correction", len(client.requests))
|
|
}
|
|
messages := client.requests[1].Prompt.Messages
|
|
guidance := messages[len(messages)-1].Content
|
|
for _, expected := range []string{"aliases of the same denomination", "do not combine currency with non-currency items", "candidate-number range 1 through 5", "source positions"} {
|
|
if !strings.Contains(guidance, expected) {
|
|
t.Fatalf("missing domain/shared guidance %q", expected)
|
|
}
|
|
}
|
|
if strings.Contains(guidance, "member_unknown") || strings.Contains(guidance, ReasonCodeItemSemanticProposalInvalid) {
|
|
t.Fatal("internal diagnostics copied into correction")
|
|
}
|
|
var warning string
|
|
for _, group := range result.Diagnostics {
|
|
if group.ReasonCode == ReasonCodeItemSemanticReconciliationExhausted && len(group.Samples) > 0 {
|
|
warning = group.Samples[0].Message
|
|
}
|
|
}
|
|
if !strings.Contains(warning, "2 proposal group(s)") || !strings.Contains(warning, "preserved separately") || !strings.Contains(warning, "denomination") {
|
|
t.Fatalf("fallback warning = %q", warning)
|
|
}
|
|
}
|
|
|
|
func TestPreparationLimitsFollowResolvedModeWithoutModelOrAssessment(t *testing.T) {
|
|
doc := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{{ID: 1, Kind: "narration", Text: "A crowded storeroom"}}}
|
|
limit := semanticreconcile.DefaultLimits().MaximumCandidates
|
|
input := dnd.ItemRegistry{Items: make([]dnd.Item, limit+1)}
|
|
for i := range input.Items {
|
|
name := fmt.Sprintf("Item %d", i)
|
|
input.Items[i] = dnd.Item{ID: identity.DeriveID(name), Name: name, SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 1}}}
|
|
}
|
|
for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} {
|
|
t.Run(string(mode), func(t *testing.T) {
|
|
client := &recordingNormalizerClient{}
|
|
request := normalizeRequestWithSource(input, doc)
|
|
request.Operation.Controls.Mode = mode
|
|
assessed := 0
|
|
request.Operation.Assess = func(context.Context, dnd.ItemRegistry) (contracts.CandidateAssessment, error) {
|
|
assessed++
|
|
return contracts.CandidateAssessment{}, nil
|
|
}
|
|
result, err := newNormalizer(t, client).Normalize(context.Background(), request)
|
|
if len(client.requests) != 0 || assessed != 0 {
|
|
t.Fatal("unready preparation caused model execution or ordinary validation")
|
|
}
|
|
if mode == contracts.ModeStrict {
|
|
if err == nil {
|
|
t.Fatal("strict preparation-limit failure succeeded")
|
|
}
|
|
} else if err != nil || result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || !reflect.DeepEqual(result.Value, input) {
|
|
t.Fatalf("permissive preparation limit: %#v %v", result, err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestNormalizationRemainsIdempotent(t *testing.T) {
|
|
doc := semanticDocument()
|
|
input := dnd.ItemRegistry{Items: []dnd.Item{{Name: "Star Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}}, {Name: "Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}}}}
|
|
first, err := newNormalizer(t, &recordingNormalizerClient{}).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
second, err := newNormalizer(t, &recordingNormalizerClient{}).Normalize(context.Background(), normalizeRequestWithSource(first.Value, doc))
|
|
if err != nil || !reflect.DeepEqual(first.Value, second.Value) {
|
|
t.Fatalf("repeated normalization changed output: %#v %v", second, err)
|
|
}
|
|
}
|
|
|
|
func TestNativeNormalizationPromptPairPreparesWithSameContract(t *testing.T) {
|
|
client := &recordingNormalizerClient{}
|
|
normalizer := newNormalizer(t, client)
|
|
var initial promptkit.PreparedRun
|
|
for _, id := range []string{PromptID, CorrectionPromptID} {
|
|
inputs := map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline("Keep currency denominations separate.")}
|
|
if id == PromptID {
|
|
inputs = map[string]promptkit.ArtifactRef{"candidates": promptkit.Inline(`{"candidate_number_range":{"first":1,"last":2},"candidates":[]}`), "transcript": promptkit.Inline(`{"windows":[]}`)}
|
|
}
|
|
handle, err := normalizer.engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: id, APIKey: "test-only", Inputs: inputs})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
prepared := handle.Details()
|
|
handle.Discard()
|
|
if prepared.SelectedProfileID != "notarius/dnd-extraction" || prepared.StructuredOutput == nil {
|
|
t.Fatal("missing native defaults/schema")
|
|
}
|
|
if id == PromptID {
|
|
initial = prepared
|
|
var system string
|
|
for _, message := range prepared.Messages {
|
|
if message.Role == "system" {
|
|
system += message.Content
|
|
}
|
|
}
|
|
for _, rule := range []string{"candidate_number_range", "start_unit_id", "currency denominations", "materially different item", "canonical_candidate_number"} {
|
|
if !strings.Contains(system, rule) {
|
|
t.Fatalf("missing enduring rule %q", rule)
|
|
}
|
|
}
|
|
} else if !reflect.DeepEqual(initial.OutputContract, prepared.OutputContract) || len(prepared.Messages) != 1 || prepared.Messages[0].Role != "user" || !strings.Contains(prepared.Messages[0].Content, "Keep currency denominations separate.") {
|
|
t.Fatal("correction did not preserve the output contract or supply only findings/assignment")
|
|
}
|
|
}
|
|
if len(client.requests) != 0 {
|
|
t.Fatal("offline preparation invoked generation")
|
|
}
|
|
}
|
|
|
|
type recordingNormalizerClient struct {
|
|
response string
|
|
responses []string
|
|
err error
|
|
errors []error
|
|
requests []promptkit.GenerateRequest
|
|
onGenerate func(context.Context, promptkit.GenerateRequest)
|
|
}
|
|
|
|
func (c *recordingNormalizerClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
index := len(c.requests)
|
|
c.requests = append(c.requests, request)
|
|
if c.onGenerate != nil {
|
|
c.onGenerate(ctx, request)
|
|
}
|
|
if c.err != nil {
|
|
return nil, c.err
|
|
}
|
|
if index < len(c.errors) && c.errors[index] != nil {
|
|
return nil, c.errors[index]
|
|
}
|
|
response := c.response
|
|
if index < len(c.responses) {
|
|
response = c.responses[index]
|
|
}
|
|
if response == "" {
|
|
response = `{"duplicate_groups":[]}`
|
|
}
|
|
return &promptkit.GenerateResponse{Content: response}, nil
|
|
}
|
|
|
|
func newNormalizer(t *testing.T, client promptkit.LLMClient) *Normalizer {
|
|
t.Helper()
|
|
type settings struct {
|
|
PromptKit promptkit.Settings `yaml:"promptkit"`
|
|
}
|
|
definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit })
|
|
filesystem := configkit.NewFilesystem("Notarius assets", assets.FS())
|
|
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("normalizer fixture", []byte(`promptkit:
|
|
profiles:
|
|
- id: item-normalize-test
|
|
endpoint: https://example.invalid/v1
|
|
model: normalization-test-model
|
|
api_key_required: false
|
|
`))}, configkit.LayeredOptions[settings]{Definition: definition, Underlays: []configkit.Underlay{
|
|
promptkit.BundledOpenRouterUnderlay(),
|
|
{Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}},
|
|
}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
normalizer, err := New(engine, Options{})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return normalizer
|
|
}
|
|
|
|
func renderedInput(t *testing.T, request promptkit.GenerateRequest, name string) []byte {
|
|
t.Helper()
|
|
key := "candidate_number_range"
|
|
if name == "transcript" {
|
|
key = "windows"
|
|
}
|
|
for _, message := range request.Prompt.Messages {
|
|
start := strings.Index(message.Content, "{")
|
|
if start < 0 {
|
|
continue
|
|
}
|
|
content := []byte(message.Content[start:])
|
|
var fields map[string]json.RawMessage
|
|
if json.Unmarshal(content, &fields) == nil && fields[key] != nil {
|
|
return content
|
|
}
|
|
}
|
|
t.Fatalf("missing rendered %s input", name)
|
|
return nil
|
|
}
|
|
func normalizeRequest(value dnd.ItemRegistry) contracts.TypedNormalizeRequest[dnd.ItemRegistry] {
|
|
return contracts.TypedNormalizeRequest[dnd.ItemRegistry]{
|
|
Source: &source.SourceDocument{},
|
|
LLMProfile: "item-normalize-test",
|
|
Operation: contracts.ProducerOperation[dnd.ItemRegistry]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, Cache: contracts.CacheBypass}, ReuseEligible: true, Assess: func(ctx context.Context, value dnd.ItemRegistry) (contracts.CandidateAssessment, error) {
|
|
result, err := identityvalidator.New(identityvalidator.Options{}).Validate(ctx, contracts.TypedValidationRequest[dnd.ItemRegistry]{Value: value, Stage: string(pipeline.StageNormalize)})
|
|
if err != nil {
|
|
return contracts.CandidateAssessment{}, err
|
|
}
|
|
decision := contracts.AssessmentApproved
|
|
if !result.Approved {
|
|
decision = contracts.AssessmentRejected
|
|
}
|
|
return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{ValidatorName: identityvalidator.Key, Decision: decision, Result: result}}}, nil
|
|
}},
|
|
MergeOutput: contracts.MergeArtifact[dnd.ItemRegistry]{Value: value},
|
|
}
|
|
}
|
|
func normalizeRequestWithSource(value dnd.ItemRegistry, doc *source.SourceDocument) contracts.TypedNormalizeRequest[dnd.ItemRegistry] {
|
|
request := normalizeRequest(value)
|
|
request.Source = doc
|
|
return request
|
|
}
|
|
func semanticDocument() *source.SourceDocument {
|
|
return &source.SourceDocument{ID: "item-session", Units: []source.SourceUnit{{ID: 10, Text: "The Star Compass points north."}, {ID: 20, Text: "The compass of the stars glows."}, {ID: 30, Text: "The chest holds gold pieces."}}}
|
|
}
|
|
func hasDiagnostic(diagnostics []contracts.ProducerDiagnostic, reason string, disposition contracts.DiagnosticDisposition) bool {
|
|
for _, diagnostic := range diagnostics {
|
|
if diagnostic.ReasonCode == reason && diagnostic.Disposition == disposition {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func permissiveRequestWithSource(value dnd.ItemRegistry, doc *source.SourceDocument) contracts.TypedNormalizeRequest[dnd.ItemRegistry] {
|
|
request := normalizeRequestWithSource(value, doc)
|
|
request.Operation.Controls.Mode = contracts.ModePermissive
|
|
return request
|
|
}
|