164 lines
6.2 KiB
Go
164 lines
6.2 KiB
Go
package semanticreconcile
|
|
|
|
import (
|
|
"context"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/assets"
|
|
"go.maximumdirect.net/configkit"
|
|
"go.maximumdirect.net/promptkit"
|
|
)
|
|
|
|
type nativeReconciliationClient struct {
|
|
requests []promptkit.GenerateRequest
|
|
output string
|
|
}
|
|
|
|
func (client *nativeReconciliationClient) Generate(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
client.requests = append(client.requests, request)
|
|
return &promptkit.GenerateResponse{Content: client.output}, nil
|
|
}
|
|
|
|
func newReconciliationEngine(t *testing.T, client promptkit.LLMClient) *promptkit.Engine {
|
|
t.Helper()
|
|
type settings struct {
|
|
PromptKit promptkit.Settings `yaml:"promptkit"`
|
|
}
|
|
definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit })
|
|
filesystem := configkit.NewFilesystem("Notarius assets", assets.FS())
|
|
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("reconciliation fixture", []byte(`promptkit:
|
|
profiles:
|
|
- id: test-profile
|
|
endpoint: https://example.invalid/v1
|
|
model: reconciliation-test-model
|
|
api_key_required: false
|
|
`))}, configkit.LayeredOptions[settings]{
|
|
Definition: definition,
|
|
Underlays: []configkit.Underlay{
|
|
promptkit.BundledOpenRouterUnderlay(),
|
|
{Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}},
|
|
}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return engine
|
|
}
|
|
|
|
func TestNativeRequestUsesFixedPreparedEvidenceAndRouting(t *testing.T) {
|
|
preparation := proposalPreparation(t)
|
|
before := preparation.Materials()
|
|
request, err := InitialRequest(preparation, PromptID, "test-profile", "session-1")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if request.PromptID != PromptID || request.ProfileID != "test-profile" || request.SessionID != "session-1" {
|
|
t.Fatalf("routing = %#v", request)
|
|
}
|
|
client := &nativeReconciliationClient{output: `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":2}]}`}
|
|
engine := newReconciliationEngine(t, client)
|
|
result, err := engine.Run(context.Background(), request)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result == nil || !result.Validation.IsValid {
|
|
t.Fatalf("invalid native result: %#v", result)
|
|
}
|
|
response, err := DecodeProposal(result.RawOutput)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
assessment := preparation.Assess(response)
|
|
if got, want := snapshotPlan(assessment.Plan()), []planGroupSnapshot{{members: []int{0, 2}, canonical: 2}}; !reflect.DeepEqual(got, want) {
|
|
t.Fatalf("plan = %#v, want %#v", got, want)
|
|
}
|
|
if len(client.requests) != 1 || client.requests[0].Prompt.SessionID != "session-1" || client.requests[0].Target.Model != "reconciliation-test-model" {
|
|
t.Fatal("native provider routing was not preserved")
|
|
}
|
|
var text string
|
|
for _, message := range client.requests[0].Prompt.Messages {
|
|
text += message.Content
|
|
}
|
|
for _, content := range before {
|
|
if !strings.Contains(text, string(content)) {
|
|
t.Fatal("request changed prepared evidence")
|
|
}
|
|
}
|
|
returned := preparation.Materials()
|
|
returned[candidateInputName][0] = 'x'
|
|
request.Inputs[candidateInputName] = promptkit.Inline("caller mutation")
|
|
if !reflect.DeepEqual(preparation.Materials(), before) {
|
|
t.Fatal("request exposed retained preparation")
|
|
}
|
|
}
|
|
|
|
func TestRequestConstructionRejectsUnreadyPreparation(t *testing.T) {
|
|
for _, preparation := range []Preparation{{}, {disposition: InsufficientCandidates}, {disposition: LimitExceeded}} {
|
|
request, err := InitialRequest(preparation, PromptID, "test-profile", "")
|
|
if err == nil || request.PromptID != "" {
|
|
t.Fatal("unready material produced a model request")
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestNativeStructuralFailureIsNotAReconciliationRetry(t *testing.T) {
|
|
client := &nativeReconciliationClient{output: `{"duplicate_groups":null}`}
|
|
engine := newReconciliationEngine(t, client)
|
|
request, err := InitialRequest(proposalPreparation(t), PromptID, "test-profile", "")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// A test-selected native budget gives a terminal result without depending on
|
|
// the upstream default number of structural repair calls.
|
|
zero := 0
|
|
request.Validation = &promptkit.OutputContractInput{
|
|
Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema,
|
|
SchemaPath: "generic/normalize/deduplication/schemas/semantic_reconciliation_llm.v1.json", RepairAttempts: &zero,
|
|
}
|
|
result, err := engine.Run(context.Background(), request)
|
|
if err == nil && (result == nil || result.Validation.Status != promptkit.ValidationFailed || result.Validation.IsValid) {
|
|
t.Fatalf("native structural failure lost: %#v %v", result, err)
|
|
}
|
|
if len(client.requests) != 1 {
|
|
t.Fatalf("native test budget not respected: %d", len(client.requests))
|
|
}
|
|
}
|
|
|
|
func TestDecodeProposalReportsMalformedOutput(t *testing.T) {
|
|
if _, err := DecodeProposal("{"); err == nil {
|
|
t.Fatal("malformed JSON decoded")
|
|
}
|
|
}
|
|
|
|
func TestRepeatedProposalsUseTheSameMappingWithoutPlanUnion(t *testing.T) {
|
|
preparation := proposalPreparation(t)
|
|
before := preparation.CandidateMappings()
|
|
first, err := DecodeProposal(`{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":1},{"candidate_numbers":[4,99],"canonical_candidate_number":4}]}`)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
earlier := preparation.Assess(first)
|
|
if earlier.DiscardedGroupCount() != 1 || len(earlier.Plan().Groups()) != 1 {
|
|
t.Fatalf("first assessment = %#v", earlier)
|
|
}
|
|
second, err := DecodeProposal(`{"duplicate_groups":[{"candidate_numbers":[4,5],"canonical_candidate_number":5}]}`)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
current := preparation.Assess(second)
|
|
if got, want := snapshotPlan(current.Plan()), []planGroupSnapshot{{members: []int{5, 6}, canonical: 6}}; !reflect.DeepEqual(got, want) {
|
|
t.Fatalf("current plan retained or remapped earlier groups: %#v", got)
|
|
}
|
|
if !reflect.DeepEqual(preparation.CandidateMappings(), before) {
|
|
t.Fatal("proposal assessment changed fixed handles")
|
|
}
|
|
if len(earlier.Plan().Groups()) != 1 || earlier.DiscardedGroupCount() != 1 {
|
|
t.Fatal("later proposal mutated earlier assessment")
|
|
}
|
|
}
|