Files
notarius/internal/framework/semanticreconcile/engine_test.go

164 lines
6.2 KiB
Go

package semanticreconcile
import (
"context"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/assets"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
type nativeReconciliationClient struct {
requests []promptkit.GenerateRequest
output string
}
func (client *nativeReconciliationClient) Generate(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
client.requests = append(client.requests, request)
return &promptkit.GenerateResponse{Content: client.output}, nil
}
func newReconciliationEngine(t *testing.T, client promptkit.LLMClient) *promptkit.Engine {
t.Helper()
type settings struct {
PromptKit promptkit.Settings `yaml:"promptkit"`
}
definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit })
filesystem := configkit.NewFilesystem("Notarius assets", assets.FS())
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("reconciliation fixture", []byte(`promptkit:
profiles:
- id: test-profile
endpoint: https://example.invalid/v1
model: reconciliation-test-model
api_key_required: false
`))}, configkit.LayeredOptions[settings]{
Definition: definition,
Underlays: []configkit.Underlay{
promptkit.BundledOpenRouterUnderlay(),
{Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}},
}})
if err != nil {
t.Fatal(err)
}
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client))
if err != nil {
t.Fatal(err)
}
return engine
}
func TestNativeRequestUsesFixedPreparedEvidenceAndRouting(t *testing.T) {
preparation := proposalPreparation(t)
before := preparation.Materials()
request, err := InitialRequest(preparation, PromptID, "test-profile", "session-1")
if err != nil {
t.Fatal(err)
}
if request.PromptID != PromptID || request.ProfileID != "test-profile" || request.SessionID != "session-1" {
t.Fatalf("routing = %#v", request)
}
client := &nativeReconciliationClient{output: `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":2}]}`}
engine := newReconciliationEngine(t, client)
result, err := engine.Run(context.Background(), request)
if err != nil {
t.Fatal(err)
}
if result == nil || !result.Validation.IsValid {
t.Fatalf("invalid native result: %#v", result)
}
response, err := DecodeProposal(result.RawOutput)
if err != nil {
t.Fatal(err)
}
assessment := preparation.Assess(response)
if got, want := snapshotPlan(assessment.Plan()), []planGroupSnapshot{{members: []int{0, 2}, canonical: 2}}; !reflect.DeepEqual(got, want) {
t.Fatalf("plan = %#v, want %#v", got, want)
}
if len(client.requests) != 1 || client.requests[0].Prompt.SessionID != "session-1" || client.requests[0].Target.Model != "reconciliation-test-model" {
t.Fatal("native provider routing was not preserved")
}
var text string
for _, message := range client.requests[0].Prompt.Messages {
text += message.Content
}
for _, content := range before {
if !strings.Contains(text, string(content)) {
t.Fatal("request changed prepared evidence")
}
}
returned := preparation.Materials()
returned[candidateInputName][0] = 'x'
request.Inputs[candidateInputName] = promptkit.Inline("caller mutation")
if !reflect.DeepEqual(preparation.Materials(), before) {
t.Fatal("request exposed retained preparation")
}
}
func TestRequestConstructionRejectsUnreadyPreparation(t *testing.T) {
for _, preparation := range []Preparation{{}, {disposition: InsufficientCandidates}, {disposition: LimitExceeded}} {
request, err := InitialRequest(preparation, PromptID, "test-profile", "")
if err == nil || request.PromptID != "" {
t.Fatal("unready material produced a model request")
}
}
}
func TestNativeStructuralFailureIsNotAReconciliationRetry(t *testing.T) {
client := &nativeReconciliationClient{output: `{"duplicate_groups":null}`}
engine := newReconciliationEngine(t, client)
request, err := InitialRequest(proposalPreparation(t), PromptID, "test-profile", "")
if err != nil {
t.Fatal(err)
}
// A test-selected native budget gives a terminal result without depending on
// the upstream default number of structural repair calls.
zero := 0
request.Validation = &promptkit.OutputContractInput{
Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema,
SchemaPath: "generic/normalize/deduplication/schemas/semantic_reconciliation_llm.v1.json", RepairAttempts: &zero,
}
result, err := engine.Run(context.Background(), request)
if err == nil && (result == nil || result.Validation.Status != promptkit.ValidationFailed || result.Validation.IsValid) {
t.Fatalf("native structural failure lost: %#v %v", result, err)
}
if len(client.requests) != 1 {
t.Fatalf("native test budget not respected: %d", len(client.requests))
}
}
func TestDecodeProposalReportsMalformedOutput(t *testing.T) {
if _, err := DecodeProposal("{"); err == nil {
t.Fatal("malformed JSON decoded")
}
}
func TestRepeatedProposalsUseTheSameMappingWithoutPlanUnion(t *testing.T) {
preparation := proposalPreparation(t)
before := preparation.CandidateMappings()
first, err := DecodeProposal(`{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":1},{"candidate_numbers":[4,99],"canonical_candidate_number":4}]}`)
if err != nil {
t.Fatal(err)
}
earlier := preparation.Assess(first)
if earlier.DiscardedGroupCount() != 1 || len(earlier.Plan().Groups()) != 1 {
t.Fatalf("first assessment = %#v", earlier)
}
second, err := DecodeProposal(`{"duplicate_groups":[{"candidate_numbers":[4,5],"canonical_candidate_number":5}]}`)
if err != nil {
t.Fatal(err)
}
current := preparation.Assess(second)
if got, want := snapshotPlan(current.Plan()), []planGroupSnapshot{{members: []int{5, 6}, canonical: 6}}; !reflect.DeepEqual(got, want) {
t.Fatalf("current plan retained or remapped earlier groups: %#v", got)
}
if !reflect.DeepEqual(preparation.CandidateMappings(), before) {
t.Fatal("proposal assessment changed fixed handles")
}
if len(earlier.Plan().Groups()) != 1 || earlier.DiscardedGroupCount() != 1 {
t.Fatal("later proposal mutated earlier assessment")
}
}