249 lines
11 KiB
Go
249 lines
11 KiB
Go
|
|
package itemregistry
|
||
|
|
|
||
|
|
import (
|
||
|
|
"context"
|
||
|
|
"errors"
|
||
|
|
"reflect"
|
||
|
|
"strings"
|
||
|
|
"testing"
|
||
|
|
|
||
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||
|
|
itemcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/itemregistry"
|
||
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity"
|
||
|
|
"go.maximumdirect.net/promptkit"
|
||
|
|
)
|
||
|
|
|
||
|
|
const safeFirstProposal = `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":2}]}`
|
||
|
|
const invalidItemProposal = `{"duplicate_groups":[{"candidate_numbers":[1,99],"canonical_candidate_number":1}]}`
|
||
|
|
|
||
|
|
func mergedItems() dnd.ItemRegistry {
|
||
|
|
doc := semanticDocument()
|
||
|
|
result := dnd.ItemRegistry{Items: []dnd.Item{
|
||
|
|
{Name: "Star Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}},
|
||
|
|
{Name: "Compass of the Stars", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}},
|
||
|
|
{Name: "Astral Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}},
|
||
|
|
}}
|
||
|
|
for i := range result.Items {
|
||
|
|
result.Items[i].ID = identity.DeriveID(result.Items[i].Name)
|
||
|
|
}
|
||
|
|
return result
|
||
|
|
}
|
||
|
|
|
||
|
|
func itemNames(value dnd.ItemRegistry) []string {
|
||
|
|
names := make([]string, len(value.Items))
|
||
|
|
for i, item := range value.Items {
|
||
|
|
names[i] = item.Name
|
||
|
|
}
|
||
|
|
return names
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestNativeNormalizationReassessesRevisionsAndUsesOnlyTheFinalProposal(t *testing.T) {
|
||
|
|
finalPartial := `{"duplicate_groups":[{"candidate_numbers":[2,3],"canonical_candidate_number":3},{"candidate_numbers":[1,99],"canonical_candidate_number":1}]}`
|
||
|
|
for _, test := range []struct {
|
||
|
|
name string
|
||
|
|
mode contracts.ExecutionMode
|
||
|
|
responses []string
|
||
|
|
rejectFirst bool
|
||
|
|
wantError bool
|
||
|
|
wantFallback bool
|
||
|
|
wantNames []string
|
||
|
|
}{
|
||
|
|
{"corrected proposal accepted", contracts.ModeStrict, []string{invalidItemProposal, safeFirstProposal}, false, false, false, []string{"Compass of the Stars", "Astral Compass"}},
|
||
|
|
{"unchanged candidate reassessed", contracts.ModeStrict, []string{invalidItemProposal, invalidItemProposal}, false, true, false, nil},
|
||
|
|
{"final partial proposal only", contracts.ModePermissive, []string{safeFirstProposal, finalPartial}, true, false, true, []string{"Star Compass", "Astral Compass"}},
|
||
|
|
} {
|
||
|
|
t.Run(test.name, func(t *testing.T) {
|
||
|
|
input := mergedItems()
|
||
|
|
before := cloneRegistry(input)
|
||
|
|
client := &recordingNormalizerClient{responses: test.responses}
|
||
|
|
request := normalizeRequestWithSource(input, semanticDocument())
|
||
|
|
request.Operation.Controls.Mode = test.mode
|
||
|
|
request.Operation.Controls.MaxAdditionalTurns = len(test.responses) - 1
|
||
|
|
request.SessionID = "normalization-session"
|
||
|
|
model := "run-wide-normalization-model"
|
||
|
|
request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: model}
|
||
|
|
var prepared []promptkit.PreparedRun
|
||
|
|
request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = append(prepared, value) }
|
||
|
|
base := request.Operation.Assess
|
||
|
|
var assessed []dnd.ItemRegistry
|
||
|
|
request.Operation.Assess = func(ctx context.Context, value dnd.ItemRegistry) (contracts.CandidateAssessment, error) {
|
||
|
|
assessed = append(assessed, cloneRegistry(value))
|
||
|
|
assessment, err := base(ctx, value)
|
||
|
|
if test.rejectFirst {
|
||
|
|
finding := contracts.ValidatorAssessment{ValidatorName: "configured-domain-policy", Decision: contracts.AssessmentApproved, Result: contracts.ValidationResult{Approved: true}}
|
||
|
|
if len(assessed) == 1 {
|
||
|
|
finding.Decision = contracts.AssessmentRejected
|
||
|
|
finding.Result = contracts.ValidationResult{Message: "operator-only-rejection", CorrectionGuidance: "Reconsider the first compass grouping."}
|
||
|
|
}
|
||
|
|
assessment.Validators = append(assessment.Validators, finding)
|
||
|
|
}
|
||
|
|
return assessment, err
|
||
|
|
}
|
||
|
|
result, err := newNormalizer(t, client).Normalize(context.Background(), request)
|
||
|
|
if (err != nil) != test.wantError {
|
||
|
|
t.Fatalf("result = %#v %v", result, err)
|
||
|
|
}
|
||
|
|
if len(client.requests) != len(test.responses) || len(assessed) != len(test.responses) || len(prepared) != len(test.responses) {
|
||
|
|
t.Fatalf("provider/assessment/preparation = %d/%d/%d, want %d", len(client.requests), len(assessed), len(prepared), len(test.responses))
|
||
|
|
}
|
||
|
|
if !reflect.DeepEqual(input, before) {
|
||
|
|
t.Fatal("proposal mutated fixed originals")
|
||
|
|
}
|
||
|
|
if test.name == "unchanged candidate reassessed" && !reflect.DeepEqual(assessed[0], assessed[1]) {
|
||
|
|
t.Fatal("fixture did not exercise an unchanged typed candidate")
|
||
|
|
}
|
||
|
|
for i, call := range client.requests {
|
||
|
|
if call.Target.Model != model || call.Prompt.SessionID != "normalization-session" || prepared[i].SelectedProfileID != "item-normalize-test" {
|
||
|
|
t.Fatal("native override lost")
|
||
|
|
}
|
||
|
|
wantID := PromptID
|
||
|
|
if i > 0 {
|
||
|
|
wantID = CorrectionPromptID
|
||
|
|
}
|
||
|
|
if prepared[i].PromptID != wantID {
|
||
|
|
t.Fatalf("prompt = %q, want %q", prepared[i].PromptID, wantID)
|
||
|
|
}
|
||
|
|
if i == 0 {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
old := client.requests[i-1].Prompt.Messages
|
||
|
|
current := call.Prompt.Messages
|
||
|
|
if len(current) != len(old)+2 || !reflect.DeepEqual(current[:len(old)], old) {
|
||
|
|
t.Fatal("native continuation changed fixed evidence/history")
|
||
|
|
}
|
||
|
|
if current[len(old)].Role != "assistant" || current[len(old)].Content != test.responses[i-1] {
|
||
|
|
t.Fatal("native continuation lost the prior response")
|
||
|
|
}
|
||
|
|
if strings.Contains(current[len(old)+1].Content, "operator-only-rejection") {
|
||
|
|
t.Fatal("operator diagnostic became model guidance")
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if test.wantError {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
if !reflect.DeepEqual(itemNames(result.Value), test.wantNames) || (result.Outcome.Disposition == contracts.ProducerFallback) != test.wantFallback || result.Outcome.ReuseEligible == test.wantFallback {
|
||
|
|
t.Fatalf("final outcome = %#v", result)
|
||
|
|
}
|
||
|
|
if _, err := itemcodec.New().Encode(result.Value); err != nil {
|
||
|
|
t.Fatalf("durable artifact encoding: %v", err)
|
||
|
|
}
|
||
|
|
if test.wantFallback {
|
||
|
|
refs := result.Value.Items[1].SourceRefs
|
||
|
|
if len(refs) != 2 || refs[0].StartUnitID != 20 || refs[1].StartUnitID != 30 {
|
||
|
|
t.Fatalf("final proposal carried earlier evidence: %#v", refs)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
})
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestNativeFailureUsesMergedInputWithoutRecoveringEarlierProposal(t *testing.T) {
|
||
|
|
failure := errors.New("provider unavailable")
|
||
|
|
for _, failCorrection := range []bool{false, true} {
|
||
|
|
for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} {
|
||
|
|
input := mergedItems()
|
||
|
|
client := &recordingNormalizerClient{err: failure}
|
||
|
|
if failCorrection {
|
||
|
|
client.err = nil
|
||
|
|
client.responses = []string{safeFirstProposal}
|
||
|
|
client.errors = []error{nil, failure}
|
||
|
|
}
|
||
|
|
request := normalizeRequestWithSource(input, semanticDocument())
|
||
|
|
request.Operation.Controls.Mode = mode
|
||
|
|
request.Operation.Controls.MaxAdditionalTurns = 1
|
||
|
|
assessments := 0
|
||
|
|
request.Operation.Assess = func(context.Context, dnd.ItemRegistry) (contracts.CandidateAssessment, error) {
|
||
|
|
assessments++
|
||
|
|
return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{ValidatorName: "policy", Decision: contracts.AssessmentRejected, Result: contracts.ValidationResult{CorrectionGuidance: "Keep the distinct compass records."}}}}, nil
|
||
|
|
}
|
||
|
|
result, err := newNormalizer(t, client).Normalize(context.Background(), request)
|
||
|
|
wantCalls, wantAssess := 1, 0
|
||
|
|
if failCorrection {
|
||
|
|
wantCalls, wantAssess = 2, 1
|
||
|
|
}
|
||
|
|
if len(client.requests) != wantCalls || assessments != wantAssess {
|
||
|
|
t.Fatalf("calls = %d/%d, want %d/%d", len(client.requests), assessments, wantCalls, wantAssess)
|
||
|
|
}
|
||
|
|
if mode == contracts.ModeStrict {
|
||
|
|
if !errors.Is(err, failure) {
|
||
|
|
t.Fatalf("strict failure = %v", err)
|
||
|
|
}
|
||
|
|
} else if err != nil || !reflect.DeepEqual(result.Value, input) || result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || hasDiagnostic(result.Diagnostics, ReasonCodeDuplicateItemCollapsed, contracts.DiagnosticDispositionObservation) {
|
||
|
|
t.Fatalf("fallback recovered an earlier proposal: %#v %v", result, err)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestNativeStructuralFailureProducesNoSemanticCandidate(t *testing.T) {
|
||
|
|
input := mergedItems()
|
||
|
|
client := &recordingNormalizerClient{response: `{"duplicate_groups":null}`}
|
||
|
|
request := permissiveRequestWithSource(input, semanticDocument())
|
||
|
|
request.Operation.Controls.MaxAdditionalTurns = 2
|
||
|
|
assessed := 0
|
||
|
|
request.Operation.Assess = func(context.Context, dnd.ItemRegistry) (contracts.CandidateAssessment, error) {
|
||
|
|
assessed++
|
||
|
|
return contracts.CandidateAssessment{}, nil
|
||
|
|
}
|
||
|
|
result, err := newNormalizer(t, client).Normalize(context.Background(), request)
|
||
|
|
if err != nil || assessed != 0 || !reflect.DeepEqual(result.Value, input) || result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible {
|
||
|
|
t.Fatalf("native structural failure = %#v %v, assessed %d", result, err, assessed)
|
||
|
|
}
|
||
|
|
for _, call := range client.requests {
|
||
|
|
for _, message := range call.Prompt.Messages {
|
||
|
|
if strings.Contains(message.Content, "Revise the previous duplicate-entity proposal using these findings:") {
|
||
|
|
t.Fatal("native structural failure started semantic continuation")
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestInsufficientCandidatesUseAssessmentWithoutNativeExecution(t *testing.T) {
|
||
|
|
input := mergedItems()
|
||
|
|
input.Items = input.Items[:1]
|
||
|
|
client := &recordingNormalizerClient{}
|
||
|
|
request := normalizeRequestWithSource(input, semanticDocument())
|
||
|
|
base := request.Operation.Assess
|
||
|
|
assessed := 0
|
||
|
|
request.Operation.Assess = func(ctx context.Context, value dnd.ItemRegistry) (contracts.CandidateAssessment, error) {
|
||
|
|
assessed++
|
||
|
|
return base(ctx, value)
|
||
|
|
}
|
||
|
|
result, err := newNormalizer(t, client).Normalize(context.Background(), request)
|
||
|
|
if err != nil || len(client.requests) != 0 || assessed != 1 || result.Outcome.Disposition != contracts.ProducerAccepted || !result.Outcome.ReuseEligible || !reflect.DeepEqual(result.Value, input) {
|
||
|
|
t.Fatalf("no-model outcome = %#v %v, assessed %d", result, err, assessed)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestCancellationNeverDeliversNormalizationFallback(t *testing.T) {
|
||
|
|
ctx, cancel := context.WithCancel(context.Background())
|
||
|
|
defer cancel()
|
||
|
|
client := &recordingNormalizerClient{onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }}
|
||
|
|
result, err := newNormalizer(t, client).Normalize(ctx, permissiveRequestWithSource(mergedItems(), semanticDocument()))
|
||
|
|
if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" || len(client.requests) != 1 {
|
||
|
|
t.Fatalf("cancellation = %#v %v, calls %d", result, err, len(client.requests))
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestFallbackCannotPublishAnUnusableMergedRegistry(t *testing.T) {
|
||
|
|
failure := errors.New("provider unavailable")
|
||
|
|
for _, test := range []struct {
|
||
|
|
name string
|
||
|
|
mutate func(*dnd.ItemRegistry)
|
||
|
|
}{
|
||
|
|
{"identity mismatch", func(value *dnd.ItemRegistry) { value.Items[0].ID = identity.DeriveID("Wrong identity") }},
|
||
|
|
{"foreign source", func(value *dnd.ItemRegistry) { value.Items[0].SourceRefs[0].SourceID = "other-session" }},
|
||
|
|
} {
|
||
|
|
t.Run(test.name, func(t *testing.T) {
|
||
|
|
input := mergedItems()
|
||
|
|
test.mutate(&input)
|
||
|
|
result, err := newNormalizer(t, &recordingNormalizerClient{err: failure}).Normalize(context.Background(), permissiveRequestWithSource(input, semanticDocument()))
|
||
|
|
if !errors.Is(err, failure) || result.Outcome.Disposition != "" {
|
||
|
|
t.Fatalf("invalid fallback = %#v %v", result, err)
|
||
|
|
}
|
||
|
|
})
|
||
|
|
}
|
||
|
|
}
|