200 lines
9.3 KiB
Go
200 lines
9.3 KiB
Go
|
|
package locationregistry
|
||
|
|
|
||
|
|
import (
|
||
|
|
"context"
|
||
|
|
"errors"
|
||
|
|
"reflect"
|
||
|
|
"strings"
|
||
|
|
"testing"
|
||
|
|
|
||
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/locations/identity"
|
||
|
|
"go.maximumdirect.net/promptkit"
|
||
|
|
)
|
||
|
|
|
||
|
|
const safeProposal = `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":2}]}`
|
||
|
|
const unknownHandleProposal = `{"duplicate_groups":[{"candidate_numbers":[1,99],"canonical_candidate_number":1}]}`
|
||
|
|
|
||
|
|
func mergedRegistry() dnd.LocationRegistry {
|
||
|
|
doc := semanticDocument()
|
||
|
|
result := dnd.LocationRegistry{Locations: []dnd.Location{
|
||
|
|
{Name: "Old Mill", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}},
|
||
|
|
{Name: "Greencloak Refuge", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}},
|
||
|
|
{Name: "The Tavern", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}},
|
||
|
|
}}
|
||
|
|
for i := range result.Locations {
|
||
|
|
result.Locations[i].ID = identity.DeriveID(result.Locations[i].Name, result.Locations[i].SourceRefs)
|
||
|
|
}
|
||
|
|
return result
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestNativeCorrectionPreservesFixedHistoryAndAssessesEachRevision(t *testing.T) {
|
||
|
|
finalPartial := `{"duplicate_groups":[{"candidate_numbers":[2,3],"canonical_candidate_number":3},{"candidate_numbers":[1,99],"canonical_candidate_number":1}]}`
|
||
|
|
for _, test := range []struct {
|
||
|
|
name string
|
||
|
|
mode contracts.ExecutionMode
|
||
|
|
responses []string
|
||
|
|
rejectFirst, wantError, wantFallback bool
|
||
|
|
names []string
|
||
|
|
}{
|
||
|
|
{"corrected", contracts.ModeStrict, []string{unknownHandleProposal, safeProposal}, false, false, false, []string{"Greencloak Refuge", "The Tavern"}},
|
||
|
|
{"unchanged strict rejection", contracts.ModeStrict, []string{unknownHandleProposal, unknownHandleProposal}, false, true, false, nil},
|
||
|
|
{"only final safe groups", contracts.ModePermissive, []string{safeProposal, finalPartial}, true, false, true, []string{"Old Mill", "The Tavern"}},
|
||
|
|
} {
|
||
|
|
t.Run(test.name, func(t *testing.T) {
|
||
|
|
input := mergedRegistry()
|
||
|
|
before := cloneRegistry(input)
|
||
|
|
client := &recordingLocationNormalizerClient{responses: test.responses}
|
||
|
|
request := normalizeRequestWithSource(input, semanticDocument())
|
||
|
|
request.Operation.Controls.Mode = test.mode
|
||
|
|
request.Operation.Controls.MaxAdditionalTurns = 1
|
||
|
|
request.SessionID = "normalization-session"
|
||
|
|
request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "normalization-override"}
|
||
|
|
var prepared []promptkit.PreparedRun
|
||
|
|
request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = append(prepared, value) }
|
||
|
|
var assessed []dnd.LocationRegistry
|
||
|
|
request.Operation.Assess = func(_ context.Context, value dnd.LocationRegistry) (contracts.CandidateAssessment, error) {
|
||
|
|
assessed = append(assessed, cloneRegistry(value))
|
||
|
|
if test.rejectFirst && len(assessed) == 1 {
|
||
|
|
return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{
|
||
|
|
ValidatorName: "domain-policy", Decision: contracts.AssessmentRejected,
|
||
|
|
Result: contracts.ValidationResult{Message: "operator-only-rejection", CorrectionGuidance: "Reconsider the proposed grouping."},
|
||
|
|
}}}, nil
|
||
|
|
}
|
||
|
|
return contracts.CandidateAssessment{}, nil
|
||
|
|
}
|
||
|
|
result, err := newNormalizer(t, client).Normalize(context.Background(), request)
|
||
|
|
if (err != nil) != test.wantError || len(assessed) != 2 || len(client.requests) != 2 || len(prepared) != 2 {
|
||
|
|
t.Fatalf("result=%#v error=%v assessments/provider/prepared=%d/%d/%d", result, err, len(assessed), len(client.requests), len(prepared))
|
||
|
|
}
|
||
|
|
if !reflect.DeepEqual(input, before) {
|
||
|
|
t.Fatal("normalization mutated fixed originals")
|
||
|
|
}
|
||
|
|
if test.name == "unchanged strict rejection" && !reflect.DeepEqual(assessed[0], assessed[1]) {
|
||
|
|
t.Fatal("fixture did not assess an unchanged candidate")
|
||
|
|
}
|
||
|
|
for i, call := range client.requests {
|
||
|
|
wantID := PromptID
|
||
|
|
if i > 0 {
|
||
|
|
wantID = CorrectionPromptID
|
||
|
|
}
|
||
|
|
if prepared[i].PromptID != wantID || prepared[i].SelectedProfileID != request.LLMProfile || call.Target.Model != "normalization-override" || call.Prompt.SessionID != request.SessionID {
|
||
|
|
t.Fatal("native request selection/overrides lost")
|
||
|
|
}
|
||
|
|
}
|
||
|
|
initial, revised := client.requests[0].Prompt.Messages, client.requests[1].Prompt.Messages
|
||
|
|
if len(revised) != len(initial)+2 || !reflect.DeepEqual(revised[:len(initial)], initial) ||
|
||
|
|
revised[len(initial)].Role != "assistant" || revised[len(initial)].Content != test.responses[0] {
|
||
|
|
t.Fatal("continuation changed original evidence or lost the prior response")
|
||
|
|
}
|
||
|
|
if strings.Contains(revised[len(initial)+1].Content, "operator-only-rejection") {
|
||
|
|
t.Fatal("operator text became correction guidance")
|
||
|
|
}
|
||
|
|
if test.wantError {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
names := make([]string, len(result.Value.Locations))
|
||
|
|
for i, record := range result.Value.Locations {
|
||
|
|
names[i] = record.Name
|
||
|
|
}
|
||
|
|
if !reflect.DeepEqual(names, test.names) || (result.Outcome.Disposition == contracts.ProducerFallback) != test.wantFallback || result.Outcome.ReuseEligible == test.wantFallback {
|
||
|
|
t.Fatalf("unexpected final output: %#v", result)
|
||
|
|
}
|
||
|
|
if test.wantFallback {
|
||
|
|
refs := result.Value.Locations[1].SourceRefs
|
||
|
|
if len(refs) != 2 || refs[0].StartUnitID != 20 || refs[1].StartUnitID != 30 {
|
||
|
|
t.Fatalf("earlier proposal evidence leaked into final output: %#v", refs)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
})
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestNativeFailureCannotRecoverEarlierProposal(t *testing.T) {
|
||
|
|
failure := errors.New("provider unavailable")
|
||
|
|
for _, test := range []struct {
|
||
|
|
name string
|
||
|
|
response string
|
||
|
|
providerErrors []error
|
||
|
|
wantAssessments int
|
||
|
|
}{
|
||
|
|
{"initial provider failure", "", []error{failure}, 0},
|
||
|
|
{"failed correction", safeProposal, []error{nil, failure}, 1},
|
||
|
|
{"structural failure", `{"duplicate_groups":null}`, nil, 0},
|
||
|
|
} {
|
||
|
|
for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} {
|
||
|
|
t.Run(test.name+"/"+string(mode), func(t *testing.T) {
|
||
|
|
input := mergedRegistry()
|
||
|
|
client := &recordingLocationNormalizerClient{response: test.response, errors: test.providerErrors}
|
||
|
|
request := normalizeRequestWithSource(input, semanticDocument())
|
||
|
|
request.Operation.Controls.Mode = mode
|
||
|
|
request.Operation.Controls.MaxAdditionalTurns = 1
|
||
|
|
assessments := 0
|
||
|
|
request.Operation.Assess = func(context.Context, dnd.LocationRegistry) (contracts.CandidateAssessment, error) {
|
||
|
|
assessments++
|
||
|
|
return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{
|
||
|
|
ValidatorName: "policy", Decision: contracts.AssessmentRejected,
|
||
|
|
Result: contracts.ValidationResult{CorrectionGuidance: "Keep the distinct records."},
|
||
|
|
}}}, nil
|
||
|
|
}
|
||
|
|
result, err := newNormalizer(t, client).Normalize(context.Background(), request)
|
||
|
|
if assessments != test.wantAssessments {
|
||
|
|
t.Fatalf("assessments=%d, want %d", assessments, test.wantAssessments)
|
||
|
|
}
|
||
|
|
if test.providerErrors != nil && len(client.requests) != len(test.providerErrors) {
|
||
|
|
t.Fatalf("provider restarted: %d calls", len(client.requests))
|
||
|
|
}
|
||
|
|
if mode == contracts.ModeStrict {
|
||
|
|
if err == nil || result.Outcome.Disposition != "" {
|
||
|
|
t.Fatalf("strict failure=%#v %v", result, err)
|
||
|
|
}
|
||
|
|
if test.providerErrors != nil && !errors.Is(err, failure) {
|
||
|
|
t.Fatalf("lost native cause: %v", err)
|
||
|
|
}
|
||
|
|
return
|
||
|
|
}
|
||
|
|
if err != nil || !reflect.DeepEqual(result.Value, input) || result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible {
|
||
|
|
t.Fatalf("fallback recovered an earlier proposal: %#v %v", result, err)
|
||
|
|
}
|
||
|
|
for _, diagnostic := range result.Diagnostics {
|
||
|
|
if diagnostic.ReasonCode == ReasonCodeDuplicateLocationCollapsed {
|
||
|
|
t.Fatal("discarded proposal left collapse diagnostics")
|
||
|
|
}
|
||
|
|
}
|
||
|
|
})
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestCancellationDoesNotDeliverFallback(t *testing.T) {
|
||
|
|
ctx, cancel := context.WithCancel(context.Background())
|
||
|
|
defer cancel()
|
||
|
|
client := &recordingLocationNormalizerClient{onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }}
|
||
|
|
request := normalizeRequestWithSource(mergedRegistry(), semanticDocument())
|
||
|
|
request.Operation.Controls.Mode = contracts.ModePermissive
|
||
|
|
result, err := newNormalizer(t, client).Normalize(ctx, request)
|
||
|
|
if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" || len(client.requests) != 1 {
|
||
|
|
t.Fatalf("cancellation=%#v %v", result, err)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestFallbackRequiresUsableMergedIdentityAndSource(t *testing.T) {
|
||
|
|
for _, invalidIdentity := range []bool{false, true} {
|
||
|
|
input := mergedRegistry()
|
||
|
|
if invalidIdentity {
|
||
|
|
input.Locations[0].ID = "wrong"
|
||
|
|
} else {
|
||
|
|
input.Locations[0].SourceRefs[0].SourceID = "foreign-source"
|
||
|
|
}
|
||
|
|
request := normalizeRequestWithSource(input, semanticDocument())
|
||
|
|
request.Operation.Controls.Mode = contracts.ModePermissive
|
||
|
|
failure := errors.New("provider unavailable")
|
||
|
|
result, err := newNormalizer(t, &recordingLocationNormalizerClient{err: failure}).Normalize(context.Background(), request)
|
||
|
|
if !errors.Is(err, failure) || result.Outcome.Disposition != "" {
|
||
|
|
t.Fatalf("unusable fallback=%#v %v", result, err)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|