Migrate registry extraction to native correction and explicit empty fallbacks
This commit is contained in:
@@ -0,0 +1,156 @@
|
||||
package locationregistry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
registrycodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/locationregistry"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/locations/identity"
|
||||
shape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/locationregistry/shape"
|
||||
refs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/locationregistry/source_refs"
|
||||
relatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/locationregistry/source_relatedness"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
const ungroundedRegistry = `{"locations":[{"name":"Old Mill","source_refs":[{"start_unit_id":999,"end_unit_id":999}]}]}`
|
||||
const groundedRegistry = `{"locations":[{"name":"Old Mill","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`
|
||||
const emptyRegistry = `{"locations":[]}`
|
||||
|
||||
func assessRegistry(ctx context.Context, request contracts.TypedExtractionRequest[dnd.LocationRegistry], value dnd.LocationRegistry) (contracts.CandidateAssessment, error) {
|
||||
assessment := contracts.CandidateAssessment{}
|
||||
for _, validator := range []contracts.TypedValidator[dnd.LocationRegistry]{
|
||||
shape.New(shape.Options{}), refs.New(refs.Options{}), relatedness.New(relatedness.Options{}),
|
||||
} {
|
||||
result, err := validator.Validate(ctx, contracts.TypedValidationRequest[dnd.LocationRegistry]{Stage: "extract", Source: request.Source, Chunk: request.Chunk, Value: value})
|
||||
if err != nil {
|
||||
return contracts.CandidateAssessment{}, err
|
||||
}
|
||||
decision := contracts.AssessmentRejected
|
||||
if result.Approved {
|
||||
decision = contracts.AssessmentApproved
|
||||
}
|
||||
assessment.Validators = append(assessment.Validators, contracts.ValidatorAssessment{ValidatorName: validator.Name(), Decision: decision, Result: result})
|
||||
}
|
||||
return assessment, nil
|
||||
}
|
||||
|
||||
func TestNativeExtractionCorrectsGroundingWithFixedInputs(t *testing.T) {
|
||||
client := &fakeLocationsLLMClient{responses: []string{ungroundedRegistry, groundedRegistry}}
|
||||
request := extractionRequest()
|
||||
request.References = contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{"glossary": {Items: []contracts.ReferenceItem{{Content: []byte("fixed-reference")}}}}}
|
||||
request.Operation.Controls.MaxAdditionalTurns = 1
|
||||
request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "registry-override"}
|
||||
var prepared []promptkit.PreparedRun
|
||||
request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = append(prepared, value) }
|
||||
assessments := 0
|
||||
request.Operation.Assess = func(ctx context.Context, value dnd.LocationRegistry) (contracts.CandidateAssessment, error) {
|
||||
assessments++
|
||||
assessment, err := assessRegistry(ctx, request, value)
|
||||
if assessments == 1 {
|
||||
if assessment.Accepted() {
|
||||
t.Fatal("unresolvable citation was accepted")
|
||||
}
|
||||
// Changes after capture must not cause source/reference preparation again.
|
||||
request.SourceInput.Content[0] = '!'
|
||||
request.References.Slots["glossary"].Items[0].Content[0] = '!'
|
||||
}
|
||||
return assessment, err
|
||||
}
|
||||
result, err := newExtractor(t, client).Extract(context.Background(), request)
|
||||
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || !result.Outcome.ReuseEligible ||
|
||||
assessments != 2 || len(client.requests) != 2 || len(prepared) != 2 || len(result.Value.Locations) != 1 {
|
||||
t.Fatalf("corrected registry=%#v %v, assessments/provider/prepared=%d/%d/%d", result, err, assessments, len(client.requests), len(prepared))
|
||||
}
|
||||
record := result.Value.Locations[0]
|
||||
if record.Name != "Old Mill" || len(record.SourceRefs) != 1 || record.SourceRefs[0].SourceID != request.Source.ID ||
|
||||
record.SourceRefs[0].StartUnitID != 1 || record.SourceRefs[0].EndUnitID != 1 || record.ID != identity.DeriveID(record.Name, record.SourceRefs) {
|
||||
t.Fatalf("canonical record identity/evidence=%#v", record)
|
||||
}
|
||||
if _, err := registrycodec.New().Encode(result.Value); err != nil {
|
||||
t.Fatalf("durable registry encoding: %v", err)
|
||||
}
|
||||
for i, call := range client.requests {
|
||||
id := PromptID
|
||||
if i > 0 {
|
||||
id = CorrectionPromptID
|
||||
}
|
||||
if prepared[i].PromptID != id || prepared[i].SelectedProfileID != request.LLMProfile || call.Target.Model != "registry-override" || call.Prompt.SessionID != request.SessionID {
|
||||
t.Fatal("native selection/overrides lost")
|
||||
}
|
||||
}
|
||||
initial, revised := client.requests[0].Prompt.Messages, client.requests[1].Prompt.Messages
|
||||
if len(revised) != len(initial)+2 || !reflect.DeepEqual(revised[:len(initial)], initial) ||
|
||||
revised[len(initial)].Role != "assistant" || revised[len(initial)].Content != ungroundedRegistry {
|
||||
t.Fatal("native correction lost fixed inputs or prior response")
|
||||
}
|
||||
messageContaining(t, client.requests[1], "fixed-reference")
|
||||
messageContaining(t, client.requests[1], string(request.Chunk.Content))
|
||||
}
|
||||
|
||||
func TestNativeExtractionDistinguishesEmptyAcceptanceFromFallback(t *testing.T) {
|
||||
failure := errors.New("provider unavailable")
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
mode contracts.ExecutionMode
|
||||
responses []string
|
||||
providerErrors []error
|
||||
wantAssessments, wantRuns int
|
||||
wantError, wantFallback bool
|
||||
}{
|
||||
{"ordinary empty", contracts.ModeStrict, []string{emptyRegistry}, nil, 1, 1, false, false},
|
||||
{"strict rejection", contracts.ModeStrict, []string{ungroundedRegistry, ungroundedRegistry}, nil, 2, 2, true, false},
|
||||
{"permissive rejection", contracts.ModePermissive, []string{ungroundedRegistry, ungroundedRegistry}, nil, 2, 2, false, true},
|
||||
{"provider failure", contracts.ModePermissive, nil, []error{failure}, 0, 1, false, true},
|
||||
{"failed correction", contracts.ModePermissive, []string{ungroundedRegistry}, []error{nil, failure}, 1, 2, false, true},
|
||||
{"structural failure", contracts.ModePermissive, []string{`{"locations":null}`}, nil, 0, 1, false, true},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
client := &fakeLocationsLLMClient{responses: test.responses, errors: test.providerErrors}
|
||||
if test.name == "structural failure" {
|
||||
client.content = []byte(test.responses[0])
|
||||
}
|
||||
request := extractionRequest()
|
||||
request.Operation.Controls.Mode = test.mode
|
||||
request.Operation.Controls.MaxAdditionalTurns = 1
|
||||
assessments, prepared := 0, 0
|
||||
request.Operation.Execution.OnPrepared = func(context.Context, promptkit.PreparedRun) { prepared++ }
|
||||
request.Operation.Assess = func(ctx context.Context, value dnd.LocationRegistry) (contracts.CandidateAssessment, error) {
|
||||
assessments++
|
||||
return assessRegistry(ctx, request, value)
|
||||
}
|
||||
result, err := newExtractor(t, client).Extract(context.Background(), request)
|
||||
if (err != nil) != test.wantError || assessments != test.wantAssessments || prepared != test.wantRuns {
|
||||
t.Fatalf("registry=%#v %v, assessments=%d runs=%d", result, err, assessments, prepared)
|
||||
}
|
||||
if test.wantError {
|
||||
if result.Outcome.Disposition != "" {
|
||||
t.Fatal("strict rejection produced an outcome")
|
||||
}
|
||||
return
|
||||
}
|
||||
if result.Value.Locations == nil || len(result.Value.Locations) != 0 || (result.Outcome.Disposition == contracts.ProducerFallback) != test.wantFallback ||
|
||||
result.Outcome.ReuseEligible == test.wantFallback {
|
||||
t.Fatalf("unqualified empty registry=%#v", result)
|
||||
}
|
||||
if _, err := registrycodec.New().Encode(result.Value); err != nil {
|
||||
t.Fatalf("empty durable registry invalid: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCanceledNativeExtractionCannotDeliverFallback(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
client := &fakeLocationsLLMClient{content: []byte(groundedRegistry), onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }}
|
||||
request := extractionRequest()
|
||||
request.Operation.Controls.Mode = contracts.ModePermissive
|
||||
result, err := newExtractor(t, client).Extract(ctx, request)
|
||||
if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" || len(client.requests) != 1 {
|
||||
t.Fatalf("cancellation=%#v %v", result, err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user