163 lines
8.2 KiB
Go
163 lines
8.2 KiB
Go
|
|
package locationoccurrences
|
||
|
|
|
||
|
|
import (
|
||
|
|
"context"
|
||
|
|
"errors"
|
||
|
|
"reflect"
|
||
|
|
"strings"
|
||
|
|
"testing"
|
||
|
|
|
||
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||
|
|
occurrencecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/locationoccurrences"
|
||
|
|
refs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/locationoccurrences/source_refs"
|
||
|
|
|
||
|
|
"go.maximumdirect.net/promptkit"
|
||
|
|
)
|
||
|
|
|
||
|
|
const unresolvedOccurrence = `{"occurrences":[{"name":"The Tavern","registry_refs":[],"kind":"visited","source_refs":[{"start_unit_id":10,"end_unit_id":10}]}]}`
|
||
|
|
const knownOccurrence = `{"occurrences":[{"name":"The Tavern","registry_refs":[{"start_unit_id":20,"end_unit_id":20}],"kind":"visited","source_refs":[{"start_unit_id":10,"end_unit_id":10}]}]}`
|
||
|
|
const invalidCitation = `{"occurrences":[{"name":"The Tavern","registry_refs":[{"start_unit_id":20,"end_unit_id":20}],"kind":"visited","source_refs":[{"start_unit_id":999,"end_unit_id":999}]}]}`
|
||
|
|
|
||
|
|
func operationReferences(t *testing.T) (contracts.ReferenceSet, string) {
|
||
|
|
locations := locationRegistry(t, "The Tavern", "The Tavern")
|
||
|
|
return registryReferences(t, locations), locations.Locations[1].ID
|
||
|
|
}
|
||
|
|
|
||
|
|
func sourceAssessment(ctx context.Context, request contracts.TypedExtractionRequest[dnd.LocationOccurrenceList], value dnd.LocationOccurrenceList) (contracts.CandidateAssessment, error) {
|
||
|
|
validator := refs.New(refs.Options{})
|
||
|
|
result, err := validator.Validate(ctx, contracts.TypedValidationRequest[dnd.LocationOccurrenceList]{Stage: "extract", Source: request.Source, Chunk: request.Chunk, Value: value})
|
||
|
|
if err != nil {
|
||
|
|
return contracts.CandidateAssessment{}, err
|
||
|
|
}
|
||
|
|
decision := contracts.AssessmentRejected
|
||
|
|
if result.Approved {
|
||
|
|
decision = contracts.AssessmentApproved
|
||
|
|
}
|
||
|
|
return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{ValidatorName: validator.Name(), Decision: decision, Result: result}}}, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestNativeCorrectionKeepsRegistryResolutionAndInputsFixed(t *testing.T) {
|
||
|
|
for _, first := range []string{unresolvedOccurrence, invalidCitation} {
|
||
|
|
request := extractionRequest()
|
||
|
|
references, expectedID := operationReferences(t)
|
||
|
|
request.References = references
|
||
|
|
request.Operation.Controls.MaxAdditionalTurns = 1
|
||
|
|
// Accepted work consuming an ineligible generated reference stays ineligible.
|
||
|
|
request.Operation.ReuseEligible = false
|
||
|
|
request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "occurrence-override"}
|
||
|
|
var prepared []promptkit.PreparedRun
|
||
|
|
request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = append(prepared, value) }
|
||
|
|
assessments := 0
|
||
|
|
request.Operation.Assess = func(ctx context.Context, value dnd.LocationOccurrenceList) (contracts.CandidateAssessment, error) {
|
||
|
|
assessments++
|
||
|
|
return sourceAssessment(ctx, request, value)
|
||
|
|
}
|
||
|
|
client := &fakeOccurrencesLLMClient{responses: []string{first, knownOccurrence}}
|
||
|
|
client.onGenerate = func(context.Context, promptkit.GenerateRequest) {
|
||
|
|
if len(client.requests) == 1 {
|
||
|
|
request.References.Slots[LocationRegistryReferenceSlot].Items[0].Content[0] = '!'
|
||
|
|
}
|
||
|
|
}
|
||
|
|
result, err := newExtractor(t, client, request.References).Extract(context.Background(), request)
|
||
|
|
expectedAssessments := 1
|
||
|
|
if first == invalidCitation {
|
||
|
|
expectedAssessments = 2
|
||
|
|
}
|
||
|
|
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || result.Outcome.ReuseEligible ||
|
||
|
|
len(client.requests) != 2 || len(prepared) != 2 || assessments != expectedAssessments || len(result.Value.Occurrences) != 1 {
|
||
|
|
t.Fatalf("corrected occurrences=%#v %v, assessments=%d native=%d/%d", result, err, assessments, len(client.requests), len(prepared))
|
||
|
|
}
|
||
|
|
occurrence := result.Value.Occurrences[0]
|
||
|
|
if occurrence.LocationID != expectedID || occurrence.Name != "The Tavern" || len(occurrence.SourceRefs) != 1 ||
|
||
|
|
occurrence.SourceRefs[0].SourceID != request.Source.ID || occurrence.SourceRefs[0].StartUnitID != 10 || occurrence.SourceRefs[0].EndUnitID != 10 {
|
||
|
|
t.Fatalf("identity/evidence=%#v", occurrence)
|
||
|
|
}
|
||
|
|
if _, err := occurrencecodec.New().Encode(result.Value); err != nil {
|
||
|
|
t.Fatalf("durable occurrence=%v", err)
|
||
|
|
}
|
||
|
|
for i, call := range client.requests {
|
||
|
|
id := PromptID
|
||
|
|
if i > 0 {
|
||
|
|
id = CorrectionPromptID
|
||
|
|
}
|
||
|
|
if prepared[i].PromptID != id || prepared[i].SelectedProfileID != request.LLMProfile || call.Target.Model != "occurrence-override" || call.Prompt.SessionID != request.SessionID {
|
||
|
|
t.Fatal("native selection lost")
|
||
|
|
}
|
||
|
|
}
|
||
|
|
initial, revised := client.requests[0].Prompt.Messages, client.requests[1].Prompt.Messages
|
||
|
|
if len(revised) != len(initial)+2 || !reflect.DeepEqual(revised[:len(initial)], initial) || revised[len(initial)].Role != "assistant" || revised[len(initial)].Content != first {
|
||
|
|
t.Fatal("fixed native context/history changed")
|
||
|
|
}
|
||
|
|
input := string(renderedInput(t, client.requests[1], LocationRegistryReferenceSlot))
|
||
|
|
if strings.Contains(input, expectedID) || strings.Contains(input, request.Source.ID) {
|
||
|
|
t.Fatal("registry projection exposed an opaque identity")
|
||
|
|
}
|
||
|
|
if !reflect.DeepEqual(renderedInput(t, client.requests[0], LocationRegistryReferenceSlot), renderedInput(t, client.requests[1], LocationRegistryReferenceSlot)) {
|
||
|
|
t.Fatal("registry projection was regenerated")
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestNativeOccurrenceFallbackIsAnExplicitEmptyList(t *testing.T) {
|
||
|
|
failure := errors.New("provider unavailable")
|
||
|
|
for _, test := range []struct {
|
||
|
|
name string
|
||
|
|
mode contracts.ExecutionMode
|
||
|
|
response string
|
||
|
|
errors []error
|
||
|
|
wantAssessments, wantRuns int
|
||
|
|
wantError bool
|
||
|
|
}{
|
||
|
|
{"strict unresolved", contracts.ModeStrict, unresolvedOccurrence, nil, 0, 2, true},
|
||
|
|
{"permissive unresolved", contracts.ModePermissive, unresolvedOccurrence, nil, 0, 2, false},
|
||
|
|
{"rejected citation", contracts.ModePermissive, invalidCitation, nil, 2, 2, false},
|
||
|
|
{"provider unavailable", contracts.ModePermissive, "", []error{failure}, 0, 1, false},
|
||
|
|
{"failed continuation", contracts.ModePermissive, invalidCitation, []error{nil, failure}, 1, 2, false},
|
||
|
|
{"structural failure", contracts.ModePermissive, `{"occurrences":null}`, nil, 0, 1, false},
|
||
|
|
} {
|
||
|
|
t.Run(test.name, func(t *testing.T) {
|
||
|
|
request := extractionRequest()
|
||
|
|
request.References, _ = operationReferences(t)
|
||
|
|
request.Operation.Controls.Mode = test.mode
|
||
|
|
request.Operation.Controls.MaxAdditionalTurns = 1
|
||
|
|
assessments, prepared := 0, 0
|
||
|
|
request.Operation.Execution.OnPrepared = func(context.Context, promptkit.PreparedRun) { prepared++ }
|
||
|
|
request.Operation.Assess = func(ctx context.Context, value dnd.LocationOccurrenceList) (contracts.CandidateAssessment, error) {
|
||
|
|
assessments++
|
||
|
|
return sourceAssessment(ctx, request, value)
|
||
|
|
}
|
||
|
|
client := &fakeOccurrencesLLMClient{raw: test.response, errors: test.errors}
|
||
|
|
result, err := newExtractor(t, client, request.References).Extract(context.Background(), request)
|
||
|
|
if (err != nil) != test.wantError || assessments != test.wantAssessments || prepared != test.wantRuns {
|
||
|
|
t.Fatalf("outcome=%#v %v, assessments/runs=%d/%d", result, err, assessments, prepared)
|
||
|
|
}
|
||
|
|
if test.wantError {
|
||
|
|
if result.Outcome.Disposition != "" {
|
||
|
|
t.Fatal("strict failure published an outcome")
|
||
|
|
}
|
||
|
|
return
|
||
|
|
}
|
||
|
|
if result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || result.Value.Occurrences == nil || len(result.Value.Occurrences) != 0 {
|
||
|
|
t.Fatalf("unqualified fallback=%#v", result)
|
||
|
|
}
|
||
|
|
if _, err := occurrencecodec.New().Encode(result.Value); err != nil {
|
||
|
|
t.Fatalf("empty occurrence shape=%v", err)
|
||
|
|
}
|
||
|
|
})
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestCanceledOccurrenceDeliveryCannotConstructFallback(t *testing.T) {
|
||
|
|
ctx, cancel := context.WithCancel(context.Background())
|
||
|
|
defer cancel()
|
||
|
|
request := extractionRequest()
|
||
|
|
request.References, _ = operationReferences(t)
|
||
|
|
request.Operation.Controls.Mode = contracts.ModePermissive
|
||
|
|
client := &fakeOccurrencesLLMClient{raw: knownOccurrence, onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }}
|
||
|
|
result, err := newExtractor(t, client, request.References).Extract(ctx, request)
|
||
|
|
if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" || len(client.requests) != 1 {
|
||
|
|
t.Fatalf("cancellation=%#v %v", result, err)
|
||
|
|
}
|
||
|
|
}
|