Files
notarius/internal/modules/dnd/extract/npcoccurrences/native_execution_test.go

162 lines
8.1 KiB
Go
Raw Normal View History

package npcoccurrences
import (
"context"
"errors"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
occurrencecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcoccurrences"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity"
refs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/npcoccurrences/source_refs"
"go.maximumdirect.net/promptkit"
)
const unresolvedOccurrence = `{"occurrences":[{"name":"Unknown NPC","kind":"dialogue","source_refs":[{"start_unit_id":10,"end_unit_id":10}]}]}`
const knownOccurrence = `{"occurrences":[{"name":"Mira Thorn","kind":"dialogue","source_refs":[{"start_unit_id":10,"end_unit_id":10}]}]}`
const invalidCitation = `{"occurrences":[{"name":"Mira Thorn","kind":"dialogue","source_refs":[{"start_unit_id":999,"end_unit_id":999}]}]}`
func operationReferences(t *testing.T) (contracts.ReferenceSet, string) {
return requiredRegistryReferences(t, "Mira Thorn"), identity.DeriveID("Mira Thorn")
}
func sourceAssessment(ctx context.Context, request contracts.TypedExtractionRequest[dnd.NPCOccurrenceList], value dnd.NPCOccurrenceList) (contracts.CandidateAssessment, error) {
validator := refs.New(refs.Options{})
result, err := validator.Validate(ctx, contracts.TypedValidationRequest[dnd.NPCOccurrenceList]{Stage: "extract", Source: request.Source, Chunk: request.Chunk, Value: value})
if err != nil {
return contracts.CandidateAssessment{}, err
}
decision := contracts.AssessmentRejected
if result.Approved {
decision = contracts.AssessmentApproved
}
return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{ValidatorName: validator.Name(), Decision: decision, Result: result}}}, nil
}
func TestNativeCorrectionKeepsRegistryResolutionAndInputsFixed(t *testing.T) {
for _, first := range []string{unresolvedOccurrence, invalidCitation} {
request := extractionRequest()
references, expectedID := operationReferences(t)
request.References = references
request.Operation.Controls.MaxAdditionalTurns = 1
// Accepted work consuming an ineligible generated reference stays ineligible.
request.Operation.ReuseEligible = false
request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "occurrence-override"}
var prepared []promptkit.PreparedRun
request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = append(prepared, value) }
assessments := 0
request.Operation.Assess = func(ctx context.Context, value dnd.NPCOccurrenceList) (contracts.CandidateAssessment, error) {
assessments++
return sourceAssessment(ctx, request, value)
}
client := &fakeOccurrencesLLMClient{responses: []string{first, knownOccurrence}}
client.onGenerate = func(context.Context, promptkit.GenerateRequest) {
if len(client.requests) == 1 {
request.References.Slots[NPCRegistryReferenceSlot].Items[0].Content[0] = '!'
}
}
result, err := newExtractor(t, client, request.References).Extract(context.Background(), request)
expectedAssessments := 1
if first == invalidCitation {
expectedAssessments = 2
}
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || result.Outcome.ReuseEligible ||
len(client.requests) != 2 || len(prepared) != 2 || assessments != expectedAssessments || len(result.Value.Occurrences) != 1 {
t.Fatalf("corrected occurrences=%#v %v, assessments=%d native=%d/%d", result, err, assessments, len(client.requests), len(prepared))
}
occurrence := result.Value.Occurrences[0]
if occurrence.NPCID != expectedID || occurrence.Name != "Mira Thorn" || len(occurrence.SourceRefs) != 1 ||
occurrence.SourceRefs[0].SourceID != request.Source.ID || occurrence.SourceRefs[0].StartUnitID != 10 || occurrence.SourceRefs[0].EndUnitID != 10 {
t.Fatalf("identity/evidence=%#v", occurrence)
}
if _, err := occurrencecodec.New().Encode(result.Value); err != nil {
t.Fatalf("durable occurrence=%v", err)
}
for i, call := range client.requests {
id := PromptID
if i > 0 {
id = CorrectionPromptID
}
if prepared[i].PromptID != id || prepared[i].SelectedProfileID != request.LLMProfile || call.Target.Model != "occurrence-override" || call.Prompt.SessionID != request.SessionID {
t.Fatal("native selection lost")
}
}
initial, revised := client.requests[0].Prompt.Messages, client.requests[1].Prompt.Messages
if len(revised) != len(initial)+2 || !reflect.DeepEqual(revised[:len(initial)], initial) || revised[len(initial)].Role != "assistant" || revised[len(initial)].Content != first {
t.Fatal("fixed native context/history changed")
}
input := string(renderedInput(t, client.requests[1], NPCRegistryReferenceSlot))
if strings.Contains(input, expectedID) || strings.Contains(input, request.Source.ID) {
t.Fatal("registry projection exposed an opaque identity")
}
if !reflect.DeepEqual(renderedInput(t, client.requests[0], NPCRegistryReferenceSlot), renderedInput(t, client.requests[1], NPCRegistryReferenceSlot)) {
t.Fatal("registry projection was regenerated")
}
}
}
func TestNativeOccurrenceFallbackIsAnExplicitEmptyList(t *testing.T) {
failure := errors.New("provider unavailable")
for _, test := range []struct {
name string
mode contracts.ExecutionMode
response string
errors []error
wantAssessments, wantRuns int
wantError bool
}{
{"strict unresolved", contracts.ModeStrict, unresolvedOccurrence, nil, 0, 2, true},
{"permissive unresolved", contracts.ModePermissive, unresolvedOccurrence, nil, 0, 2, false},
{"rejected citation", contracts.ModePermissive, invalidCitation, nil, 2, 2, false},
{"provider unavailable", contracts.ModePermissive, "", []error{failure}, 0, 1, false},
{"failed continuation", contracts.ModePermissive, invalidCitation, []error{nil, failure}, 1, 2, false},
{"structural failure", contracts.ModePermissive, `{"occurrences":null}`, nil, 0, 1, false},
} {
t.Run(test.name, func(t *testing.T) {
request := extractionRequest()
request.References, _ = operationReferences(t)
request.Operation.Controls.Mode = test.mode
request.Operation.Controls.MaxAdditionalTurns = 1
assessments, prepared := 0, 0
request.Operation.Execution.OnPrepared = func(context.Context, promptkit.PreparedRun) { prepared++ }
request.Operation.Assess = func(ctx context.Context, value dnd.NPCOccurrenceList) (contracts.CandidateAssessment, error) {
assessments++
return sourceAssessment(ctx, request, value)
}
client := &fakeOccurrencesLLMClient{raw: test.response, errors: test.errors}
result, err := newExtractor(t, client, request.References).Extract(context.Background(), request)
if (err != nil) != test.wantError || assessments != test.wantAssessments || prepared != test.wantRuns {
t.Fatalf("outcome=%#v %v, assessments/runs=%d/%d", result, err, assessments, prepared)
}
if test.wantError {
if result.Outcome.Disposition != "" {
t.Fatal("strict failure published an outcome")
}
return
}
if result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || result.Value.Occurrences == nil || len(result.Value.Occurrences) != 0 {
t.Fatalf("unqualified fallback=%#v", result)
}
if _, err := occurrencecodec.New().Encode(result.Value); err != nil {
t.Fatalf("empty occurrence shape=%v", err)
}
})
}
}
func TestCanceledOccurrenceDeliveryCannotConstructFallback(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
request := extractionRequest()
request.References, _ = operationReferences(t)
request.Operation.Controls.Mode = contracts.ModePermissive
client := &fakeOccurrencesLLMClient{raw: knownOccurrence, onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }}
result, err := newExtractor(t, client, request.References).Extract(ctx, request)
if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" || len(client.requests) != 1 {
t.Fatalf("cancellation=%#v %v", result, err)
}
}