Files
notarius/internal/modules/dnd/extract/itemoccurrences/native_execution_test.go

214 lines
11 KiB
Go

package itemoccurrences
import (
"context"
"encoding/json"
"errors"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
occurrencecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/itemoccurrences"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity"
registryvalidator "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemoccurrences/registry"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemoccurrences/shape"
sourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemoccurrences/source_refs"
relatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemoccurrences/source_relatedness"
"go.maximumdirect.net/promptkit"
)
const rejectedHolder = `{"occurrences":[{"name":"Torch","kind":"acquired","quantity":null,"from":"Chest","to":"party","source_refs":[{"start_unit_id":2,"end_unit_id":2}]}]}`
const correctedHolder = `{"occurrences":[{"name":"Torch","kind":"acquired","quantity":null,"from":null,"to":"party","source_refs":[{"start_unit_id":2,"end_unit_id":2}]}]}`
const unknownItem = `{"occurrences":[{"name":"Unknown Item","kind":"discovered","quantity":null,"from":null,"to":null,"source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`
func bindAssessment(t *testing.T, request *contracts.TypedExtractionRequest[dnd.ItemOccurrenceList]) (*[]string, *[]string) {
t.Helper()
registry, err := registryvalidator.New(registryvalidator.Options{}, request.References)
if err != nil {
t.Fatal(err)
}
validators := []contracts.TypedValidator[dnd.ItemOccurrenceList]{
shape.New(shape.Options{}), sourcerefs.New(sourcerefs.Options{}),
relatedness.New(relatedness.Options{}), registry,
}
var calls, guidance []string
request.Operation.Assess = func(ctx context.Context, value dnd.ItemOccurrenceList) (contracts.CandidateAssessment, error) {
var assessment contracts.CandidateAssessment
for _, validator := range validators {
calls = append(calls, validator.Name())
result, err := validator.Validate(ctx, contracts.TypedValidationRequest[dnd.ItemOccurrenceList]{
Stage: "extract", Source: request.Source, Chunk: request.Chunk, References: request.References, Value: value,
})
if err != nil {
return contracts.CandidateAssessment{}, err
}
decision := contracts.AssessmentRejected
if result.Approved {
decision = contracts.AssessmentApproved
}
assessment.Validators = append(assessment.Validators, contracts.ValidatorAssessment{ValidatorName: validator.Name(), Decision: decision, Result: result})
if result.CorrectionGuidance != "" {
guidance = append(guidance, result.CorrectionGuidance)
}
}
return assessment, nil
}
return &calls, &guidance
}
func TestExtractCarriesRejectedHolderGuidanceIntoNativeCorrection(t *testing.T) {
request := extractionRequest()
request.References = itemRegistryReferences(t)
request.Operation.Controls.MaxAdditionalTurns = 1
request.Operation.ReuseEligible = false
request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "item-occurrence-override"}
var prepared []promptkit.PreparedRun
request.Operation.Execution.OnPrepared = func(_ context.Context, run promptkit.PreparedRun) { prepared = append(prepared, run) }
calls, guidance := bindAssessment(t, &request)
client := &fakeItemOccurrencesLLMClient{responses: []string{rejectedHolder, correctedHolder}}
result, err := newExtractor(t, client, request.References).Extract(context.Background(), request)
chain := []string{shape.Key, sourcerefs.Key, relatedness.Key, registryvalidator.Key}
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || result.Outcome.ReuseEligible ||
!reflect.DeepEqual(*calls, append(append([]string(nil), chain...), chain...)) || len(result.Value.Occurrences) != 1 || len(client.requests) != 2 || len(prepared) != 2 || len(*guidance) != 1 {
t.Fatalf("correction = %#v %v, validators=%#v guidance=%#v native=%d/%d", result, err, *calls, *guidance, len(client.requests), len(prepared))
}
occurrence := result.Value.Occurrences[0]
if occurrence.ItemID != identity.DeriveID("Torch") || occurrence.Name != "Torch" || occurrence.Kind != dnd.ItemOccurrenceKindAcquired ||
occurrence.From != "" || occurrence.To != "party" || occurrence.Quantity != nil || occurrence.SourceRefs[0].SourceID != request.Source.ID {
t.Fatalf("holder mapping = %#v", occurrence)
}
first, next := client.requests[0].Prompt.Messages, client.requests[1].Prompt.Messages
if len(next) != len(first)+2 || !reflect.DeepEqual(first, next[:len(first)]) || next[len(first)].Role != "assistant" || next[len(first)].Content != rejectedHolder {
t.Fatal("native continuation did not preserve the exact defective response and fixed context")
}
correction := next[len(first)+1].Content
for _, fragment := range []string{(*guidance)[0], "For `acquired` occurrences", "`from` to JSON null", "item \"Torch\"", "source unit 2"} {
if !strings.Contains(correction, fragment) {
t.Fatalf("missing holder guidance %q", fragment)
}
}
for _, forbidden := range []string{occurrence.ItemID, "item_id", "sha256", shape.Key, shape.ReasonCode} {
if strings.Contains(correction, forbidden) {
t.Fatalf("operator identifier became guidance: %q", forbidden)
}
}
for i, call := range client.requests {
id := PromptID
if i > 0 {
id = CorrectionPromptID
}
if prepared[i].PromptID != id || prepared[i].SelectedProfileID != request.LLMProfile || call.Target.Model != "item-occurrence-override" || call.Prompt.SessionID != request.SessionID {
t.Fatal("native profile/session/override lost")
}
if string(renderedInput(t, call, ItemRegistryReferenceSlot)) != `{"items":[{"name":"Torch"}]}` {
t.Fatal("item registry was not fixed and names-only")
}
}
content, err := occurrencecodec.New().Encode(result.Value)
if err != nil {
t.Fatal(err)
}
var wire struct {
Occurrences []map[string]json.RawMessage `json:"occurrences"`
}
if err := json.Unmarshal(content, &wire); err != nil {
t.Fatal(err)
}
if _, present := wire.Occurrences[0]["from"]; present {
t.Fatal("private null holder became a durable field")
}
}
func TestItemOccurrenceTerminalModesDoNotSynthesizeHolders(t *testing.T) {
failure := errors.New("provider unavailable")
for _, test := range []struct {
name string
mode contracts.ExecutionMode
response string
failures []error
assessments, runs int
wantError bool
}{
{"strict holder rejection", contracts.ModeStrict, rejectedHolder, nil, 2, 2, true},
{"permissive holder rejection", contracts.ModePermissive, rejectedHolder, nil, 2, 2, false},
{"strict unknown item", contracts.ModeStrict, unknownItem, nil, 0, 2, true},
{"permissive unknown item", contracts.ModePermissive, unknownItem, nil, 0, 2, false},
{"initial provider failure", contracts.ModePermissive, "", []error{failure}, 0, 1, false},
{"failed correction", contracts.ModePermissive, rejectedHolder, []error{nil, failure}, 1, 2, false},
{"structural failure", contracts.ModePermissive, `{"occurrences":null}`, nil, 0, 1, false},
} {
t.Run(test.name, func(t *testing.T) {
request := extractionRequest()
request.References = itemRegistryReferences(t)
request.Operation.Controls.Mode = test.mode
request.Operation.Controls.MaxAdditionalTurns = 1
calls, _ := bindAssessment(t, &request)
runs := 0
request.Operation.Execution.OnPrepared = func(context.Context, promptkit.PreparedRun) { runs++ }
client := &fakeItemOccurrencesLLMClient{content: []byte(test.response), errors: test.failures}
result, err := newExtractor(t, client, request.References).Extract(context.Background(), request)
if (err != nil) != test.wantError || len(*calls) != test.assessments*4 || runs != test.runs {
t.Fatalf("terminal = %#v %v, validators=%#v runs=%d", result, err, *calls, runs)
}
if test.wantError {
if result.Outcome.Disposition != "" || len(result.Value.Occurrences) != 0 {
t.Fatal("strict failure returned output")
}
return
}
if result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || result.Value.Occurrences == nil || len(result.Value.Occurrences) != 0 {
t.Fatalf("fallback invented an item or holder: %#v", result)
}
if _, err := occurrencecodec.New().Encode(result.Value); err != nil {
t.Fatal(err)
}
})
}
}
func TestItemOccurrenceCancellationCannotPublishFallback(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
request := extractionRequest()
request.References = itemRegistryReferences(t)
request.Operation.Controls.Mode = contracts.ModePermissive
client := &fakeItemOccurrencesLLMClient{content: []byte(correctedHolder), onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }}
result, err := newExtractor(t, client, request.References).Extract(ctx, request)
if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" || len(client.requests) != 1 {
t.Fatalf("canceled delivery = %#v %v", result, err)
}
}
func TestItemOccurrenceKindsHoldersQuantityAndDocumentOrderSurvive(t *testing.T) {
quantity := 3
client := &fakeItemOccurrencesLLMClient{response: extractionResponse{Occurrences: []itemOccurrenceResponse{
{Name: "Torch", Kind: "transferred", From: "Aria", To: "Borin", SourceRefs: responseRefs(5, 5)},
{Name: "Torch", Kind: "consumed", From: "Aria", SourceRefs: responseRefs(4, 4)},
{Name: "Torch", Kind: "lost", From: "party", SourceRefs: responseRefs(3, 3)},
{Name: "Torch", Kind: "acquired", To: "party", Quantity: &quantity, SourceRefs: responseRefs(2, 2)},
{Name: "Torch", Kind: "discovered", SourceRefs: responseRefs(1, 1)},
}}}
request := extractionRequest()
request.References = itemRegistryReferences(t)
bindAssessment(t, &request)
result, err := newExtractor(t, client).Extract(context.Background(), request)
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || len(result.Value.Occurrences) != 5 {
t.Fatalf("kinds = %#v %v", result, err)
}
kinds := []dnd.ItemOccurrenceKind{dnd.ItemOccurrenceKindDiscovered, dnd.ItemOccurrenceKindAcquired, dnd.ItemOccurrenceKindLost, dnd.ItemOccurrenceKindConsumed, dnd.ItemOccurrenceKindTransferred}
for i, occurrence := range result.Value.Occurrences {
if occurrence.Kind != kinds[i] || occurrence.SourceRefs[0].StartUnitID != i+1 {
t.Fatalf("ordered occurrence = %#v", occurrence)
}
}
quantity = 9
if *result.Value.Occurrences[1].Quantity != 3 || result.Value.Occurrences[4].From != "Aria" || result.Value.Occurrences[4].To != "Borin" {
t.Fatal("holder or quantity interpretation changed")
}
if _, err := occurrencecodec.New().Encode(result.Value); err != nil {
t.Fatal(err)
}
}