214 lines
11 KiB
Go
214 lines
11 KiB
Go
|
|
package itemoccurrences
|
||
|
|
|
||
|
|
import (
|
||
|
|
"context"
|
||
|
|
"encoding/json"
|
||
|
|
"errors"
|
||
|
|
"reflect"
|
||
|
|
"strings"
|
||
|
|
"testing"
|
||
|
|
|
||
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||
|
|
occurrencecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/itemoccurrences"
|
||
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity"
|
||
|
|
registryvalidator "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemoccurrences/registry"
|
||
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemoccurrences/shape"
|
||
|
|
sourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemoccurrences/source_refs"
|
||
|
|
relatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemoccurrences/source_relatedness"
|
||
|
|
"go.maximumdirect.net/promptkit"
|
||
|
|
)
|
||
|
|
|
||
|
|
const rejectedHolder = `{"occurrences":[{"name":"Torch","kind":"acquired","quantity":null,"from":"Chest","to":"party","source_refs":[{"start_unit_id":2,"end_unit_id":2}]}]}`
|
||
|
|
const correctedHolder = `{"occurrences":[{"name":"Torch","kind":"acquired","quantity":null,"from":null,"to":"party","source_refs":[{"start_unit_id":2,"end_unit_id":2}]}]}`
|
||
|
|
const unknownItem = `{"occurrences":[{"name":"Unknown Item","kind":"discovered","quantity":null,"from":null,"to":null,"source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`
|
||
|
|
|
||
|
|
func bindAssessment(t *testing.T, request *contracts.TypedExtractionRequest[dnd.ItemOccurrenceList]) (*[]string, *[]string) {
|
||
|
|
t.Helper()
|
||
|
|
registry, err := registryvalidator.New(registryvalidator.Options{}, request.References)
|
||
|
|
if err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
validators := []contracts.TypedValidator[dnd.ItemOccurrenceList]{
|
||
|
|
shape.New(shape.Options{}), sourcerefs.New(sourcerefs.Options{}),
|
||
|
|
relatedness.New(relatedness.Options{}), registry,
|
||
|
|
}
|
||
|
|
var calls, guidance []string
|
||
|
|
request.Operation.Assess = func(ctx context.Context, value dnd.ItemOccurrenceList) (contracts.CandidateAssessment, error) {
|
||
|
|
var assessment contracts.CandidateAssessment
|
||
|
|
for _, validator := range validators {
|
||
|
|
calls = append(calls, validator.Name())
|
||
|
|
result, err := validator.Validate(ctx, contracts.TypedValidationRequest[dnd.ItemOccurrenceList]{
|
||
|
|
Stage: "extract", Source: request.Source, Chunk: request.Chunk, References: request.References, Value: value,
|
||
|
|
})
|
||
|
|
if err != nil {
|
||
|
|
return contracts.CandidateAssessment{}, err
|
||
|
|
}
|
||
|
|
decision := contracts.AssessmentRejected
|
||
|
|
if result.Approved {
|
||
|
|
decision = contracts.AssessmentApproved
|
||
|
|
}
|
||
|
|
assessment.Validators = append(assessment.Validators, contracts.ValidatorAssessment{ValidatorName: validator.Name(), Decision: decision, Result: result})
|
||
|
|
if result.CorrectionGuidance != "" {
|
||
|
|
guidance = append(guidance, result.CorrectionGuidance)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return assessment, nil
|
||
|
|
}
|
||
|
|
return &calls, &guidance
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestExtractCarriesRejectedHolderGuidanceIntoNativeCorrection(t *testing.T) {
|
||
|
|
request := extractionRequest()
|
||
|
|
request.References = itemRegistryReferences(t)
|
||
|
|
request.Operation.Controls.MaxAdditionalTurns = 1
|
||
|
|
request.Operation.ReuseEligible = false
|
||
|
|
request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "item-occurrence-override"}
|
||
|
|
var prepared []promptkit.PreparedRun
|
||
|
|
request.Operation.Execution.OnPrepared = func(_ context.Context, run promptkit.PreparedRun) { prepared = append(prepared, run) }
|
||
|
|
calls, guidance := bindAssessment(t, &request)
|
||
|
|
client := &fakeItemOccurrencesLLMClient{responses: []string{rejectedHolder, correctedHolder}}
|
||
|
|
result, err := newExtractor(t, client, request.References).Extract(context.Background(), request)
|
||
|
|
chain := []string{shape.Key, sourcerefs.Key, relatedness.Key, registryvalidator.Key}
|
||
|
|
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || result.Outcome.ReuseEligible ||
|
||
|
|
!reflect.DeepEqual(*calls, append(append([]string(nil), chain...), chain...)) || len(result.Value.Occurrences) != 1 || len(client.requests) != 2 || len(prepared) != 2 || len(*guidance) != 1 {
|
||
|
|
t.Fatalf("correction = %#v %v, validators=%#v guidance=%#v native=%d/%d", result, err, *calls, *guidance, len(client.requests), len(prepared))
|
||
|
|
}
|
||
|
|
occurrence := result.Value.Occurrences[0]
|
||
|
|
if occurrence.ItemID != identity.DeriveID("Torch") || occurrence.Name != "Torch" || occurrence.Kind != dnd.ItemOccurrenceKindAcquired ||
|
||
|
|
occurrence.From != "" || occurrence.To != "party" || occurrence.Quantity != nil || occurrence.SourceRefs[0].SourceID != request.Source.ID {
|
||
|
|
t.Fatalf("holder mapping = %#v", occurrence)
|
||
|
|
}
|
||
|
|
first, next := client.requests[0].Prompt.Messages, client.requests[1].Prompt.Messages
|
||
|
|
if len(next) != len(first)+2 || !reflect.DeepEqual(first, next[:len(first)]) || next[len(first)].Role != "assistant" || next[len(first)].Content != rejectedHolder {
|
||
|
|
t.Fatal("native continuation did not preserve the exact defective response and fixed context")
|
||
|
|
}
|
||
|
|
correction := next[len(first)+1].Content
|
||
|
|
for _, fragment := range []string{(*guidance)[0], "For `acquired` occurrences", "`from` to JSON null", "item \"Torch\"", "source unit 2"} {
|
||
|
|
if !strings.Contains(correction, fragment) {
|
||
|
|
t.Fatalf("missing holder guidance %q", fragment)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
for _, forbidden := range []string{occurrence.ItemID, "item_id", "sha256", shape.Key, shape.ReasonCode} {
|
||
|
|
if strings.Contains(correction, forbidden) {
|
||
|
|
t.Fatalf("operator identifier became guidance: %q", forbidden)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
for i, call := range client.requests {
|
||
|
|
id := PromptID
|
||
|
|
if i > 0 {
|
||
|
|
id = CorrectionPromptID
|
||
|
|
}
|
||
|
|
if prepared[i].PromptID != id || prepared[i].SelectedProfileID != request.LLMProfile || call.Target.Model != "item-occurrence-override" || call.Prompt.SessionID != request.SessionID {
|
||
|
|
t.Fatal("native profile/session/override lost")
|
||
|
|
}
|
||
|
|
if string(renderedInput(t, call, ItemRegistryReferenceSlot)) != `{"items":[{"name":"Torch"}]}` {
|
||
|
|
t.Fatal("item registry was not fixed and names-only")
|
||
|
|
}
|
||
|
|
}
|
||
|
|
content, err := occurrencecodec.New().Encode(result.Value)
|
||
|
|
if err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
var wire struct {
|
||
|
|
Occurrences []map[string]json.RawMessage `json:"occurrences"`
|
||
|
|
}
|
||
|
|
if err := json.Unmarshal(content, &wire); err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
if _, present := wire.Occurrences[0]["from"]; present {
|
||
|
|
t.Fatal("private null holder became a durable field")
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestItemOccurrenceTerminalModesDoNotSynthesizeHolders(t *testing.T) {
|
||
|
|
failure := errors.New("provider unavailable")
|
||
|
|
for _, test := range []struct {
|
||
|
|
name string
|
||
|
|
mode contracts.ExecutionMode
|
||
|
|
response string
|
||
|
|
failures []error
|
||
|
|
assessments, runs int
|
||
|
|
wantError bool
|
||
|
|
}{
|
||
|
|
{"strict holder rejection", contracts.ModeStrict, rejectedHolder, nil, 2, 2, true},
|
||
|
|
{"permissive holder rejection", contracts.ModePermissive, rejectedHolder, nil, 2, 2, false},
|
||
|
|
{"strict unknown item", contracts.ModeStrict, unknownItem, nil, 0, 2, true},
|
||
|
|
{"permissive unknown item", contracts.ModePermissive, unknownItem, nil, 0, 2, false},
|
||
|
|
{"initial provider failure", contracts.ModePermissive, "", []error{failure}, 0, 1, false},
|
||
|
|
{"failed correction", contracts.ModePermissive, rejectedHolder, []error{nil, failure}, 1, 2, false},
|
||
|
|
{"structural failure", contracts.ModePermissive, `{"occurrences":null}`, nil, 0, 1, false},
|
||
|
|
} {
|
||
|
|
t.Run(test.name, func(t *testing.T) {
|
||
|
|
request := extractionRequest()
|
||
|
|
request.References = itemRegistryReferences(t)
|
||
|
|
request.Operation.Controls.Mode = test.mode
|
||
|
|
request.Operation.Controls.MaxAdditionalTurns = 1
|
||
|
|
calls, _ := bindAssessment(t, &request)
|
||
|
|
runs := 0
|
||
|
|
request.Operation.Execution.OnPrepared = func(context.Context, promptkit.PreparedRun) { runs++ }
|
||
|
|
client := &fakeItemOccurrencesLLMClient{content: []byte(test.response), errors: test.failures}
|
||
|
|
result, err := newExtractor(t, client, request.References).Extract(context.Background(), request)
|
||
|
|
if (err != nil) != test.wantError || len(*calls) != test.assessments*4 || runs != test.runs {
|
||
|
|
t.Fatalf("terminal = %#v %v, validators=%#v runs=%d", result, err, *calls, runs)
|
||
|
|
}
|
||
|
|
if test.wantError {
|
||
|
|
if result.Outcome.Disposition != "" || len(result.Value.Occurrences) != 0 {
|
||
|
|
t.Fatal("strict failure returned output")
|
||
|
|
}
|
||
|
|
return
|
||
|
|
}
|
||
|
|
if result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || result.Value.Occurrences == nil || len(result.Value.Occurrences) != 0 {
|
||
|
|
t.Fatalf("fallback invented an item or holder: %#v", result)
|
||
|
|
}
|
||
|
|
if _, err := occurrencecodec.New().Encode(result.Value); err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
})
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestItemOccurrenceCancellationCannotPublishFallback(t *testing.T) {
|
||
|
|
ctx, cancel := context.WithCancel(context.Background())
|
||
|
|
defer cancel()
|
||
|
|
request := extractionRequest()
|
||
|
|
request.References = itemRegistryReferences(t)
|
||
|
|
request.Operation.Controls.Mode = contracts.ModePermissive
|
||
|
|
client := &fakeItemOccurrencesLLMClient{content: []byte(correctedHolder), onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }}
|
||
|
|
result, err := newExtractor(t, client, request.References).Extract(ctx, request)
|
||
|
|
if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" || len(client.requests) != 1 {
|
||
|
|
t.Fatalf("canceled delivery = %#v %v", result, err)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestItemOccurrenceKindsHoldersQuantityAndDocumentOrderSurvive(t *testing.T) {
|
||
|
|
quantity := 3
|
||
|
|
client := &fakeItemOccurrencesLLMClient{response: extractionResponse{Occurrences: []itemOccurrenceResponse{
|
||
|
|
{Name: "Torch", Kind: "transferred", From: "Aria", To: "Borin", SourceRefs: responseRefs(5, 5)},
|
||
|
|
{Name: "Torch", Kind: "consumed", From: "Aria", SourceRefs: responseRefs(4, 4)},
|
||
|
|
{Name: "Torch", Kind: "lost", From: "party", SourceRefs: responseRefs(3, 3)},
|
||
|
|
{Name: "Torch", Kind: "acquired", To: "party", Quantity: &quantity, SourceRefs: responseRefs(2, 2)},
|
||
|
|
{Name: "Torch", Kind: "discovered", SourceRefs: responseRefs(1, 1)},
|
||
|
|
}}}
|
||
|
|
request := extractionRequest()
|
||
|
|
request.References = itemRegistryReferences(t)
|
||
|
|
bindAssessment(t, &request)
|
||
|
|
result, err := newExtractor(t, client).Extract(context.Background(), request)
|
||
|
|
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || len(result.Value.Occurrences) != 5 {
|
||
|
|
t.Fatalf("kinds = %#v %v", result, err)
|
||
|
|
}
|
||
|
|
kinds := []dnd.ItemOccurrenceKind{dnd.ItemOccurrenceKindDiscovered, dnd.ItemOccurrenceKindAcquired, dnd.ItemOccurrenceKindLost, dnd.ItemOccurrenceKindConsumed, dnd.ItemOccurrenceKindTransferred}
|
||
|
|
for i, occurrence := range result.Value.Occurrences {
|
||
|
|
if occurrence.Kind != kinds[i] || occurrence.SourceRefs[0].StartUnitID != i+1 {
|
||
|
|
t.Fatalf("ordered occurrence = %#v", occurrence)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
quantity = 9
|
||
|
|
if *result.Value.Occurrences[1].Quantity != 3 || result.Value.Occurrences[4].From != "Aria" || result.Value.Occurrences[4].To != "Borin" {
|
||
|
|
t.Fatal("holder or quantity interpretation changed")
|
||
|
|
}
|
||
|
|
if _, err := occurrencecodec.New().Encode(result.Value); err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
}
|