200 lines
9.9 KiB
Go
200 lines
9.9 KiB
Go
|
|
package enemyevents
|
||
|
|
|
||
|
|
import (
|
||
|
|
"context"
|
||
|
|
"errors"
|
||
|
|
"reflect"
|
||
|
|
"strings"
|
||
|
|
"testing"
|
||
|
|
|
||
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||
|
|
combatcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/combatturns"
|
||
|
|
eventcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/enemyevents"
|
||
|
|
occurrencecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcoccurrences"
|
||
|
|
npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcregistry"
|
||
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/enemyevents/engagements"
|
||
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/enemyevents/shape"
|
||
|
|
sourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/enemyevents/source_refs"
|
||
|
|
relatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/enemyevents/source_relatedness"
|
||
|
|
"go.maximumdirect.net/promptkit"
|
||
|
|
)
|
||
|
|
|
||
|
|
const duplicateEngagements = `{"events":[{"name":"Ashfang","kind":"engaged","source_refs":[{"start_unit_id":1,"end_unit_id":1}]},{"name":"ASHFANG","kind":"engaged","source_refs":[{"start_unit_id":2,"end_unit_id":2}]}]}`
|
||
|
|
const invalidEventCitation = `{"events":[{"name":"Ashfang","kind":"engaged","source_refs":[{"start_unit_id":999,"end_unit_id":999}]}]}`
|
||
|
|
const knownEvents = `{"events":[{"name":"Ashfang","kind":"engaged","source_refs":[{"start_unit_id":2,"end_unit_id":1}]},{"name":"Orcs","kind":"fled","source_refs":[{"start_unit_id":4,"end_unit_id":4}]}]}`
|
||
|
|
|
||
|
|
func bindEnemyAssessment(request *contracts.TypedExtractionRequest[dnd.EnemyEventList]) *[]string {
|
||
|
|
validators := []contracts.TypedValidator[dnd.EnemyEventList]{
|
||
|
|
shape.New(shape.Options{}), sourcerefs.New(sourcerefs.Options{}),
|
||
|
|
relatedness.New(relatedness.Options{}), engagements.New(engagements.Options{}),
|
||
|
|
}
|
||
|
|
var calls []string
|
||
|
|
request.Operation.Assess = func(ctx context.Context, value dnd.EnemyEventList) (contracts.CandidateAssessment, error) {
|
||
|
|
var assessment contracts.CandidateAssessment
|
||
|
|
for _, validator := range validators {
|
||
|
|
calls = append(calls, validator.Name())
|
||
|
|
result, err := validator.Validate(ctx, contracts.TypedValidationRequest[dnd.EnemyEventList]{
|
||
|
|
Stage: "extract", Source: request.Source, Chunk: request.Chunk, Value: value, Execution: request.Operation.Execution,
|
||
|
|
})
|
||
|
|
if err != nil {
|
||
|
|
return contracts.CandidateAssessment{}, err
|
||
|
|
}
|
||
|
|
decision := contracts.AssessmentRejected
|
||
|
|
if result.Approved {
|
||
|
|
decision = contracts.AssessmentApproved
|
||
|
|
}
|
||
|
|
assessment.Validators = append(assessment.Validators, contracts.ValidatorAssessment{ValidatorName: validator.Name(), Decision: decision, Result: result})
|
||
|
|
}
|
||
|
|
return assessment, nil
|
||
|
|
}
|
||
|
|
return &calls
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestEnemyNativeCorrectionKeepsAllGroundingFixedAndSourceFree(t *testing.T) {
|
||
|
|
for _, first := range []string{duplicateEngagements, invalidEventCitation} {
|
||
|
|
request := enemyExtractionRequest(t)
|
||
|
|
request.Operation.Controls.MaxAdditionalTurns = 1
|
||
|
|
request.Operation.ReuseEligible = false
|
||
|
|
request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "enemy-override"}
|
||
|
|
var prepared []promptkit.PreparedRun
|
||
|
|
request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = append(prepared, value) }
|
||
|
|
calls := bindEnemyAssessment(&request)
|
||
|
|
client := &fakeEnemyEventsLLMClient{responses: []string{first, knownEvents}}
|
||
|
|
client.onGenerate = func(context.Context, promptkit.GenerateRequest) {
|
||
|
|
if len(client.requests) == 1 {
|
||
|
|
for _, name := range []string{NPCRegistryReferenceSlot, SceneDescriptionReferenceSlot, CombatTurnReferenceSlot, NPCOccurrenceReferenceSlot} {
|
||
|
|
request.References.Slots[name].Items[0].Content[0] = '!'
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
result, err := newEnemyExtractor(t, client, request.References).Extract(context.Background(), request)
|
||
|
|
chain := []string{shape.Key, sourcerefs.Key, relatedness.Key, engagements.Key}
|
||
|
|
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || result.Outcome.ReuseEligible ||
|
||
|
|
!reflect.DeepEqual(*calls, append(append([]string(nil), chain...), chain...)) || len(result.Value.Events) != 2 || len(client.requests) != 2 || len(prepared) != 2 {
|
||
|
|
t.Fatalf("enemy correction=%#v %v, validators=%#v native=%d/%d", result, err, *calls, len(client.requests), len(prepared))
|
||
|
|
}
|
||
|
|
if result.Value.Events[0].SourceRefs[0].StartUnitID != 1 || result.Value.Events[0].SourceRefs[0].EndUnitID != 2 || result.Value.Events[1].Name != "Orcs" || result.Value.Events[1].Kind != dnd.EnemyEventKindFled {
|
||
|
|
t.Fatalf("contextual names/evidence=%#v", result.Value)
|
||
|
|
}
|
||
|
|
for i, call := range client.requests {
|
||
|
|
id := PromptID
|
||
|
|
if i > 0 {
|
||
|
|
id = CorrectionPromptID
|
||
|
|
}
|
||
|
|
if prepared[i].PromptID != id || prepared[i].SelectedProfileID != request.LLMProfile || call.Target.Model != "enemy-override" || call.Prompt.SessionID != request.SessionID {
|
||
|
|
t.Fatal("native selection lost")
|
||
|
|
}
|
||
|
|
for _, name := range []string{NPCRegistryReferenceSlot, CombatTurnReferenceSlot, NPCOccurrenceReferenceSlot} {
|
||
|
|
material := string(renderedInput(t, call, name))
|
||
|
|
if !strings.Contains(material, "Ashfang") {
|
||
|
|
t.Fatalf("missing %s grounding", name)
|
||
|
|
}
|
||
|
|
for _, forbidden := range []string{"source_ref", "source_id", "start_unit_id", "npc:sha256:", "combat-session", "summary"} {
|
||
|
|
if strings.Contains(material, forbidden) {
|
||
|
|
t.Fatalf("%s exposed %q", name, forbidden)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
initial, revised := client.requests[0].Prompt.Messages, client.requests[1].Prompt.Messages
|
||
|
|
if len(revised) != len(initial)+2 || !reflect.DeepEqual(initial, revised[:len(initial)]) || revised[len(initial)].Role != "assistant" || revised[len(initial)].Content != first {
|
||
|
|
t.Fatal("native history changed fixed grounding")
|
||
|
|
}
|
||
|
|
guidance := revised[len(initial)+1].Content
|
||
|
|
if first == duplicateEngagements && !strings.Contains(guidance, "at most one") {
|
||
|
|
t.Fatal("engagement guidance lost")
|
||
|
|
}
|
||
|
|
if first == invalidEventCitation && !strings.Contains(guidance, "source range endpoints") {
|
||
|
|
t.Fatal("source guidance lost")
|
||
|
|
}
|
||
|
|
content, err := eventcodec.New().Encode(result.Value)
|
||
|
|
if err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
decoded, err := eventcodec.New().Decode(content)
|
||
|
|
if err != nil || !reflect.DeepEqual(result.Value, decoded) {
|
||
|
|
t.Fatalf("event wire round trip=%#v %v", decoded, err)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestEnemyTerminalModesAndCancellation(t *testing.T) {
|
||
|
|
failure := errors.New("provider unavailable")
|
||
|
|
for _, test := range []struct {
|
||
|
|
name string
|
||
|
|
mode contracts.ExecutionMode
|
||
|
|
response string
|
||
|
|
failures []error
|
||
|
|
assessments, runs int
|
||
|
|
wantError bool
|
||
|
|
}{
|
||
|
|
{"strict engagement rejection", contracts.ModeStrict, duplicateEngagements, nil, 2, 2, true},
|
||
|
|
{"permissive engagement rejection", contracts.ModePermissive, duplicateEngagements, nil, 2, 2, false},
|
||
|
|
{"initial provider failure", contracts.ModePermissive, "", []error{failure}, 0, 1, false},
|
||
|
|
{"failed continuation", contracts.ModePermissive, invalidEventCitation, []error{nil, failure}, 1, 2, false},
|
||
|
|
{"structural failure", contracts.ModePermissive, `{"events":null}`, nil, 0, 1, false},
|
||
|
|
} {
|
||
|
|
t.Run(test.name, func(t *testing.T) {
|
||
|
|
request := enemyExtractionRequest(t)
|
||
|
|
request.Operation.Controls.Mode = test.mode
|
||
|
|
request.Operation.Controls.MaxAdditionalTurns = 1
|
||
|
|
calls := bindEnemyAssessment(&request)
|
||
|
|
runs := 0
|
||
|
|
request.Operation.Execution.OnPrepared = func(context.Context, promptkit.PreparedRun) { runs++ }
|
||
|
|
client := &fakeEnemyEventsLLMClient{raw: test.response, errors: test.failures}
|
||
|
|
result, err := newEnemyExtractor(t, client, request.References).Extract(context.Background(), request)
|
||
|
|
if (err != nil) != test.wantError || len(*calls) != test.assessments*4 || runs != test.runs {
|
||
|
|
t.Fatalf("terminal=%#v %v, validators=%#v runs=%d", result, err, *calls, runs)
|
||
|
|
}
|
||
|
|
if test.wantError {
|
||
|
|
if result.Outcome.Disposition != "" {
|
||
|
|
t.Fatal("strict failure published output")
|
||
|
|
}
|
||
|
|
return
|
||
|
|
}
|
||
|
|
if result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || result.Value.Events == nil || len(result.Value.Events) != 0 {
|
||
|
|
t.Fatalf("fallback invented events=%#v", result)
|
||
|
|
}
|
||
|
|
if _, err := eventcodec.New().Encode(result.Value); err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
})
|
||
|
|
}
|
||
|
|
ctx, cancel := context.WithCancel(context.Background())
|
||
|
|
defer cancel()
|
||
|
|
request := enemyExtractionRequest(t)
|
||
|
|
request.Operation.Controls.Mode = contracts.ModePermissive
|
||
|
|
client := &fakeEnemyEventsLLMClient{raw: knownEvents, onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }}
|
||
|
|
result, err := newEnemyExtractor(t, client).Extract(ctx, request)
|
||
|
|
if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" {
|
||
|
|
t.Fatalf("canceled outcome=%#v %v", result, err)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestEnemyAcceptsEmptyFallbackReferencesWithoutInventingEvents(t *testing.T) {
|
||
|
|
request := enemyExtractionRequest(t)
|
||
|
|
npc, err := npccodec.New().Encode(dnd.NPCRegistry{NPCs: []dnd.NPC{}})
|
||
|
|
if err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
turns, err := combatcodec.New().Encode(dnd.CombatTurnList{CombatTurns: []dnd.CombatTurn{}})
|
||
|
|
if err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
occurrences, err := occurrencecodec.New().Encode(dnd.NPCOccurrenceList{Occurrences: []dnd.NPCOccurrence{}})
|
||
|
|
if err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
for name, content := range map[string][]byte{NPCRegistryReferenceSlot: npc, CombatTurnReferenceSlot: turns, NPCOccurrenceReferenceSlot: occurrences} {
|
||
|
|
request.References = replaceItem(request.References, name, newReferenceItem(name, content))
|
||
|
|
}
|
||
|
|
request.Operation.ReuseEligible = false
|
||
|
|
bindEnemyAssessment(&request)
|
||
|
|
client := &fakeEnemyEventsLLMClient{raw: `{"events":[]}`}
|
||
|
|
result, err := newEnemyExtractor(t, client, contracts.ReferenceSet{}).Extract(context.Background(), request)
|
||
|
|
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || result.Outcome.ReuseEligible || result.Value.Events == nil || len(result.Value.Events) != 0 || len(client.requests) != 1 {
|
||
|
|
t.Fatalf("empty reference handoff=%#v %v", result, err)
|
||
|
|
}
|
||
|
|
}
|