Files
notarius/internal/modules/dnd/extract/combatturns/native_execution_test.go

198 lines
10 KiB
Go
Raw Normal View History

package combatturns
import (
"context"
"errors"
"reflect"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
combatcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/combatturns"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/combatturns/shape"
sourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/combatturns/source_refs"
relatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/combatturns/source_relatedness"
"go.maximumdirect.net/promptkit"
)
const missingActor = `{"combat_turns":[{"actor":"","turn_kind":"reaction","source_refs":[{"start_unit_id":10,"end_unit_id":10}]}]}`
const invalidCitation = `{"combat_turns":[{"actor":"Aria","turn_kind":"reaction","source_refs":[{"start_unit_id":999,"end_unit_id":999}]}]}`
const knownTurn = `{"combat_turns":[{"actor":"Aria","turn_kind":"reaction","source_refs":[{"start_unit_id":10,"end_unit_id":2}]}]}`
func nativeCombatRequest(t *testing.T) contracts.TypedExtractionRequest[dnd.CombatTurnList] {
t.Helper()
request := extractionRequest(t)
// These end-to-end native cases must also satisfy durable numeric range
// constraints. The interpretation-only fixtures separately cover arbitrary
// document ordering; no durable codec rule is relaxed for them.
request.Source.Units[0], request.Source.Units[1] = request.Source.Units[1], request.Source.Units[0]
request.Chunk.Units = append(request.Chunk.Units[:0], request.Source.Units...)
request.Chunk.Content = []byte(`{"units":[2,10]}`)
request.SourceInput = contracts.NewLLMInputMaterial("source", request.Chunk.MediaType, request.Chunk.Content, "sha256:chunk", "file:///session-alpha.json")
return request
}
func bindCombatAssessment(request *contracts.TypedExtractionRequest[dnd.CombatTurnList]) *[]string {
validators := []contracts.TypedValidator[dnd.CombatTurnList]{
shape.New(shape.Options{}), sourcerefs.New(sourcerefs.Options{}), relatedness.New(relatedness.Options{}),
}
var calls []string
request.Operation.Assess = func(ctx context.Context, value dnd.CombatTurnList) (contracts.CandidateAssessment, error) {
var assessment contracts.CandidateAssessment
for _, validator := range validators {
calls = append(calls, validator.Name())
result, err := validator.Validate(ctx, contracts.TypedValidationRequest[dnd.CombatTurnList]{Stage: "extract", Source: request.Source, Chunk: request.Chunk, Value: value})
if err != nil {
return contracts.CandidateAssessment{}, err
}
decision := contracts.AssessmentRejected
if result.Approved {
decision = contracts.AssessmentApproved
}
assessment.Validators = append(assessment.Validators, contracts.ValidatorAssessment{ValidatorName: validator.Name(), Decision: decision, Result: result})
}
return assessment, nil
}
return &calls
}
func TestCombatNativeCorrectionKeepsEligibilityAndSourceContextFixed(t *testing.T) {
for _, first := range []string{missingActor, invalidCitation} {
request := nativeCombatRequest(t)
request.References.Slots[NPCRegistryReferenceSlot] = contracts.ResolvedReferenceSlot{Items: []contracts.ReferenceItem{
{SlotName: NPCRegistryReferenceSlot, MediaType: "application/json", Content: npcRegistryJSON(t)},
}}
request.Operation.Controls.MaxAdditionalTurns = 1
request.Operation.ReuseEligible = false
request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "combat-override"}
var prepared []promptkit.PreparedRun
request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = append(prepared, value) }
calls := bindCombatAssessment(&request)
client := &fakeCombatTurnsLLMClient{responses: []string{first, knownTurn}}
client.onGenerate = func(context.Context, promptkit.GenerateRequest) {
if len(client.requests) == 1 {
// Eligibility and projection were already captured. Neither may be resolved again.
request.References.Slots[SceneDescriptionReferenceSlot].Items[0].Content[0] = '!'
request.References.Slots[NPCRegistryReferenceSlot].Items[0].Content[0] = '!'
}
}
result, err := newExtractor(t, client, request.References).Extract(context.Background(), request)
chain := []string{shape.Key, sourcerefs.Key, relatedness.Key}
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || result.Outcome.ReuseEligible ||
!reflect.DeepEqual(*calls, append(append([]string(nil), chain...), chain...)) || len(client.requests) != 2 || len(prepared) != 2 || len(result.Value.CombatTurns) != 1 {
t.Fatalf("combat correction=%#v %v, validators=%#v native=%d/%d", result, err, *calls, len(client.requests), len(prepared))
}
turn := result.Value.CombatTurns[0]
if turn.Actor != "Aria" || turn.SourceRefs[0].SourceID != request.Source.ID || turn.SourceRefs[0].StartUnitID != 2 || turn.SourceRefs[0].EndUnitID != 10 {
t.Fatalf("document-order endpoints=%#v", turn)
}
for i, call := range client.requests {
id := PromptID
if i > 0 {
id = CorrectionPromptID
}
if prepared[i].PromptID != id || prepared[i].SelectedProfileID != request.LLMProfile || call.Target.Model != "combat-override" || call.Prompt.SessionID != request.SessionID {
t.Fatal("native selection lost")
}
if string(renderedInput(t, call, NPCRegistryReferenceSlot)) != `{"npcs":[{"name":"Mira Thorn"}]}` {
t.Fatal("names-only projection changed")
}
}
initial, revised := client.requests[0].Prompt.Messages, client.requests[1].Prompt.Messages
if len(revised) != len(initial)+2 || !reflect.DeepEqual(initial, revised[:len(initial)]) || revised[len(initial)].Role != "assistant" || revised[len(initial)].Content != first {
t.Fatal("native continuation lost fixed context or defective response")
}
if _, err := combatcodec.New().Encode(result.Value); err != nil {
t.Fatal(err)
}
}
}
func TestResolvableReversedCombatEndpointsNeedNoSemanticCorrection(t *testing.T) {
request := nativeCombatRequest(t)
request.Operation.Controls.MaxAdditionalTurns = 0
calls := bindCombatAssessment(&request)
client := &fakeCombatTurnsLLMClient{raw: knownTurn}
result, err := newExtractor(t, client).Extract(context.Background(), request)
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || len(client.requests) != 1 || len(*calls) != 3 ||
result.Value.CombatTurns[0].SourceRefs[0].StartUnitID != 2 || result.Value.CombatTurns[0].SourceRefs[0].EndUnitID != 10 {
t.Fatalf("endpoint reliability=%#v %v, native=%d assessments=%#v", result, err, len(client.requests), *calls)
}
}
func TestSkippedCombatIsAssessedWithoutNativePreparation(t *testing.T) {
request := extractionRequest(t)
request.References = sceneReferences(t, request.Chunk, dnd.SceneKindNarrative)
request.LLMProfile = "missing-profile-is-not-needed"
request.Operation.ReuseEligible = false
request.Operation.Controls.Cache = contracts.CacheAuto
prepared := 0
request.Operation.Execution.OnPrepared = func(context.Context, promptkit.PreparedRun) { prepared++ }
calls := bindCombatAssessment(&request)
client := &fakeCombatTurnsLLMClient{err: errors.New("must not execute")}
result, err := newExtractor(t, client).Extract(context.Background(), request)
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || result.Outcome.ReuseEligible ||
result.Value.CombatTurns == nil || len(result.Value.CombatTurns) != 0 || len(*calls) != 3 || len(client.requests) != 0 || prepared != 0 {
t.Fatalf("no-model outcome=%#v %v, native=%d/%d assessments=%#v", result, err, len(client.requests), prepared, *calls)
}
}
func TestCombatTerminalModesAndCancellation(t *testing.T) {
failure := errors.New("provider unavailable")
for _, test := range []struct {
name string
mode contracts.ExecutionMode
response string
failures []error
assessments, runs int
wantError bool
}{
{"strict exhaustion", contracts.ModeStrict, invalidCitation, nil, 2, 2, true},
{"permissive exhaustion", contracts.ModePermissive, invalidCitation, nil, 2, 2, false},
{"initial provider failure", contracts.ModePermissive, "", []error{failure}, 0, 1, false},
{"failed continuation", contracts.ModePermissive, invalidCitation, []error{nil, failure}, 1, 2, false},
{"structural failure", contracts.ModePermissive, `{"combat_turns":null}`, nil, 0, 1, false},
} {
t.Run(test.name, func(t *testing.T) {
request := extractionRequest(t)
request.Operation.Controls.Mode = test.mode
request.Operation.Controls.MaxAdditionalTurns = 1
calls := bindCombatAssessment(&request)
runs := 0
request.Operation.Execution.OnPrepared = func(context.Context, promptkit.PreparedRun) { runs++ }
client := &fakeCombatTurnsLLMClient{raw: test.response, errors: test.failures}
result, err := newExtractor(t, client).Extract(context.Background(), request)
if (err != nil) != test.wantError || len(*calls) != test.assessments*3 || runs != test.runs {
t.Fatalf("terminal=%#v %v, validators=%#v runs=%d", result, err, *calls, runs)
}
if test.wantError {
if result.Outcome.Disposition != "" {
t.Fatal("strict failure returned output")
}
return
}
if result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || result.Value.CombatTurns == nil || len(result.Value.CombatTurns) != 0 {
t.Fatalf("fallback=%#v", result)
}
if _, err := combatcodec.New().Encode(result.Value); err != nil {
t.Fatal(err)
}
})
}
for _, kind := range []dnd.SceneKind{dnd.SceneKindCombat, dnd.SceneKindNarrative} {
ctx, cancel := context.WithCancel(context.Background())
request := extractionRequest(t)
request.References = sceneReferences(t, request.Chunk, kind)
request.Operation.Controls.Mode = contracts.ModePermissive
client := &fakeCombatTurnsLLMClient{raw: knownTurn, onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }}
if kind == dnd.SceneKindNarrative {
cancel()
}
result, err := newExtractor(t, client).Extract(ctx, request)
cancel()
if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" {
t.Fatalf("cancellation=%#v %v", result, err)
}
}
}