Files
notarius/internal/modules/dnd/extract/scenedescriptions/native_execution_test.go

185 lines
9.7 KiB
Go
Raw Normal View History

package scenedescriptions
import (
"context"
"errors"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
scenecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/scenedescriptions"
semantics "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/combat_semantics"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/shape"
sourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/source_refs"
relatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/source_relatedness"
"go.maximumdirect.net/promptkit"
)
const narrativeScene = `{"kind":"narrative","title":"The watchtower","summary":"The party faces the bandits."}`
const combatScene = `{"kind":"combat","title":"Watchtower fight","summary":"Bandits attack and the party drives them back."}`
const blankScene = `{"kind":"combat","title":"","summary":""}`
const combatJudgment = `{"classification":"combat","explanation":"Bandits attack the party during the current scene."}`
func bindSceneAssessment(t *testing.T, extractor *Extractor, request *contracts.TypedExtractionRequest[dnd.SceneDescriptionList], withSemantic bool) *[]string {
t.Helper()
validators := []contracts.TypedValidator[dnd.SceneDescriptionList]{shape.New(shape.Options{}), sourcerefs.New(sourcerefs.Options{}), relatedness.New(relatedness.Options{})}
if withSemantic {
validator, err := semantics.New(extractor.engine, semantics.Options{})
if err != nil {
t.Fatal(err)
}
validators = append(validators, validator)
}
var calls []string
request.Operation.Assess = func(ctx context.Context, value dnd.SceneDescriptionList) (contracts.CandidateAssessment, error) {
var assessment contracts.CandidateAssessment
for _, validator := range validators {
calls = append(calls, validator.Name())
profile := request.LLMProfile
if validator.Name() == semantics.Key {
profile = "scene-validator-profile"
}
result, err := validator.Validate(ctx, contracts.TypedValidationRequest[dnd.SceneDescriptionList]{
Stage: "extract", Source: request.Source, Chunk: request.Chunk, SourceInput: request.SourceInput,
LLMProfile: profile, SessionID: request.SessionID, Execution: request.Operation.Execution, Value: value,
})
if err != nil {
return contracts.CandidateAssessment{}, err
}
decision := contracts.AssessmentRejected
if result.Approved {
decision = contracts.AssessmentApproved
}
assessment.Validators = append(assessment.Validators, contracts.ValidatorAssessment{ValidatorName: validator.Name(), Decision: decision, Result: result})
}
return assessment, nil
}
return &calls
}
func TestSceneCorrectionUsesRealNativeCombatJudgmentsAndPreservesSourceIdentity(t *testing.T) {
request := extractionRequest()
request.Operation.Controls.MaxAdditionalTurns = 1
request.Operation.ReuseEligible = false
request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "run-wide-scene-model"}
var prepared []promptkit.PreparedRun
request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = append(prepared, value) }
client := &fakeSceneDescriptionsLLMClient{responses: []string{narrativeScene, combatJudgment, combatScene, combatJudgment}}
extractor := newExtractor(t, client)
calls := bindSceneAssessment(t, extractor, &request, true)
result, err := extractor.Extract(context.Background(), request)
chain := []string{shape.Key, sourcerefs.Key, relatedness.Key, semantics.Key}
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || result.Outcome.ReuseEligible ||
!reflect.DeepEqual(*calls, append(append([]string(nil), chain...), chain...)) || len(client.requests) != 4 || len(prepared) != 4 || len(result.Value.Scenes) != 1 {
t.Fatalf("scene correction=%#v %v calls=%#v native=%d/%d", result, err, *calls, len(client.requests), len(prepared))
}
scene := result.Value.Scenes[0]
if scene.ID != request.Chunk.ID || scene.SourceRef != request.Chunk.Ref || scene.Kind != dnd.SceneKindCombat || scene.Title != "Watchtower fight" {
t.Fatalf("mapped scene=%#v", scene)
}
for i, call := range client.requests {
id, profile := PromptID, request.LLMProfile
if i == 2 {
id = CorrectionPromptID
}
if i%2 == 1 {
id, profile = semantics.PromptID, "scene-validator-profile"
}
if prepared[i].PromptID != id || prepared[i].SelectedProfileID != profile || call.Target.Model != "run-wide-scene-model" || call.Prompt.SessionID != request.SessionID {
t.Fatal("producer/validator native selections lost")
}
}
initial, revised := client.requests[0].Prompt.Messages, client.requests[2].Prompt.Messages
if len(revised) != len(initial)+2 || !reflect.DeepEqual(initial, revised[:len(initial)]) || revised[len(initial)].Role != "assistant" || revised[len(initial)].Content != narrativeScene {
t.Fatal("continuation includes wrong history or regenerated context")
}
if !strings.Contains(revised[len(initial)+1].Content, "Return kind: combat for this scene.") || !strings.Contains(revised[len(initial)+1].Content, "Bandits attack the party") {
t.Fatal("combat judgment guidance lost")
}
if _, err := scenecodec.New().Encode(result.Value); err != nil {
t.Fatal(err)
}
}
func TestPermissiveSceneRetainsOnlyFinalUsableCandidateWithoutSemanticRevalidation(t *testing.T) {
request := extractionRequest()
request.Operation.Controls.Mode = contracts.ModePermissive
request.Operation.Controls.MaxAdditionalTurns = 1
// Both scenes remain non-combat, but the final title/summary must win.
final := `{"kind":"narrative","title":"Final watchtower description","summary":"The party is attacked by bandits."}`
client := &fakeSceneDescriptionsLLMClient{responses: []string{narrativeScene, combatJudgment, final, combatJudgment}}
extractor := newExtractor(t, client)
calls := bindSceneAssessment(t, extractor, &request, true)
result, err := extractor.Extract(context.Background(), request)
if err != nil || result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || result.Outcome.FallbackReason == "" || len(client.requests) != 4 || len(*calls) != 8 || len(result.Value.Scenes) != 1 {
t.Fatalf("semantic fallback=%#v %v, native=%d assessments=%#v", result, err, len(client.requests), *calls)
}
scene := result.Value.Scenes[0]
if scene.Title != "Final watchtower description" || scene.Kind != dnd.SceneKindNarrative || scene.SourceRef != request.Chunk.Ref || scene.ID != request.Chunk.ID {
t.Fatalf("fallback recovered earlier scene or invented classification=%#v", scene)
}
if _, err := scenecodec.New().Encode(result.Value); err != nil {
t.Fatal(err)
}
}
func TestSceneCannotFallbackAfterMissingOrIntrinsicallyUnusableFinalResponse(t *testing.T) {
failure := errors.New("provider unavailable")
for _, test := range []struct {
name string
responses []string
failures []error
wantAssessments int
}{
{"no first response", nil, []error{failure}, 0},
{"structural failure", []string{`{"kind":"unknown","title":"Bad","summary":"Bad"}`}, nil, 0},
{"blank final scene", []string{blankScene, blankScene}, nil, 2},
{"failed correction", []string{blankScene}, []error{nil, failure}, 1},
// An initially usable scene cannot be recovered after a result-less correction.
{"lost usable prior scene", []string{narrativeScene}, []error{nil, failure}, 1},
{"structurally unusable revision", []string{narrativeScene, `{"kind":"unknown","title":"Bad","summary":"Bad"}`}, nil, 1},
} {
t.Run(test.name, func(t *testing.T) {
request := extractionRequest()
request.Operation.Controls.Mode = contracts.ModePermissive
request.Operation.Controls.MaxAdditionalTurns = 1
assessments := 0
request.Operation.Assess = func(context.Context, dnd.SceneDescriptionList) (contracts.CandidateAssessment, error) {
assessments++
return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{ValidatorName: "test/reject", Decision: contracts.AssessmentRejected,
Result: contracts.ValidationResult{ReasonCode: "test_rejection", Message: "Scene rejected.", CorrectionGuidance: "Correct the scene description."}}}}, nil
}
client := &fakeSceneDescriptionsLLMClient{responses: test.responses, errors: test.failures}
if len(test.responses) == 1 {
client.raw = test.responses[0]
}
result, err := newExtractor(t, client).Extract(context.Background(), request)
if err == nil || result.Outcome.Disposition != "" || len(result.Value.Scenes) != 0 || assessments != test.wantAssessments {
t.Fatalf("unusable scene=%#v %v, assessments=%d", result, err, assessments)
}
})
}
}
func TestStrictSceneRejectionAndCancellationDoNotPublishFallback(t *testing.T) {
request := extractionRequest()
request.Operation.Controls.MaxAdditionalTurns = 0
client := &fakeSceneDescriptionsLLMClient{responses: []string{narrativeScene, combatJudgment}}
extractor := newExtractor(t, client)
bindSceneAssessment(t, extractor, &request, true)
result, err := extractor.Extract(context.Background(), request)
if err == nil || result.Outcome.Disposition != "" || len(client.requests) != 2 {
t.Fatalf("strict scene=%#v %v", result, err)
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
request.Operation.Controls.Mode = contracts.ModePermissive
canceled := &fakeSceneDescriptionsLLMClient{raw: combatScene, onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }}
result, err = newExtractor(t, canceled).Extract(ctx, request)
if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" || len(canceled.requests) != 1 {
t.Fatalf("cancellation=%#v %v", result, err)
}
}