185 lines
9.7 KiB
Go
185 lines
9.7 KiB
Go
|
|
package scenedescriptions
|
||
|
|
|
||
|
|
import (
|
||
|
|
"context"
|
||
|
|
"errors"
|
||
|
|
"reflect"
|
||
|
|
"strings"
|
||
|
|
"testing"
|
||
|
|
|
||
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||
|
|
scenecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/scenedescriptions"
|
||
|
|
semantics "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/combat_semantics"
|
||
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/shape"
|
||
|
|
sourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/source_refs"
|
||
|
|
relatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/source_relatedness"
|
||
|
|
"go.maximumdirect.net/promptkit"
|
||
|
|
)
|
||
|
|
|
||
|
|
const narrativeScene = `{"kind":"narrative","title":"The watchtower","summary":"The party faces the bandits."}`
|
||
|
|
const combatScene = `{"kind":"combat","title":"Watchtower fight","summary":"Bandits attack and the party drives them back."}`
|
||
|
|
const blankScene = `{"kind":"combat","title":"","summary":""}`
|
||
|
|
const combatJudgment = `{"classification":"combat","explanation":"Bandits attack the party during the current scene."}`
|
||
|
|
|
||
|
|
func bindSceneAssessment(t *testing.T, extractor *Extractor, request *contracts.TypedExtractionRequest[dnd.SceneDescriptionList], withSemantic bool) *[]string {
|
||
|
|
t.Helper()
|
||
|
|
validators := []contracts.TypedValidator[dnd.SceneDescriptionList]{shape.New(shape.Options{}), sourcerefs.New(sourcerefs.Options{}), relatedness.New(relatedness.Options{})}
|
||
|
|
if withSemantic {
|
||
|
|
validator, err := semantics.New(extractor.engine, semantics.Options{})
|
||
|
|
if err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
validators = append(validators, validator)
|
||
|
|
}
|
||
|
|
var calls []string
|
||
|
|
request.Operation.Assess = func(ctx context.Context, value dnd.SceneDescriptionList) (contracts.CandidateAssessment, error) {
|
||
|
|
var assessment contracts.CandidateAssessment
|
||
|
|
for _, validator := range validators {
|
||
|
|
calls = append(calls, validator.Name())
|
||
|
|
profile := request.LLMProfile
|
||
|
|
if validator.Name() == semantics.Key {
|
||
|
|
profile = "scene-validator-profile"
|
||
|
|
}
|
||
|
|
result, err := validator.Validate(ctx, contracts.TypedValidationRequest[dnd.SceneDescriptionList]{
|
||
|
|
Stage: "extract", Source: request.Source, Chunk: request.Chunk, SourceInput: request.SourceInput,
|
||
|
|
LLMProfile: profile, SessionID: request.SessionID, Execution: request.Operation.Execution, Value: value,
|
||
|
|
})
|
||
|
|
if err != nil {
|
||
|
|
return contracts.CandidateAssessment{}, err
|
||
|
|
}
|
||
|
|
decision := contracts.AssessmentRejected
|
||
|
|
if result.Approved {
|
||
|
|
decision = contracts.AssessmentApproved
|
||
|
|
}
|
||
|
|
assessment.Validators = append(assessment.Validators, contracts.ValidatorAssessment{ValidatorName: validator.Name(), Decision: decision, Result: result})
|
||
|
|
}
|
||
|
|
return assessment, nil
|
||
|
|
}
|
||
|
|
return &calls
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestSceneCorrectionUsesRealNativeCombatJudgmentsAndPreservesSourceIdentity(t *testing.T) {
|
||
|
|
request := extractionRequest()
|
||
|
|
request.Operation.Controls.MaxAdditionalTurns = 1
|
||
|
|
request.Operation.ReuseEligible = false
|
||
|
|
request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "run-wide-scene-model"}
|
||
|
|
var prepared []promptkit.PreparedRun
|
||
|
|
request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = append(prepared, value) }
|
||
|
|
client := &fakeSceneDescriptionsLLMClient{responses: []string{narrativeScene, combatJudgment, combatScene, combatJudgment}}
|
||
|
|
extractor := newExtractor(t, client)
|
||
|
|
calls := bindSceneAssessment(t, extractor, &request, true)
|
||
|
|
result, err := extractor.Extract(context.Background(), request)
|
||
|
|
chain := []string{shape.Key, sourcerefs.Key, relatedness.Key, semantics.Key}
|
||
|
|
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || result.Outcome.ReuseEligible ||
|
||
|
|
!reflect.DeepEqual(*calls, append(append([]string(nil), chain...), chain...)) || len(client.requests) != 4 || len(prepared) != 4 || len(result.Value.Scenes) != 1 {
|
||
|
|
t.Fatalf("scene correction=%#v %v calls=%#v native=%d/%d", result, err, *calls, len(client.requests), len(prepared))
|
||
|
|
}
|
||
|
|
scene := result.Value.Scenes[0]
|
||
|
|
if scene.ID != request.Chunk.ID || scene.SourceRef != request.Chunk.Ref || scene.Kind != dnd.SceneKindCombat || scene.Title != "Watchtower fight" {
|
||
|
|
t.Fatalf("mapped scene=%#v", scene)
|
||
|
|
}
|
||
|
|
for i, call := range client.requests {
|
||
|
|
id, profile := PromptID, request.LLMProfile
|
||
|
|
if i == 2 {
|
||
|
|
id = CorrectionPromptID
|
||
|
|
}
|
||
|
|
if i%2 == 1 {
|
||
|
|
id, profile = semantics.PromptID, "scene-validator-profile"
|
||
|
|
}
|
||
|
|
if prepared[i].PromptID != id || prepared[i].SelectedProfileID != profile || call.Target.Model != "run-wide-scene-model" || call.Prompt.SessionID != request.SessionID {
|
||
|
|
t.Fatal("producer/validator native selections lost")
|
||
|
|
}
|
||
|
|
}
|
||
|
|
initial, revised := client.requests[0].Prompt.Messages, client.requests[2].Prompt.Messages
|
||
|
|
if len(revised) != len(initial)+2 || !reflect.DeepEqual(initial, revised[:len(initial)]) || revised[len(initial)].Role != "assistant" || revised[len(initial)].Content != narrativeScene {
|
||
|
|
t.Fatal("continuation includes wrong history or regenerated context")
|
||
|
|
}
|
||
|
|
if !strings.Contains(revised[len(initial)+1].Content, "Return kind: combat for this scene.") || !strings.Contains(revised[len(initial)+1].Content, "Bandits attack the party") {
|
||
|
|
t.Fatal("combat judgment guidance lost")
|
||
|
|
}
|
||
|
|
if _, err := scenecodec.New().Encode(result.Value); err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestPermissiveSceneRetainsOnlyFinalUsableCandidateWithoutSemanticRevalidation(t *testing.T) {
|
||
|
|
request := extractionRequest()
|
||
|
|
request.Operation.Controls.Mode = contracts.ModePermissive
|
||
|
|
request.Operation.Controls.MaxAdditionalTurns = 1
|
||
|
|
// Both scenes remain non-combat, but the final title/summary must win.
|
||
|
|
final := `{"kind":"narrative","title":"Final watchtower description","summary":"The party is attacked by bandits."}`
|
||
|
|
client := &fakeSceneDescriptionsLLMClient{responses: []string{narrativeScene, combatJudgment, final, combatJudgment}}
|
||
|
|
extractor := newExtractor(t, client)
|
||
|
|
calls := bindSceneAssessment(t, extractor, &request, true)
|
||
|
|
result, err := extractor.Extract(context.Background(), request)
|
||
|
|
if err != nil || result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || result.Outcome.FallbackReason == "" || len(client.requests) != 4 || len(*calls) != 8 || len(result.Value.Scenes) != 1 {
|
||
|
|
t.Fatalf("semantic fallback=%#v %v, native=%d assessments=%#v", result, err, len(client.requests), *calls)
|
||
|
|
}
|
||
|
|
scene := result.Value.Scenes[0]
|
||
|
|
if scene.Title != "Final watchtower description" || scene.Kind != dnd.SceneKindNarrative || scene.SourceRef != request.Chunk.Ref || scene.ID != request.Chunk.ID {
|
||
|
|
t.Fatalf("fallback recovered earlier scene or invented classification=%#v", scene)
|
||
|
|
}
|
||
|
|
if _, err := scenecodec.New().Encode(result.Value); err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestSceneCannotFallbackAfterMissingOrIntrinsicallyUnusableFinalResponse(t *testing.T) {
|
||
|
|
failure := errors.New("provider unavailable")
|
||
|
|
for _, test := range []struct {
|
||
|
|
name string
|
||
|
|
responses []string
|
||
|
|
failures []error
|
||
|
|
wantAssessments int
|
||
|
|
}{
|
||
|
|
{"no first response", nil, []error{failure}, 0},
|
||
|
|
{"structural failure", []string{`{"kind":"unknown","title":"Bad","summary":"Bad"}`}, nil, 0},
|
||
|
|
{"blank final scene", []string{blankScene, blankScene}, nil, 2},
|
||
|
|
{"failed correction", []string{blankScene}, []error{nil, failure}, 1},
|
||
|
|
// An initially usable scene cannot be recovered after a result-less correction.
|
||
|
|
{"lost usable prior scene", []string{narrativeScene}, []error{nil, failure}, 1},
|
||
|
|
{"structurally unusable revision", []string{narrativeScene, `{"kind":"unknown","title":"Bad","summary":"Bad"}`}, nil, 1},
|
||
|
|
} {
|
||
|
|
t.Run(test.name, func(t *testing.T) {
|
||
|
|
request := extractionRequest()
|
||
|
|
request.Operation.Controls.Mode = contracts.ModePermissive
|
||
|
|
request.Operation.Controls.MaxAdditionalTurns = 1
|
||
|
|
assessments := 0
|
||
|
|
request.Operation.Assess = func(context.Context, dnd.SceneDescriptionList) (contracts.CandidateAssessment, error) {
|
||
|
|
assessments++
|
||
|
|
return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{ValidatorName: "test/reject", Decision: contracts.AssessmentRejected,
|
||
|
|
Result: contracts.ValidationResult{ReasonCode: "test_rejection", Message: "Scene rejected.", CorrectionGuidance: "Correct the scene description."}}}}, nil
|
||
|
|
}
|
||
|
|
client := &fakeSceneDescriptionsLLMClient{responses: test.responses, errors: test.failures}
|
||
|
|
if len(test.responses) == 1 {
|
||
|
|
client.raw = test.responses[0]
|
||
|
|
}
|
||
|
|
result, err := newExtractor(t, client).Extract(context.Background(), request)
|
||
|
|
if err == nil || result.Outcome.Disposition != "" || len(result.Value.Scenes) != 0 || assessments != test.wantAssessments {
|
||
|
|
t.Fatalf("unusable scene=%#v %v, assessments=%d", result, err, assessments)
|
||
|
|
}
|
||
|
|
})
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestStrictSceneRejectionAndCancellationDoNotPublishFallback(t *testing.T) {
|
||
|
|
request := extractionRequest()
|
||
|
|
request.Operation.Controls.MaxAdditionalTurns = 0
|
||
|
|
client := &fakeSceneDescriptionsLLMClient{responses: []string{narrativeScene, combatJudgment}}
|
||
|
|
extractor := newExtractor(t, client)
|
||
|
|
bindSceneAssessment(t, extractor, &request, true)
|
||
|
|
result, err := extractor.Extract(context.Background(), request)
|
||
|
|
if err == nil || result.Outcome.Disposition != "" || len(client.requests) != 2 {
|
||
|
|
t.Fatalf("strict scene=%#v %v", result, err)
|
||
|
|
}
|
||
|
|
ctx, cancel := context.WithCancel(context.Background())
|
||
|
|
defer cancel()
|
||
|
|
request.Operation.Controls.Mode = contracts.ModePermissive
|
||
|
|
canceled := &fakeSceneDescriptionsLLMClient{raw: combatScene, onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }}
|
||
|
|
result, err = newExtractor(t, canceled).Extract(ctx, request)
|
||
|
|
if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" || len(canceled.requests) != 1 {
|
||
|
|
t.Fatalf("cancellation=%#v %v", result, err)
|
||
|
|
}
|
||
|
|
}
|