package scenedescriptions import ( "context" "errors" "reflect" "strings" "testing" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" scenecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/scenedescriptions" semantics "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/combat_semantics" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/shape" sourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/source_refs" relatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/source_relatedness" "go.maximumdirect.net/promptkit" ) const narrativeScene = `{"kind":"narrative","title":"The watchtower","summary":"The party faces the bandits."}` const combatScene = `{"kind":"combat","title":"Watchtower fight","summary":"Bandits attack and the party drives them back."}` const blankScene = `{"kind":"combat","title":"","summary":""}` const combatJudgment = `{"classification":"combat","explanation":"Bandits attack the party during the current scene."}` func bindSceneAssessment(t *testing.T, extractor *Extractor, request *contracts.TypedExtractionRequest[dnd.SceneDescriptionList], withSemantic bool) *[]string { t.Helper() validators := []contracts.TypedValidator[dnd.SceneDescriptionList]{shape.New(shape.Options{}), sourcerefs.New(sourcerefs.Options{}), relatedness.New(relatedness.Options{})} if withSemantic { validator, err := semantics.New(extractor.engine, semantics.Options{}) if err != nil { t.Fatal(err) } validators = append(validators, validator) } var calls []string request.Operation.Assess = func(ctx context.Context, value dnd.SceneDescriptionList) (contracts.CandidateAssessment, error) { var assessment contracts.CandidateAssessment for _, validator := range validators { calls = append(calls, validator.Name()) profile := request.LLMProfile if validator.Name() == semantics.Key { profile = "scene-validator-profile" } result, err := validator.Validate(ctx, contracts.TypedValidationRequest[dnd.SceneDescriptionList]{ Stage: "extract", Source: request.Source, Chunk: request.Chunk, SourceInput: request.SourceInput, LLMProfile: profile, SessionID: request.SessionID, Execution: request.Operation.Execution, Value: value, }) if err != nil { return contracts.CandidateAssessment{}, err } decision := contracts.AssessmentRejected if result.Approved { decision = contracts.AssessmentApproved } assessment.Validators = append(assessment.Validators, contracts.ValidatorAssessment{ValidatorName: validator.Name(), Decision: decision, Result: result}) } return assessment, nil } return &calls } func TestSceneCorrectionUsesRealNativeCombatJudgmentsAndPreservesSourceIdentity(t *testing.T) { request := extractionRequest() request.Operation.Controls.MaxAdditionalTurns = 1 request.Operation.ReuseEligible = false request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "run-wide-scene-model"} var prepared []promptkit.PreparedRun request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = append(prepared, value) } client := &fakeSceneDescriptionsLLMClient{responses: []string{narrativeScene, combatJudgment, combatScene, combatJudgment}} extractor := newExtractor(t, client) calls := bindSceneAssessment(t, extractor, &request, true) result, err := extractor.Extract(context.Background(), request) chain := []string{shape.Key, sourcerefs.Key, relatedness.Key, semantics.Key} if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || result.Outcome.ReuseEligible || !reflect.DeepEqual(*calls, append(append([]string(nil), chain...), chain...)) || len(client.requests) != 4 || len(prepared) != 4 || len(result.Value.Scenes) != 1 { t.Fatalf("scene correction=%#v %v calls=%#v native=%d/%d", result, err, *calls, len(client.requests), len(prepared)) } scene := result.Value.Scenes[0] if scene.ID != request.Chunk.ID || scene.SourceRef != request.Chunk.Ref || scene.Kind != dnd.SceneKindCombat || scene.Title != "Watchtower fight" { t.Fatalf("mapped scene=%#v", scene) } for i, call := range client.requests { id, profile := PromptID, request.LLMProfile if i == 2 { id = CorrectionPromptID } if i%2 == 1 { id, profile = semantics.PromptID, "scene-validator-profile" } if prepared[i].PromptID != id || prepared[i].SelectedProfileID != profile || call.Target.Model != "run-wide-scene-model" || call.Prompt.SessionID != request.SessionID { t.Fatal("producer/validator native selections lost") } } initial, revised := client.requests[0].Prompt.Messages, client.requests[2].Prompt.Messages if len(revised) != len(initial)+2 || !reflect.DeepEqual(initial, revised[:len(initial)]) || revised[len(initial)].Role != "assistant" || revised[len(initial)].Content != narrativeScene { t.Fatal("continuation includes wrong history or regenerated context") } if !strings.Contains(revised[len(initial)+1].Content, "Return kind: combat for this scene.") || !strings.Contains(revised[len(initial)+1].Content, "Bandits attack the party") { t.Fatal("combat judgment guidance lost") } if _, err := scenecodec.New().Encode(result.Value); err != nil { t.Fatal(err) } } func TestPermissiveSceneRetainsOnlyFinalUsableCandidateWithoutSemanticRevalidation(t *testing.T) { request := extractionRequest() request.Operation.Controls.Mode = contracts.ModePermissive request.Operation.Controls.MaxAdditionalTurns = 1 // Both scenes remain non-combat, but the final title/summary must win. final := `{"kind":"narrative","title":"Final watchtower description","summary":"The party is attacked by bandits."}` client := &fakeSceneDescriptionsLLMClient{responses: []string{narrativeScene, combatJudgment, final, combatJudgment}} extractor := newExtractor(t, client) calls := bindSceneAssessment(t, extractor, &request, true) result, err := extractor.Extract(context.Background(), request) if err != nil || result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || result.Outcome.FallbackReason == "" || len(client.requests) != 4 || len(*calls) != 8 || len(result.Value.Scenes) != 1 { t.Fatalf("semantic fallback=%#v %v, native=%d assessments=%#v", result, err, len(client.requests), *calls) } scene := result.Value.Scenes[0] if scene.Title != "Final watchtower description" || scene.Kind != dnd.SceneKindNarrative || scene.SourceRef != request.Chunk.Ref || scene.ID != request.Chunk.ID { t.Fatalf("fallback recovered earlier scene or invented classification=%#v", scene) } if _, err := scenecodec.New().Encode(result.Value); err != nil { t.Fatal(err) } } func TestSceneCannotFallbackAfterMissingOrIntrinsicallyUnusableFinalResponse(t *testing.T) { failure := errors.New("provider unavailable") for _, test := range []struct { name string responses []string failures []error wantAssessments int }{ {"no first response", nil, []error{failure}, 0}, {"structural failure", []string{`{"kind":"unknown","title":"Bad","summary":"Bad"}`}, nil, 0}, {"blank final scene", []string{blankScene, blankScene}, nil, 2}, {"failed correction", []string{blankScene}, []error{nil, failure}, 1}, // An initially usable scene cannot be recovered after a result-less correction. {"lost usable prior scene", []string{narrativeScene}, []error{nil, failure}, 1}, {"structurally unusable revision", []string{narrativeScene, `{"kind":"unknown","title":"Bad","summary":"Bad"}`}, nil, 1}, } { t.Run(test.name, func(t *testing.T) { request := extractionRequest() request.Operation.Controls.Mode = contracts.ModePermissive request.Operation.Controls.MaxAdditionalTurns = 1 assessments := 0 request.Operation.Assess = func(context.Context, dnd.SceneDescriptionList) (contracts.CandidateAssessment, error) { assessments++ return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{ValidatorName: "test/reject", Decision: contracts.AssessmentRejected, Result: contracts.ValidationResult{ReasonCode: "test_rejection", Message: "Scene rejected.", CorrectionGuidance: "Correct the scene description."}}}}, nil } client := &fakeSceneDescriptionsLLMClient{responses: test.responses, errors: test.failures} if len(test.responses) == 1 { client.raw = test.responses[0] } result, err := newExtractor(t, client).Extract(context.Background(), request) if err == nil || result.Outcome.Disposition != "" || len(result.Value.Scenes) != 0 || assessments != test.wantAssessments { t.Fatalf("unusable scene=%#v %v, assessments=%d", result, err, assessments) } }) } } func TestStrictSceneRejectionAndCancellationDoNotPublishFallback(t *testing.T) { request := extractionRequest() request.Operation.Controls.MaxAdditionalTurns = 0 client := &fakeSceneDescriptionsLLMClient{responses: []string{narrativeScene, combatJudgment}} extractor := newExtractor(t, client) bindSceneAssessment(t, extractor, &request, true) result, err := extractor.Extract(context.Background(), request) if err == nil || result.Outcome.Disposition != "" || len(client.requests) != 2 { t.Fatalf("strict scene=%#v %v", result, err) } ctx, cancel := context.WithCancel(context.Background()) defer cancel() request.Operation.Controls.Mode = contracts.ModePermissive canceled := &fakeSceneDescriptionsLLMClient{raw: combatScene, onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }} result, err = newExtractor(t, canceled).Extract(ctx, request) if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" || len(canceled.requests) != 1 { t.Fatalf("cancellation=%#v %v", result, err) } }