package cli import ( "bytes" "context" "encoding/json" "errors" "fmt" "os" "path/filepath" "reflect" "strings" "sync" "testing" "time" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" "go.maximumdirect.net/commandkit" "go.maximumdirect.net/promptkit" ) // Use the maintained ordered domain composition and existing provider fake. // Only external failures and one rejected scene candidate are controlled here. type recoveryClient struct { base enemyEventLLMClient mu sync.Mutex requests []promptkit.GenerateRequest failEvents, failNPC, correctScene bool failure error } func (client *recoveryClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { role := enemyEventRequestRole(request) if productionSchemaHasProperty(request, "classification") { role = "classification" } client.mu.Lock() client.requests = append(client.requests, request) failEvents, failNPC := client.failEvents, client.failNPC transcript, _ := productionJSONInput(request, "units") combat := strings.Contains(string(transcript), "Roll initiative") incorrectScene := role == "kind" && combat && client.correctScene if incorrectScene { client.correctScene = false } client.mu.Unlock() if role == "events" && failEvents || role == "npcs" && combat && failNPC { return nil, client.failure } if incorrectScene { return &promptkit.GenerateResponse{Content: `{"kind":"narrative","title":"Arrival","summary":"An initially missed combat scene."}`}, nil } if role == "classification" { classification := "non_combat" if combat { classification = "combat" } return &promptkit.GenerateResponse{Content: fmt.Sprintf(`{"classification":%q,"explanation":"Classification follows the supplied scene transcript."}`, classification)}, nil } // Empty generated registries remain valid current-run context. Do not invent // identities or events merely to fill the dependent lanes in the fallback run. if role == "npc-occurrences" || role == "combat_turns" || role == "events" { if content, err := productionJSONInput(request, "npcs"); err == nil { var registry struct { NPCs []json.RawMessage `json:"npcs"` } if err := json.Unmarshal(content, ®istry); err != nil { return nil, err } if len(registry.NPCs) == 0 { property := "occurrences" if role == "combat_turns" { property = "combat_turns" } if role == "events" { property = "events" } return &promptkit.GenerateResponse{Content: fmt.Sprintf(`{%q:[]}`, property)}, nil } } } return client.base.Generate(ctx, request) } func (client *recoveryClient) snapshot() (map[string]int, []promptkit.GenerateRequest) { client.mu.Lock() defer client.mu.Unlock() counts := map[string]int{} for _, request := range client.requests { role := enemyEventRequestRole(request) if productionSchemaHasProperty(request, "classification") { role = "classification" } counts[role]++ } return counts, append([]promptkit.GenerateRequest(nil), client.requests...) } type recoveryFixture struct { config, cache, output, debug string client *recoveryClient options Options } func newRecoveryFixture(t *testing.T) *recoveryFixture { t.Helper() root := t.TempDir() for _, name := range []string{"prompts.yml", "transcript.json", "party.txt", "glossary.txt", "spell-catalog.json", "pipelines/dnd-session.yml", "pipelines/dnd-session.describe-session.yml"} { content, err := os.ReadFile(repositoryPath("examples", "dnd-complete", name)) if err != nil { t.Fatal(err) } writeConfigKitFile(t, filepath.Join(root, name), string(content)) } description := filepath.Join(root, "pipelines", "dnd-session.describe-session.yml") content, err := os.ReadFile(description) if err != nil { t.Fatal(err) } configured := replaceRequiredOnce(t, string(content), " extract:\n module: dnd/scene-descriptions\n", ` extract: module: dnd/scene-descriptions validators: - module: generic/valid_json - module: extract/dnd/scene-descriptions/shape - module: extract/dnd/scene-descriptions/source_refs - module: generic/valid_json_schema - module: extract/dnd/scene-descriptions/source_relatedness - module: extract/dnd/scene-descriptions/combat_semantics `) writeConfigKitFile(t, description, configured) fixture := &recoveryFixture{config: filepath.Join(root, "config.yml"), cache: filepath.Join(root, "cache"), output: filepath.Join(root, "output"), debug: filepath.Join(root, "debug"), client: &recoveryClient{failure: errors.New("controlled downstream provider failure")}} writeConfigKitFile(t, fixture.config, fmt.Sprintf(`version: 6 config: imports: - path: pipelines/dnd-session.yml required: true output: {directory: %q} cache_directory: %q debug: {directory: %q} concurrency: {workers: 1} promptkit: backends: - id: fixture endpoint: https://example.invalid/v1 concurrency_limit: 1 collections: notarius: profiles: - id: dnd-extraction backend: fixture model: recovery-model api_key_required: false timeout_seconds: 2 - id: dnd-extraction-example extends: {id: dnd-extraction} `, fixture.output, fixture.cache, fixture.debug)) fixture.options = productionRunOptions(t, fixture.client) run := 0 fixture.options.RunIDGenerator = func(time.Time) (string, error) { run++; return fmt.Sprintf("recovery-%d", run), nil } return fixture } func (fixture *recoveryFixture) run(t *testing.T) (commandkit.Outcome, runResult, string) { t.Helper() args := []string{"run", "dnd-session", "--config", fixture.config, "--input-file", filepath.Join(filepath.Dir(fixture.config), "transcript.json"), "--debug", "--progress=false", "--format=json", "--llm-session-id=recovery-session", "--llm-transient-retries=0", "--set", "/pipelines/dnd-session/max_additional_turns=1", } app, err := NewApplication("development", fixture.options) if err != nil { t.Fatal(err) } ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() var stdout, stderr bytes.Buffer outcome := app.Execute(ctx, args, commandkit.Streams{Out: &stdout, Err: &stderr}) var report runResult if outcome.Status == 0 { if err := json.Unmarshal(stdout.Bytes(), &report); err != nil { t.Fatalf("completion=%q: %v", stdout.String(), err) } } else if stdout.Len() != 0 { t.Fatalf("failed operation emitted success output: %q", stdout.String()) } return outcome, report, stderr.String() } func TestAssembledRecoveryReusesCorrectedPredecessorsAndChunkPlan(t *testing.T) { fixture := newRecoveryFixture(t) fixture.client.failEvents, fixture.client.correctScene = true, true outcome, _, diagnostics := fixture.run(t) if outcome.Status != 1 || !errors.Is(outcome.Cause, fixture.client.failure) { t.Fatalf("downstream failure=%#v diagnostics=%q", outcome, diagnostics) } assertAbsent(t, fixture.output) firstDebug := filepath.Join(fixture.debug, "recovery-1") assertFile(t, filepath.Join(firstDebug, "outcome.json")) counts, requests := fixture.client.snapshot() if counts["scenes"] != 1 || counts["kind"] != 3 || counts["classification"] != 3 || counts["duplicate_groups"] == 0 || counts["events"] != 1 { t.Fatalf("ordered native roles/correction=%v", counts) } for _, request := range requests { if request.Prompt.SessionID != "recovery-session" || request.Target.Model != "recovery-model" || request.Target.BackendID != "fixture" { t.Fatalf("lost shared native selection: %#v", request) } } summary := readProductionJSON[promptkit.ExecutionGroupSummary](t, filepath.Join(firstDebug, "executions.json")) if len(summary.Members) != len(requests) || summary.Attempts != uint64(len(requests)) || !summary.TerminalCoverageComplete || summary.Usage.TotalTokens.Known { t.Fatalf("failed-run native facts lost or duplicated: %#v", summary) } // Locate only the initial scene request's native hash in available capture; // assert domain bytes at that key, not a persisted execution ledger. var correctedKey string for name, content := range readTree(t, firstDebug) { if !strings.HasSuffix(name, "/prepared.json") { continue } var details promptkit.PreparedRun if json.Unmarshal(content, &details) != nil || details.PromptID != "notarius/dnd.scene_descriptions" { continue } for _, message := range details.Messages { if strings.Contains(message.Content, "Roll initiative") { correctedKey = details.MessageContentHash } } } if correctedKey == "" { t.Fatal("no prepared initial combat scene evidence") } corrected := readProductionJSON[dnd.SceneDescriptionList](t, filepath.Join(fixture.cache, correctedKey+".json")) if len(corrected.Scenes) != 1 || corrected.Scenes[0].Kind != dnd.SceneKindCombat { t.Fatalf("cache did not retain the corrected domain candidate: %#v", corrected) } fixture.client.failEvents = false outcome, report, diagnostics := fixture.run(t) if outcome.Status != 0 || report.FallbackCount != 0 || report.ArtifactCount != 12 { t.Fatalf("recovery=%#v report=%#v diagnostics=%q", outcome, report, diagnostics) } after, _ := fixture.client.snapshot() for role, count := range counts { want := count if role == "events" { want++ } if after[role] != want { t.Fatalf("recovery reran accepted %s: %v -> %v", role, counts, after) } } artifact := readProductionJSON[dnd.EnemyEventList](t, filepath.Join(report.OutputDirectory, "lanes", "enemy-events.json")) if len(artifact.Events) != 1 || artifact.Events[0].SourceRefs[0].SourceID != "session-ravenfall" { t.Fatalf("recovered domain evidence=%#v", artifact) } stable := readTree(t, fixture.cache) outcome, hitReport, diagnostics := fixture.run(t) if outcome.Status != 0 || hitReport.ArtifactCount != 12 { t.Fatalf("cached run=%#v diagnostics=%q", outcome, diagnostics) } last, _ := fixture.client.snapshot() if !reflect.DeepEqual(last, after) || !sameFiles(stable, readTree(t, fixture.cache)) { t.Fatalf("full hit executed or rewrote acceptance: %v -> %v", after, last) } for name := range readTree(t, hitReport.DebugDirectory) { if strings.Contains(name, "/native/") && strings.HasSuffix(name, "/result.json") { t.Fatalf("full hit fabricated native execution: %s", name) } } hitSummary := readProductionJSON[promptkit.ExecutionGroupSummary](t, filepath.Join(hitReport.DebugDirectory, "executions.json")) if len(hitSummary.Members) != 0 || !hitSummary.TerminalCoverageComplete { t.Fatalf("full hit fabricated historical execution totals: %#v", hitSummary) } } func TestAssembledFallbackSkipsDependentReuseAndPreservesFailedRefresh(t *testing.T) { fixture := newRecoveryFixture(t) outcome, _, diagnostics := fixture.run(t) if outcome.Status != 0 { t.Fatalf("initial run=%#v diagnostics=%q", outcome, diagnostics) } before, _ := fixture.client.snapshot() accepted := readTree(t, fixture.cache) fixture.client.failNPC = true description := filepath.Join(filepath.Dir(fixture.config), "pipelines", "dnd-session.describe-session.yml") content, err := os.ReadFile(description) if err != nil { t.Fatal(err) } writeConfigKitFile(t, description, replaceRequiredOnce(t, string(content), " extract:\n module: dnd/npc-registry\n", " extract:\n module: dnd/npc-registry\n cache: refresh\n mode: permissive\n")) outcome, report, diagnostics := fixture.run(t) if outcome.Status != 0 || report.FallbackCount != 1 { t.Fatalf("fallback=%#v report=%#v diagnostics=%q", outcome, report, diagnostics) } after, _ := fixture.client.snapshot() for _, role := range []string{"scenes", "kind", "classification", "locations", "items", "location-occurrences", "duplicate_groups"} { if after[role] != before[role] { t.Fatalf("unaffected %s reran: %v -> %v", role, before, after) } } for _, role := range []string{"npcs", "spell_casts", "npc-occurrences", "combat_turns", "events"} { if after[role] <= before[role] { t.Fatalf("affected %s reused earlier acceptance: %v -> %v", role, before, after) } } if !sameFiles(accepted, readTree(t, fixture.cache)) { t.Fatal("failed refresh/fallback replaced accepted cache data") } registry := readProductionJSON[dnd.NPCRegistry](t, filepath.Join(report.OutputDirectory, "lanes", "npc-registry.json")) if registry.NPCs == nil || len(registry.NPCs) != 0 { t.Fatalf("fallback registry=%#v", registry) } fixture.client.failNPC = false writeConfigKitFile(t, description, string(content)) outcome, report, diagnostics = fixture.run(t) if outcome.Status != 0 || report.FallbackCount != 0 { t.Fatalf("restored acceptance=%#v report=%#v diagnostics=%q", outcome, report, diagnostics) } last, _ := fixture.client.snapshot() if !reflect.DeepEqual(last, after) { t.Fatalf("failed refresh lost earlier reusable results: %v -> %v", after, last) } } var _ promptkit.LLMClient = (*recoveryClient)(nil)