Files
notarius/internal/cli/recovery_contract_test.go

323 lines
13 KiB
Go

package cli
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"reflect"
"strings"
"sync"
"testing"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
"go.maximumdirect.net/commandkit"
"go.maximumdirect.net/promptkit"
)
// Use the maintained ordered domain composition and existing provider fake.
// Only external failures and one rejected scene candidate are controlled here.
type recoveryClient struct {
base enemyEventLLMClient
mu sync.Mutex
requests []promptkit.GenerateRequest
failEvents, failNPC, correctScene bool
failure error
}
func (client *recoveryClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
role := enemyEventRequestRole(request)
if productionSchemaHasProperty(request, "classification") {
role = "classification"
}
client.mu.Lock()
client.requests = append(client.requests, request)
failEvents, failNPC := client.failEvents, client.failNPC
transcript, _ := productionJSONInput(request, "units")
combat := strings.Contains(string(transcript), "Roll initiative")
incorrectScene := role == "kind" && combat && client.correctScene
if incorrectScene {
client.correctScene = false
}
client.mu.Unlock()
if role == "events" && failEvents || role == "npcs" && combat && failNPC {
return nil, client.failure
}
if incorrectScene {
return &promptkit.GenerateResponse{Content: `{"kind":"narrative","title":"Arrival","summary":"An initially missed combat scene."}`}, nil
}
if role == "classification" {
classification := "non_combat"
if combat {
classification = "combat"
}
return &promptkit.GenerateResponse{Content: fmt.Sprintf(`{"classification":%q,"explanation":"Classification follows the supplied scene transcript."}`, classification)}, nil
}
// Empty generated registries remain valid current-run context. Do not invent
// identities or events merely to fill the dependent lanes in the fallback run.
if role == "npc-occurrences" || role == "combat_turns" || role == "events" {
if content, err := productionJSONInput(request, "npcs"); err == nil {
var registry struct {
NPCs []json.RawMessage `json:"npcs"`
}
if err := json.Unmarshal(content, &registry); err != nil {
return nil, err
}
if len(registry.NPCs) == 0 {
property := "occurrences"
if role == "combat_turns" {
property = "combat_turns"
}
if role == "events" {
property = "events"
}
return &promptkit.GenerateResponse{Content: fmt.Sprintf(`{%q:[]}`, property)}, nil
}
}
}
return client.base.Generate(ctx, request)
}
func (client *recoveryClient) snapshot() (map[string]int, []promptkit.GenerateRequest) {
client.mu.Lock()
defer client.mu.Unlock()
counts := map[string]int{}
for _, request := range client.requests {
role := enemyEventRequestRole(request)
if productionSchemaHasProperty(request, "classification") {
role = "classification"
}
counts[role]++
}
return counts, append([]promptkit.GenerateRequest(nil), client.requests...)
}
type recoveryFixture struct {
config, cache, output, debug string
client *recoveryClient
options Options
}
func newRecoveryFixture(t *testing.T) *recoveryFixture {
t.Helper()
root := t.TempDir()
for _, name := range []string{"prompts.yml", "transcript.json", "party.txt", "glossary.txt", "spell-catalog.json", "pipelines/dnd-session.yml", "pipelines/dnd-session.describe-session.yml"} {
content, err := os.ReadFile(repositoryPath("examples", "dnd-complete", name))
if err != nil {
t.Fatal(err)
}
writeConfigKitFile(t, filepath.Join(root, name), string(content))
}
description := filepath.Join(root, "pipelines", "dnd-session.describe-session.yml")
content, err := os.ReadFile(description)
if err != nil {
t.Fatal(err)
}
configured := replaceRequiredOnce(t, string(content), " extract:\n module: dnd/scene-descriptions\n", ` extract:
module: dnd/scene-descriptions
validators:
- module: generic/valid_json
- module: extract/dnd/scene-descriptions/shape
- module: extract/dnd/scene-descriptions/source_refs
- module: generic/valid_json_schema
- module: extract/dnd/scene-descriptions/source_relatedness
- module: extract/dnd/scene-descriptions/combat_semantics
`)
writeConfigKitFile(t, description, configured)
fixture := &recoveryFixture{config: filepath.Join(root, "config.yml"), cache: filepath.Join(root, "cache"), output: filepath.Join(root, "output"), debug: filepath.Join(root, "debug"), client: &recoveryClient{failure: errors.New("controlled downstream provider failure")}}
writeConfigKitFile(t, fixture.config, fmt.Sprintf(`version: 6
config:
imports:
- path: pipelines/dnd-session.yml
required: true
output: {directory: %q}
cache_directory: %q
debug: {directory: %q}
concurrency: {workers: 1}
promptkit:
backends:
- id: fixture
endpoint: https://example.invalid/v1
concurrency_limit: 1
collections:
notarius:
profiles:
- id: dnd-extraction
backend: fixture
model: recovery-model
api_key_required: false
timeout_seconds: 2
- id: dnd-extraction-example
extends: {id: dnd-extraction}
`, fixture.output, fixture.cache, fixture.debug))
fixture.options = productionRunOptions(t, fixture.client)
run := 0
fixture.options.RunIDGenerator = func(time.Time) (string, error) { run++; return fmt.Sprintf("recovery-%d", run), nil }
return fixture
}
func (fixture *recoveryFixture) run(t *testing.T) (commandkit.Outcome, runResult, string) {
t.Helper()
args := []string{"run", "dnd-session", "--config", fixture.config, "--input-file", filepath.Join(filepath.Dir(fixture.config), "transcript.json"), "--debug", "--progress=false", "--format=json", "--llm-session-id=recovery-session", "--llm-transient-retries=0",
"--set", "/pipelines/dnd-session/max_additional_turns=1",
}
app, err := NewApplication("development", fixture.options)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
var stdout, stderr bytes.Buffer
outcome := app.Execute(ctx, args, commandkit.Streams{Out: &stdout, Err: &stderr})
var report runResult
if outcome.Status == 0 {
if err := json.Unmarshal(stdout.Bytes(), &report); err != nil {
t.Fatalf("completion=%q: %v", stdout.String(), err)
}
} else if stdout.Len() != 0 {
t.Fatalf("failed operation emitted success output: %q", stdout.String())
}
return outcome, report, stderr.String()
}
func TestAssembledRecoveryReusesCorrectedPredecessorsAndChunkPlan(t *testing.T) {
fixture := newRecoveryFixture(t)
fixture.client.failEvents, fixture.client.correctScene = true, true
outcome, _, diagnostics := fixture.run(t)
if outcome.Status != 1 || !errors.Is(outcome.Cause, fixture.client.failure) {
t.Fatalf("downstream failure=%#v diagnostics=%q", outcome, diagnostics)
}
assertAbsent(t, fixture.output)
firstDebug := filepath.Join(fixture.debug, "recovery-1")
assertFile(t, filepath.Join(firstDebug, "outcome.json"))
counts, requests := fixture.client.snapshot()
if counts["scenes"] != 1 || counts["kind"] != 3 || counts["classification"] != 3 || counts["duplicate_groups"] == 0 || counts["events"] != 1 {
t.Fatalf("ordered native roles/correction=%v", counts)
}
for _, request := range requests {
if request.Prompt.SessionID != "recovery-session" || request.Target.Model != "recovery-model" || request.Target.BackendID != "fixture" {
t.Fatalf("lost shared native selection: %#v", request)
}
}
summary := readProductionJSON[promptkit.ExecutionGroupSummary](t, filepath.Join(firstDebug, "executions.json"))
if len(summary.Members) != len(requests) || summary.Attempts != uint64(len(requests)) || !summary.TerminalCoverageComplete || summary.Usage.TotalTokens.Known {
t.Fatalf("failed-run native facts lost or duplicated: %#v", summary)
}
// Locate only the initial scene request's native hash in available capture;
// assert domain bytes at that key, not a persisted execution ledger.
var correctedKey string
for name, content := range readTree(t, firstDebug) {
if !strings.HasSuffix(name, "/prepared.json") {
continue
}
var details promptkit.PreparedRun
if json.Unmarshal(content, &details) != nil || details.PromptID != "notarius/dnd.scene_descriptions" {
continue
}
for _, message := range details.Messages {
if strings.Contains(message.Content, "Roll initiative") {
correctedKey = details.MessageContentHash
}
}
}
if correctedKey == "" {
t.Fatal("no prepared initial combat scene evidence")
}
corrected := readProductionJSON[dnd.SceneDescriptionList](t, filepath.Join(fixture.cache, correctedKey+".json"))
if len(corrected.Scenes) != 1 || corrected.Scenes[0].Kind != dnd.SceneKindCombat {
t.Fatalf("cache did not retain the corrected domain candidate: %#v", corrected)
}
fixture.client.failEvents = false
outcome, report, diagnostics := fixture.run(t)
if outcome.Status != 0 || report.FallbackCount != 0 || report.ArtifactCount != 12 {
t.Fatalf("recovery=%#v report=%#v diagnostics=%q", outcome, report, diagnostics)
}
after, _ := fixture.client.snapshot()
for role, count := range counts {
want := count
if role == "events" {
want++
}
if after[role] != want {
t.Fatalf("recovery reran accepted %s: %v -> %v", role, counts, after)
}
}
artifact := readProductionJSON[dnd.EnemyEventList](t, filepath.Join(report.OutputDirectory, "lanes", "enemy-events.json"))
if len(artifact.Events) != 1 || artifact.Events[0].SourceRefs[0].SourceID != "session-ravenfall" {
t.Fatalf("recovered domain evidence=%#v", artifact)
}
stable := readTree(t, fixture.cache)
outcome, hitReport, diagnostics := fixture.run(t)
if outcome.Status != 0 || hitReport.ArtifactCount != 12 {
t.Fatalf("cached run=%#v diagnostics=%q", outcome, diagnostics)
}
last, _ := fixture.client.snapshot()
if !reflect.DeepEqual(last, after) || !sameFiles(stable, readTree(t, fixture.cache)) {
t.Fatalf("full hit executed or rewrote acceptance: %v -> %v", after, last)
}
for name := range readTree(t, hitReport.DebugDirectory) {
if strings.Contains(name, "/native/") && strings.HasSuffix(name, "/result.json") {
t.Fatalf("full hit fabricated native execution: %s", name)
}
}
hitSummary := readProductionJSON[promptkit.ExecutionGroupSummary](t, filepath.Join(hitReport.DebugDirectory, "executions.json"))
if len(hitSummary.Members) != 0 || !hitSummary.TerminalCoverageComplete {
t.Fatalf("full hit fabricated historical execution totals: %#v", hitSummary)
}
}
func TestAssembledFallbackSkipsDependentReuseAndPreservesFailedRefresh(t *testing.T) {
fixture := newRecoveryFixture(t)
outcome, _, diagnostics := fixture.run(t)
if outcome.Status != 0 {
t.Fatalf("initial run=%#v diagnostics=%q", outcome, diagnostics)
}
before, _ := fixture.client.snapshot()
accepted := readTree(t, fixture.cache)
fixture.client.failNPC = true
description := filepath.Join(filepath.Dir(fixture.config), "pipelines", "dnd-session.describe-session.yml")
content, err := os.ReadFile(description)
if err != nil {
t.Fatal(err)
}
writeConfigKitFile(t, description, replaceRequiredOnce(t, string(content), " extract:\n module: dnd/npc-registry\n", " extract:\n module: dnd/npc-registry\n cache: refresh\n mode: permissive\n"))
outcome, report, diagnostics := fixture.run(t)
if outcome.Status != 0 || report.FallbackCount != 1 {
t.Fatalf("fallback=%#v report=%#v diagnostics=%q", outcome, report, diagnostics)
}
after, _ := fixture.client.snapshot()
for _, role := range []string{"scenes", "kind", "classification", "locations", "items", "location-occurrences", "duplicate_groups"} {
if after[role] != before[role] {
t.Fatalf("unaffected %s reran: %v -> %v", role, before, after)
}
}
for _, role := range []string{"npcs", "spell_casts", "npc-occurrences", "combat_turns", "events"} {
if after[role] <= before[role] {
t.Fatalf("affected %s reused earlier acceptance: %v -> %v", role, before, after)
}
}
if !sameFiles(accepted, readTree(t, fixture.cache)) {
t.Fatal("failed refresh/fallback replaced accepted cache data")
}
registry := readProductionJSON[dnd.NPCRegistry](t, filepath.Join(report.OutputDirectory, "lanes", "npc-registry.json"))
if registry.NPCs == nil || len(registry.NPCs) != 0 {
t.Fatalf("fallback registry=%#v", registry)
}
fixture.client.failNPC = false
writeConfigKitFile(t, description, string(content))
outcome, report, diagnostics = fixture.run(t)
if outcome.Status != 0 || report.FallbackCount != 0 {
t.Fatalf("restored acceptance=%#v report=%#v diagnostics=%q", outcome, report, diagnostics)
}
last, _ := fixture.client.snapshot()
if !reflect.DeepEqual(last, after) {
t.Fatalf("failed refresh lost earlier reusable results: %v -> %v", after, last)
}
}
var _ promptkit.LLMClient = (*recoveryClient)(nil)