323 lines
13 KiB
Go
323 lines
13 KiB
Go
package cli
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"reflect"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
|
"go.maximumdirect.net/commandkit"
|
|
"go.maximumdirect.net/promptkit"
|
|
)
|
|
|
|
// Use the maintained ordered domain composition and existing provider fake.
|
|
// Only external failures and one rejected scene candidate are controlled here.
|
|
type recoveryClient struct {
|
|
base enemyEventLLMClient
|
|
mu sync.Mutex
|
|
requests []promptkit.GenerateRequest
|
|
failEvents, failNPC, correctScene bool
|
|
failure error
|
|
}
|
|
|
|
func (client *recoveryClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
role := enemyEventRequestRole(request)
|
|
if productionSchemaHasProperty(request, "classification") {
|
|
role = "classification"
|
|
}
|
|
client.mu.Lock()
|
|
client.requests = append(client.requests, request)
|
|
failEvents, failNPC := client.failEvents, client.failNPC
|
|
transcript, _ := productionJSONInput(request, "units")
|
|
combat := strings.Contains(string(transcript), "Roll initiative")
|
|
incorrectScene := role == "kind" && combat && client.correctScene
|
|
if incorrectScene {
|
|
client.correctScene = false
|
|
}
|
|
client.mu.Unlock()
|
|
if role == "events" && failEvents || role == "npcs" && combat && failNPC {
|
|
return nil, client.failure
|
|
}
|
|
if incorrectScene {
|
|
return &promptkit.GenerateResponse{Content: `{"kind":"narrative","title":"Arrival","summary":"An initially missed combat scene."}`}, nil
|
|
}
|
|
if role == "classification" {
|
|
classification := "non_combat"
|
|
if combat {
|
|
classification = "combat"
|
|
}
|
|
return &promptkit.GenerateResponse{Content: fmt.Sprintf(`{"classification":%q,"explanation":"Classification follows the supplied scene transcript."}`, classification)}, nil
|
|
}
|
|
// Empty generated registries remain valid current-run context. Do not invent
|
|
// identities or events merely to fill the dependent lanes in the fallback run.
|
|
if role == "npc-occurrences" || role == "combat_turns" || role == "events" {
|
|
if content, err := productionJSONInput(request, "npcs"); err == nil {
|
|
var registry struct {
|
|
NPCs []json.RawMessage `json:"npcs"`
|
|
}
|
|
if err := json.Unmarshal(content, ®istry); err != nil {
|
|
return nil, err
|
|
}
|
|
if len(registry.NPCs) == 0 {
|
|
property := "occurrences"
|
|
if role == "combat_turns" {
|
|
property = "combat_turns"
|
|
}
|
|
if role == "events" {
|
|
property = "events"
|
|
}
|
|
return &promptkit.GenerateResponse{Content: fmt.Sprintf(`{%q:[]}`, property)}, nil
|
|
}
|
|
}
|
|
}
|
|
return client.base.Generate(ctx, request)
|
|
}
|
|
|
|
func (client *recoveryClient) snapshot() (map[string]int, []promptkit.GenerateRequest) {
|
|
client.mu.Lock()
|
|
defer client.mu.Unlock()
|
|
counts := map[string]int{}
|
|
for _, request := range client.requests {
|
|
role := enemyEventRequestRole(request)
|
|
if productionSchemaHasProperty(request, "classification") {
|
|
role = "classification"
|
|
}
|
|
counts[role]++
|
|
}
|
|
return counts, append([]promptkit.GenerateRequest(nil), client.requests...)
|
|
}
|
|
|
|
type recoveryFixture struct {
|
|
config, cache, output, debug string
|
|
client *recoveryClient
|
|
options Options
|
|
}
|
|
|
|
func newRecoveryFixture(t *testing.T) *recoveryFixture {
|
|
t.Helper()
|
|
root := t.TempDir()
|
|
for _, name := range []string{"prompts.yml", "transcript.json", "party.txt", "glossary.txt", "spell-catalog.json", "pipelines/dnd-session.yml", "pipelines/dnd-session.describe-session.yml"} {
|
|
content, err := os.ReadFile(repositoryPath("examples", "dnd-complete", name))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
writeConfigKitFile(t, filepath.Join(root, name), string(content))
|
|
}
|
|
description := filepath.Join(root, "pipelines", "dnd-session.describe-session.yml")
|
|
content, err := os.ReadFile(description)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
configured := replaceRequiredOnce(t, string(content), " extract:\n module: dnd/scene-descriptions\n", ` extract:
|
|
module: dnd/scene-descriptions
|
|
validators:
|
|
- module: generic/valid_json
|
|
- module: extract/dnd/scene-descriptions/shape
|
|
- module: extract/dnd/scene-descriptions/source_refs
|
|
- module: generic/valid_json_schema
|
|
- module: extract/dnd/scene-descriptions/source_relatedness
|
|
- module: extract/dnd/scene-descriptions/combat_semantics
|
|
`)
|
|
writeConfigKitFile(t, description, configured)
|
|
fixture := &recoveryFixture{config: filepath.Join(root, "config.yml"), cache: filepath.Join(root, "cache"), output: filepath.Join(root, "output"), debug: filepath.Join(root, "debug"), client: &recoveryClient{failure: errors.New("controlled downstream provider failure")}}
|
|
writeConfigKitFile(t, fixture.config, fmt.Sprintf(`version: 6
|
|
config:
|
|
imports:
|
|
- path: pipelines/dnd-session.yml
|
|
required: true
|
|
output: {directory: %q}
|
|
cache_directory: %q
|
|
debug: {directory: %q}
|
|
concurrency: {workers: 1}
|
|
promptkit:
|
|
backends:
|
|
- id: fixture
|
|
endpoint: https://example.invalid/v1
|
|
concurrency_limit: 1
|
|
collections:
|
|
notarius:
|
|
profiles:
|
|
- id: dnd-extraction
|
|
backend: fixture
|
|
model: recovery-model
|
|
api_key_required: false
|
|
timeout_seconds: 2
|
|
- id: dnd-extraction-example
|
|
extends: {id: dnd-extraction}
|
|
`, fixture.output, fixture.cache, fixture.debug))
|
|
fixture.options = productionRunOptions(t, fixture.client)
|
|
run := 0
|
|
fixture.options.RunIDGenerator = func(time.Time) (string, error) { run++; return fmt.Sprintf("recovery-%d", run), nil }
|
|
return fixture
|
|
}
|
|
|
|
func (fixture *recoveryFixture) run(t *testing.T) (commandkit.Outcome, runResult, string) {
|
|
t.Helper()
|
|
args := []string{"run", "dnd-session", "--config", fixture.config, "--input-file", filepath.Join(filepath.Dir(fixture.config), "transcript.json"), "--debug", "--progress=false", "--format=json", "--llm-session-id=recovery-session", "--llm-transient-retries=0",
|
|
"--set", "/pipelines/dnd-session/max_additional_turns=1",
|
|
}
|
|
app, err := NewApplication("development", fixture.options)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
var stdout, stderr bytes.Buffer
|
|
outcome := app.Execute(ctx, args, commandkit.Streams{Out: &stdout, Err: &stderr})
|
|
var report runResult
|
|
if outcome.Status == 0 {
|
|
if err := json.Unmarshal(stdout.Bytes(), &report); err != nil {
|
|
t.Fatalf("completion=%q: %v", stdout.String(), err)
|
|
}
|
|
} else if stdout.Len() != 0 {
|
|
t.Fatalf("failed operation emitted success output: %q", stdout.String())
|
|
}
|
|
return outcome, report, stderr.String()
|
|
}
|
|
|
|
func TestAssembledRecoveryReusesCorrectedPredecessorsAndChunkPlan(t *testing.T) {
|
|
fixture := newRecoveryFixture(t)
|
|
fixture.client.failEvents, fixture.client.correctScene = true, true
|
|
outcome, _, diagnostics := fixture.run(t)
|
|
if outcome.Status != 1 || !errors.Is(outcome.Cause, fixture.client.failure) {
|
|
t.Fatalf("downstream failure=%#v diagnostics=%q", outcome, diagnostics)
|
|
}
|
|
assertAbsent(t, fixture.output)
|
|
firstDebug := filepath.Join(fixture.debug, "recovery-1")
|
|
assertFile(t, filepath.Join(firstDebug, "outcome.json"))
|
|
counts, requests := fixture.client.snapshot()
|
|
if counts["scenes"] != 1 || counts["kind"] != 3 || counts["classification"] != 3 || counts["duplicate_groups"] == 0 || counts["events"] != 1 {
|
|
t.Fatalf("ordered native roles/correction=%v", counts)
|
|
}
|
|
for _, request := range requests {
|
|
if request.Prompt.SessionID != "recovery-session" || request.Target.Model != "recovery-model" || request.Target.BackendID != "fixture" {
|
|
t.Fatalf("lost shared native selection: %#v", request)
|
|
}
|
|
}
|
|
summary := readProductionJSON[promptkit.ExecutionGroupSummary](t, filepath.Join(firstDebug, "executions.json"))
|
|
if len(summary.Members) != len(requests) || summary.Attempts != uint64(len(requests)) || !summary.TerminalCoverageComplete || summary.Usage.TotalTokens.Known {
|
|
t.Fatalf("failed-run native facts lost or duplicated: %#v", summary)
|
|
}
|
|
// Locate only the initial scene request's native hash in available capture;
|
|
// assert domain bytes at that key, not a persisted execution ledger.
|
|
var correctedKey string
|
|
for name, content := range readTree(t, firstDebug) {
|
|
if !strings.HasSuffix(name, "/prepared.json") {
|
|
continue
|
|
}
|
|
var details promptkit.PreparedRun
|
|
if json.Unmarshal(content, &details) != nil || details.PromptID != "notarius/dnd.scene_descriptions" {
|
|
continue
|
|
}
|
|
for _, message := range details.Messages {
|
|
if strings.Contains(message.Content, "Roll initiative") {
|
|
correctedKey = details.MessageContentHash
|
|
}
|
|
}
|
|
}
|
|
if correctedKey == "" {
|
|
t.Fatal("no prepared initial combat scene evidence")
|
|
}
|
|
corrected := readProductionJSON[dnd.SceneDescriptionList](t, filepath.Join(fixture.cache, correctedKey+".json"))
|
|
if len(corrected.Scenes) != 1 || corrected.Scenes[0].Kind != dnd.SceneKindCombat {
|
|
t.Fatalf("cache did not retain the corrected domain candidate: %#v", corrected)
|
|
}
|
|
fixture.client.failEvents = false
|
|
outcome, report, diagnostics := fixture.run(t)
|
|
if outcome.Status != 0 || report.FallbackCount != 0 || report.ArtifactCount != 12 {
|
|
t.Fatalf("recovery=%#v report=%#v diagnostics=%q", outcome, report, diagnostics)
|
|
}
|
|
after, _ := fixture.client.snapshot()
|
|
for role, count := range counts {
|
|
want := count
|
|
if role == "events" {
|
|
want++
|
|
}
|
|
if after[role] != want {
|
|
t.Fatalf("recovery reran accepted %s: %v -> %v", role, counts, after)
|
|
}
|
|
}
|
|
artifact := readProductionJSON[dnd.EnemyEventList](t, filepath.Join(report.OutputDirectory, "lanes", "enemy-events.json"))
|
|
if len(artifact.Events) != 1 || artifact.Events[0].SourceRefs[0].SourceID != "session-ravenfall" {
|
|
t.Fatalf("recovered domain evidence=%#v", artifact)
|
|
}
|
|
stable := readTree(t, fixture.cache)
|
|
outcome, hitReport, diagnostics := fixture.run(t)
|
|
if outcome.Status != 0 || hitReport.ArtifactCount != 12 {
|
|
t.Fatalf("cached run=%#v diagnostics=%q", outcome, diagnostics)
|
|
}
|
|
last, _ := fixture.client.snapshot()
|
|
if !reflect.DeepEqual(last, after) || !sameFiles(stable, readTree(t, fixture.cache)) {
|
|
t.Fatalf("full hit executed or rewrote acceptance: %v -> %v", after, last)
|
|
}
|
|
for name := range readTree(t, hitReport.DebugDirectory) {
|
|
if strings.Contains(name, "/native/") && strings.HasSuffix(name, "/result.json") {
|
|
t.Fatalf("full hit fabricated native execution: %s", name)
|
|
}
|
|
}
|
|
hitSummary := readProductionJSON[promptkit.ExecutionGroupSummary](t, filepath.Join(hitReport.DebugDirectory, "executions.json"))
|
|
if len(hitSummary.Members) != 0 || !hitSummary.TerminalCoverageComplete {
|
|
t.Fatalf("full hit fabricated historical execution totals: %#v", hitSummary)
|
|
}
|
|
}
|
|
|
|
func TestAssembledFallbackSkipsDependentReuseAndPreservesFailedRefresh(t *testing.T) {
|
|
fixture := newRecoveryFixture(t)
|
|
outcome, _, diagnostics := fixture.run(t)
|
|
if outcome.Status != 0 {
|
|
t.Fatalf("initial run=%#v diagnostics=%q", outcome, diagnostics)
|
|
}
|
|
before, _ := fixture.client.snapshot()
|
|
accepted := readTree(t, fixture.cache)
|
|
fixture.client.failNPC = true
|
|
description := filepath.Join(filepath.Dir(fixture.config), "pipelines", "dnd-session.describe-session.yml")
|
|
content, err := os.ReadFile(description)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
writeConfigKitFile(t, description, replaceRequiredOnce(t, string(content), " extract:\n module: dnd/npc-registry\n", " extract:\n module: dnd/npc-registry\n cache: refresh\n mode: permissive\n"))
|
|
outcome, report, diagnostics := fixture.run(t)
|
|
if outcome.Status != 0 || report.FallbackCount != 1 {
|
|
t.Fatalf("fallback=%#v report=%#v diagnostics=%q", outcome, report, diagnostics)
|
|
}
|
|
after, _ := fixture.client.snapshot()
|
|
for _, role := range []string{"scenes", "kind", "classification", "locations", "items", "location-occurrences", "duplicate_groups"} {
|
|
if after[role] != before[role] {
|
|
t.Fatalf("unaffected %s reran: %v -> %v", role, before, after)
|
|
}
|
|
}
|
|
for _, role := range []string{"npcs", "spell_casts", "npc-occurrences", "combat_turns", "events"} {
|
|
if after[role] <= before[role] {
|
|
t.Fatalf("affected %s reused earlier acceptance: %v -> %v", role, before, after)
|
|
}
|
|
}
|
|
if !sameFiles(accepted, readTree(t, fixture.cache)) {
|
|
t.Fatal("failed refresh/fallback replaced accepted cache data")
|
|
}
|
|
registry := readProductionJSON[dnd.NPCRegistry](t, filepath.Join(report.OutputDirectory, "lanes", "npc-registry.json"))
|
|
if registry.NPCs == nil || len(registry.NPCs) != 0 {
|
|
t.Fatalf("fallback registry=%#v", registry)
|
|
}
|
|
fixture.client.failNPC = false
|
|
writeConfigKitFile(t, description, string(content))
|
|
outcome, report, diagnostics = fixture.run(t)
|
|
if outcome.Status != 0 || report.FallbackCount != 0 {
|
|
t.Fatalf("restored acceptance=%#v report=%#v diagnostics=%q", outcome, report, diagnostics)
|
|
}
|
|
last, _ := fixture.client.snapshot()
|
|
if !reflect.DeepEqual(last, after) {
|
|
t.Fatalf("failed refresh lost earlier reusable results: %v -> %v", after, last)
|
|
}
|
|
}
|
|
|
|
var _ promptkit.LLMClient = (*recoveryClient)(nil)
|