package cli import ( "bytes" "context" "encoding/json" "errors" "fmt" "os" "path/filepath" "reflect" "strings" "testing" "time" "go.maximumdirect.net/promptkit" ) type stateRunFixture struct { configPath, inputPath, outputRoot, cacheRoot, debugRoot string client *productionFakeLLMClient options Options } func newStateRunFixture(t *testing.T) *stateRunFixture { t.Helper() root := t.TempDir() fixture := &stateRunFixture{ configPath: filepath.Join(root, "config.yml"), inputPath: repositoryPath("examples", "seriatim-minimal-transcript.json"), outputRoot: filepath.Join(root, "output"), cacheRoot: filepath.Join(root, "cache"), debugRoot: filepath.Join(root, "debug"), client: &productionFakeLLMClient{}, } contents := fmt.Sprintf(`version: 6 output: {directory: %q} cache_directory: %q debug: {directory: %q} pipelines: dnd-session: input: {module: seriatim} chunk: {module: generic} artifacts: spells: extract: {module: dnd/spells} `, fixture.outputRoot, fixture.cacheRoot, fixture.debugRoot) + productionProviderConfiguration if err := os.WriteFile(fixture.configPath, []byte(contents), 0600); err != nil { t.Fatal(err) } fixture.options = productionRunOptions(t, fixture.client) nextID := 0 fixture.options.RunIDGenerator = func(time.Time) (string, error) { nextID++; return fmt.Sprintf("run-%d", nextID), nil } return fixture } type stateRunResult struct { status int stdout, stderr string report runResult } func (fixture *stateRunFixture) run(t *testing.T, flags ...string) stateRunResult { t.Helper() args := append([]string{"run", "dnd-session", "--config", fixture.configPath, "--input-file", fixture.inputPath, "--format", "json"}, flags...) var stdout, stderr bytes.Buffer result := stateRunResult{status: RunWithOptions(args, &stdout, &stderr, fixture.options), stdout: stdout.String(), stderr: stderr.String()} if result.status == 0 { if err := json.Unmarshal(stdout.Bytes(), &result.report); err != nil { t.Fatalf("completion=%q: %v", stdout.String(), err) } } return result } func TestRunStateSurfacesFollowDebugAndNativeCacheModes(t *testing.T) { for _, debug := range []bool{false, true} { for _, mode := range []string{"auto", "bypass", "refresh"} { t.Run(fmt.Sprintf("debug=%t/cache=%s", debug, mode), func(t *testing.T) { fixture := newStateRunFixture(t) result := fixture.run(t, "--cache", mode, fmt.Sprintf("--debug=%t", debug)) if result.status != 0 || result.report.ArtifactCount != 1 || result.report.FallbackCount != 0 { t.Fatalf("run=%#v", result) } assertFile(t, filepath.Join(result.report.OutputDirectory, "lanes", "spells.json")) if !strings.HasPrefix(result.report.OutputDirectory, fixture.outputRoot+string(os.PathSeparator)) { t.Fatalf("output escaped selected root: %#v", result.report) } if mode == "bypass" { assertAbsent(t, fixture.cacheRoot) } else { files := readTree(t, fixture.cacheRoot) if len(files) != 1 { t.Fatalf("accepted cache files=%#v", files) } for name, content := range files { if !strings.HasSuffix(name, ".json") || !json.Valid(content) { t.Fatalf("invalid accepted cache entry %s", name) } } } if debug { if result.report.DebugDirectory != filepath.Join(fixture.debugRoot, filepath.Base(result.report.OutputDirectory)) { t.Fatalf("run identity differs between roots: %#v", result.report) } assertFile(t, filepath.Join(result.report.DebugDirectory, "outcome.json")) } else { assertAbsent(t, fixture.debugRoot) if result.report.DebugDirectory != "" { t.Fatalf("disabled debug path reported: %#v", result.report) } } }) } } } func TestRunKeepsStateRootsIndependentWhenReusingAcceptedArtifacts(t *testing.T) { fixture := newStateRunFixture(t) first := fixture.run(t, "--debug", "--cache", "auto") if first.status != 0 { t.Fatalf("first=%#v", first) } entries := readTree(t, fixture.cacheRoot) calls := len(fixture.client.requestsSnapshot()) second := fixture.run(t, "--cache", "auto") if second.status != 0 || len(fixture.client.requestsSnapshot()) != calls { t.Fatalf("reused=%#v provider calls=%d/%d", second, len(fixture.client.requestsSnapshot()), calls) } if !sameFiles(entries, readTree(t, fixture.cacheRoot)) { t.Fatal("accepted hit rewrote cached artifact") } if first.report.OutputDirectory == second.report.OutputDirectory { t.Fatal("rerun did not get its own output directory") } assertFile(t, filepath.Join(first.report.OutputDirectory, "lanes", "spells.json")) assertFile(t, filepath.Join(second.report.OutputDirectory, "lanes", "spells.json")) assertFile(t, filepath.Join(first.report.DebugDirectory, "outcome.json")) target, err := os.Readlink(filepath.Join(fixture.debugRoot, "latest")) if err != nil || target != filepath.Base(first.report.DebugDirectory) { t.Fatalf("disabled capture changed latest: %q %v", target, err) } } func TestRunRetainsCaptureAcrossResolutionProviderAndPublicationFailures(t *testing.T) { for _, failure := range []string{"resolution", "provider", "publication"} { t.Run(failure, func(t *testing.T) { fixture := newStateRunFixture(t) flags := []string{"--debug", "--cache", "bypass", "--llm-transient-retries=0"} fragment := "" switch failure { case "resolution": flags = append(flags, "--lane", "missing") fragment = "missing" case "provider": fixture.options.LLMClient = stateFailureClient{cause: errors.New("native provider failure")} fragment = "native provider failure" case "publication": if err := os.WriteFile(fixture.outputRoot, []byte("not a directory"), 0600); err != nil { t.Fatal(err) } fragment = fixture.outputRoot } result := fixture.run(t, flags...) if result.status == 0 || !strings.Contains(result.stderr, fragment) || result.stdout != "" { t.Fatalf("failed run=%#v", result) } bundle := onlyChildDir(t, fixture.debugRoot) if !strings.Contains(result.stderr, bundle) || !strings.Contains(readAllFiles(t, bundle), fragment) { t.Fatalf("failure evidence or path missing: %#v", result) } assertFile(t, filepath.Join(bundle, "outcome.json")) if failure != "publication" { assertAbsent(t, fixture.outputRoot) } else { data, err := os.ReadFile(fixture.outputRoot) if err != nil || string(data) != "not a directory" { t.Fatalf("blocked output parent changed: %s %v", data, err) } } }) } } func TestRunCaptureContainsSelectedApplicationDataNotUnrelatedHostData(t *testing.T) { fixture := newStateRunFixture(t) t.Setenv("NOTARIUS_TEST_UNRELATED_ENV", "HOST_ONLY_SENTINEL") if err := os.WriteFile(filepath.Join(filepath.Dir(fixture.configPath), "unrelated.txt"), []byte("HOST_ONLY_FILE_SENTINEL"), 0600); err != nil { t.Fatal(err) } result := fixture.run(t, "--debug", "--cache", "bypass") if result.status != 0 { t.Fatalf("run=%#v", result) } evidence := readAllFiles(t, result.report.DebugDirectory) if !strings.Contains(evidence, "Aria raises her holy symbol and casts Cure Wounds.") { t.Fatal("capture omitted selected source") } for _, unrelated := range []string{"HOST_ONLY_SENTINEL", "HOST_ONLY_FILE_SENTINEL"} { if strings.Contains(evidence, unrelated) { t.Fatalf("capture included unrelated host data: %s", unrelated) } } } func TestRepeatedRunIdentityCannotOverwriteFirstOutput(t *testing.T) { fixture := newStateRunFixture(t) fixture.options.RunIDGenerator = func(time.Time) (string, error) { return "fixed-run", nil } first := fixture.run(t, "--cache", "bypass") if first.status != 0 { t.Fatalf("first=%#v", first) } before := readTree(t, first.report.OutputDirectory) second := fixture.run(t, "--cache", "bypass") if second.status != 1 || !strings.Contains(second.stderr, "already exists") { t.Fatalf("second=%#v", second) } if !sameFiles(before, readTree(t, first.report.OutputDirectory)) { t.Fatal("output collision changed first run") } } func TestRunRefusesExistingOutputAndDebugDirectoriesWithoutChangingThem(t *testing.T) { for _, debug := range []bool{false, true} { t.Run(fmt.Sprintf("debug=%t", debug), func(t *testing.T) { fixture := newStateRunFixture(t) fixture.options.RunIDGenerator = func(time.Time) (string, error) { return "fixed-run", nil } selected := filepath.Join(fixture.outputRoot, "fixed-run") if debug { selected = filepath.Join(fixture.debugRoot, "fixed-run") } if err := os.MkdirAll(filepath.Join(selected, "nested"), 0700); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(selected, "nested", "sentinel"), []byte("preserve me"), 0600); err != nil { t.Fatal(err) } before := readTree(t, selected) result := fixture.run(t, "--cache", "bypass", fmt.Sprintf("--debug=%t", debug)) if result.status != 1 || !strings.Contains(result.stderr, selected) { t.Fatalf("collision=%#v", result) } if !reflect.DeepEqual(before, readTree(t, selected)) { t.Fatal("existing directory changed") } if debug { assertAbsent(t, fixture.outputRoot) if len(fixture.client.requestsSnapshot()) != 0 { t.Fatal("debug allocation failure reached provider") } } }) } } func TestRunIdentityFailuresPrecedePathUse(t *testing.T) { for _, test := range []struct { name, id string err error }{ {name: "identity generation", err: errors.New("random source unavailable")}, {name: "unsafe identity", id: "../outside"}, } { t.Run(test.name, func(t *testing.T) { fixture := newStateRunFixture(t) fixture.options.RunIDGenerator = func(time.Time) (string, error) { return test.id, test.err } result := fixture.run(t, "--debug") if result.status != 1 || result.stderr == "" || len(fixture.client.requestsSnapshot()) != 0 { t.Fatalf("identity failure=%#v", result) } assertAbsent(t, fixture.outputRoot) assertAbsent(t, fixture.debugRoot) assertAbsent(t, fixture.cacheRoot) }) } } type stateFailureClient struct{ cause error } func (client stateFailureClient) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { return nil, client.cause } func onlyChildDir(t *testing.T, root string) string { t.Helper() entries, err := os.ReadDir(root) if err != nil { t.Fatal(err) } var dirs []string for _, entry := range entries { if entry.IsDir() { dirs = append(dirs, filepath.Join(root, entry.Name())) } } if len(dirs) != 1 { t.Fatalf("directories in %q = %v, want one", root, dirs) } return dirs[0] } func assertFile(t *testing.T, path string) { t.Helper() if info, err := os.Stat(path); err != nil || info.IsDir() { t.Fatalf("file %q: %v", path, err) } } func assertAbsent(t *testing.T, path string) { t.Helper() if _, err := os.Stat(path); !os.IsNotExist(err) { t.Fatalf("%q exists or stat failed: %v", path, err) } } func readAllFiles(t *testing.T, root string) string { t.Helper() var content strings.Builder if err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { if err != nil { return err } if entry.IsDir() { return nil } data, err := os.ReadFile(path) if err != nil { return err } content.Write(data) return nil }); err != nil { t.Fatal(err) } return content.String() } func readTree(t *testing.T, root string) map[string][]byte { t.Helper() files := map[string][]byte{} if err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { if err != nil { return err } if entry.IsDir() { return nil } data, err := os.ReadFile(path) if err != nil { return err } relative, err := filepath.Rel(root, path) if err != nil { return err } files[relative] = data return nil }); err != nil { t.Fatal(err) } return files } func sameFiles(left, right map[string][]byte) bool { if len(left) != len(right) { return false } for path, data := range left { if !bytes.Equal(data, right[path]) { return false } } return true }