2026-07-18 13:00:04 +00:00
|
|
|
package cli
|
|
|
|
|
|
|
|
|
|
import (
|
|
|
|
|
"bytes"
|
|
|
|
|
"context"
|
2026-07-18 14:21:26 +00:00
|
|
|
"encoding/json"
|
2026-07-18 13:00:04 +00:00
|
|
|
"errors"
|
|
|
|
|
"fmt"
|
|
|
|
|
"os"
|
|
|
|
|
"path/filepath"
|
2026-11-22 05:24:30 +00:00
|
|
|
"reflect"
|
2026-07-18 13:00:04 +00:00
|
|
|
"strings"
|
|
|
|
|
"testing"
|
|
|
|
|
"time"
|
|
|
|
|
|
2026-11-22 05:24:30 +00:00
|
|
|
"go.maximumdirect.net/promptkit"
|
2026-07-18 13:00:04 +00:00
|
|
|
)
|
|
|
|
|
|
2026-11-22 05:24:30 +00:00
|
|
|
type stateRunFixture struct {
|
|
|
|
|
configPath, inputPath, outputRoot, cacheRoot, debugRoot string
|
|
|
|
|
client *productionFakeLLMClient
|
|
|
|
|
options Options
|
2026-07-18 13:00:04 +00:00
|
|
|
}
|
|
|
|
|
|
2026-11-22 05:24:30 +00:00
|
|
|
func newStateRunFixture(t *testing.T) *stateRunFixture {
|
|
|
|
|
t.Helper()
|
|
|
|
|
root := t.TempDir()
|
|
|
|
|
fixture := &stateRunFixture{
|
|
|
|
|
configPath: filepath.Join(root, "config.yml"),
|
|
|
|
|
inputPath: repositoryPath("examples", "seriatim-minimal-transcript.json"),
|
|
|
|
|
outputRoot: filepath.Join(root, "output"), cacheRoot: filepath.Join(root, "cache"), debugRoot: filepath.Join(root, "debug"),
|
|
|
|
|
client: &productionFakeLLMClient{},
|
|
|
|
|
}
|
|
|
|
|
contents := fmt.Sprintf(`version: 6
|
|
|
|
|
output: {directory: %q}
|
|
|
|
|
cache_directory: %q
|
|
|
|
|
debug: {directory: %q}
|
|
|
|
|
pipelines:
|
|
|
|
|
dnd-session:
|
|
|
|
|
input: {module: seriatim}
|
|
|
|
|
chunk: {module: generic}
|
|
|
|
|
artifacts:
|
|
|
|
|
spells:
|
|
|
|
|
extract: {module: dnd/spells}
|
|
|
|
|
`, fixture.outputRoot, fixture.cacheRoot, fixture.debugRoot) + productionProviderConfiguration
|
|
|
|
|
if err := os.WriteFile(fixture.configPath, []byte(contents), 0600); err != nil {
|
2026-07-18 13:00:04 +00:00
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
2026-11-22 05:24:30 +00:00
|
|
|
fixture.options = productionRunOptions(t, fixture.client)
|
|
|
|
|
nextID := 0
|
|
|
|
|
fixture.options.RunIDGenerator = func(time.Time) (string, error) { nextID++; return fmt.Sprintf("run-%d", nextID), nil }
|
|
|
|
|
return fixture
|
2026-07-18 13:00:04 +00:00
|
|
|
}
|
|
|
|
|
|
2026-11-22 05:24:30 +00:00
|
|
|
type stateRunResult struct {
|
|
|
|
|
status int
|
|
|
|
|
stdout, stderr string
|
|
|
|
|
report runResult
|
2026-07-18 13:00:04 +00:00
|
|
|
}
|
|
|
|
|
|
2026-11-22 05:24:30 +00:00
|
|
|
func (fixture *stateRunFixture) run(t *testing.T, flags ...string) stateRunResult {
|
|
|
|
|
t.Helper()
|
|
|
|
|
args := append([]string{"run", "dnd-session", "--config", fixture.configPath, "--input-file", fixture.inputPath, "--format", "json"}, flags...)
|
|
|
|
|
var stdout, stderr bytes.Buffer
|
|
|
|
|
result := stateRunResult{status: RunWithOptions(args, &stdout, &stderr, fixture.options), stdout: stdout.String(), stderr: stderr.String()}
|
|
|
|
|
if result.status == 0 {
|
|
|
|
|
if err := json.Unmarshal(stdout.Bytes(), &result.report); err != nil {
|
|
|
|
|
t.Fatalf("completion=%q: %v", stdout.String(), err)
|
2026-07-18 13:00:04 +00:00
|
|
|
}
|
2026-11-22 05:24:30 +00:00
|
|
|
}
|
|
|
|
|
return result
|
|
|
|
|
}
|
2026-07-18 13:00:04 +00:00
|
|
|
|
2026-11-22 05:24:30 +00:00
|
|
|
func TestRunStateSurfacesFollowDebugAndNativeCacheModes(t *testing.T) {
|
|
|
|
|
for _, debug := range []bool{false, true} {
|
|
|
|
|
for _, mode := range []string{"auto", "bypass", "refresh"} {
|
|
|
|
|
t.Run(fmt.Sprintf("debug=%t/cache=%s", debug, mode), func(t *testing.T) {
|
|
|
|
|
fixture := newStateRunFixture(t)
|
|
|
|
|
result := fixture.run(t, "--cache", mode, fmt.Sprintf("--debug=%t", debug))
|
|
|
|
|
if result.status != 0 || result.report.ArtifactCount != 1 || result.report.FallbackCount != 0 {
|
|
|
|
|
t.Fatalf("run=%#v", result)
|
2026-07-18 13:00:04 +00:00
|
|
|
}
|
2026-11-22 05:24:30 +00:00
|
|
|
assertFile(t, filepath.Join(result.report.OutputDirectory, "lanes", "spells.json"))
|
|
|
|
|
if !strings.HasPrefix(result.report.OutputDirectory, fixture.outputRoot+string(os.PathSeparator)) {
|
|
|
|
|
t.Fatalf("output escaped selected root: %#v", result.report)
|
2026-07-18 13:00:04 +00:00
|
|
|
}
|
2026-11-22 05:24:30 +00:00
|
|
|
if mode == "bypass" {
|
|
|
|
|
assertAbsent(t, fixture.cacheRoot)
|
|
|
|
|
} else {
|
|
|
|
|
files := readTree(t, fixture.cacheRoot)
|
|
|
|
|
if len(files) != 1 {
|
|
|
|
|
t.Fatalf("accepted cache files=%#v", files)
|
|
|
|
|
}
|
|
|
|
|
for name, content := range files {
|
|
|
|
|
if !strings.HasSuffix(name, ".json") || !json.Valid(content) {
|
|
|
|
|
t.Fatalf("invalid accepted cache entry %s", name)
|
|
|
|
|
}
|
|
|
|
|
}
|
2026-07-18 13:00:04 +00:00
|
|
|
}
|
2026-11-22 05:24:30 +00:00
|
|
|
if debug {
|
|
|
|
|
if result.report.DebugDirectory != filepath.Join(fixture.debugRoot, filepath.Base(result.report.OutputDirectory)) {
|
|
|
|
|
t.Fatalf("run identity differs between roots: %#v", result.report)
|
|
|
|
|
}
|
|
|
|
|
assertFile(t, filepath.Join(result.report.DebugDirectory, "outcome.json"))
|
|
|
|
|
} else {
|
|
|
|
|
assertAbsent(t, fixture.debugRoot)
|
|
|
|
|
if result.report.DebugDirectory != "" {
|
|
|
|
|
t.Fatalf("disabled debug path reported: %#v", result.report)
|
|
|
|
|
}
|
2026-07-18 13:00:04 +00:00
|
|
|
}
|
2026-11-22 05:24:30 +00:00
|
|
|
})
|
2026-07-18 13:00:04 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-22 05:24:30 +00:00
|
|
|
func TestRunKeepsStateRootsIndependentWhenReusingAcceptedArtifacts(t *testing.T) {
|
|
|
|
|
fixture := newStateRunFixture(t)
|
|
|
|
|
first := fixture.run(t, "--debug", "--cache", "auto")
|
|
|
|
|
if first.status != 0 {
|
|
|
|
|
t.Fatalf("first=%#v", first)
|
|
|
|
|
}
|
|
|
|
|
entries := readTree(t, fixture.cacheRoot)
|
|
|
|
|
calls := len(fixture.client.requestsSnapshot())
|
|
|
|
|
second := fixture.run(t, "--cache", "auto")
|
|
|
|
|
if second.status != 0 || len(fixture.client.requestsSnapshot()) != calls {
|
|
|
|
|
t.Fatalf("reused=%#v provider calls=%d/%d", second, len(fixture.client.requestsSnapshot()), calls)
|
|
|
|
|
}
|
|
|
|
|
if !sameFiles(entries, readTree(t, fixture.cacheRoot)) {
|
|
|
|
|
t.Fatal("accepted hit rewrote cached artifact")
|
|
|
|
|
}
|
|
|
|
|
if first.report.OutputDirectory == second.report.OutputDirectory {
|
|
|
|
|
t.Fatal("rerun did not get its own output directory")
|
|
|
|
|
}
|
|
|
|
|
assertFile(t, filepath.Join(first.report.OutputDirectory, "lanes", "spells.json"))
|
|
|
|
|
assertFile(t, filepath.Join(second.report.OutputDirectory, "lanes", "spells.json"))
|
|
|
|
|
assertFile(t, filepath.Join(first.report.DebugDirectory, "outcome.json"))
|
|
|
|
|
target, err := os.Readlink(filepath.Join(fixture.debugRoot, "latest"))
|
|
|
|
|
if err != nil || target != filepath.Base(first.report.DebugDirectory) {
|
|
|
|
|
t.Fatalf("disabled capture changed latest: %q %v", target, err)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestRunRetainsCaptureAcrossResolutionProviderAndPublicationFailures(t *testing.T) {
|
|
|
|
|
for _, failure := range []string{"resolution", "provider", "publication"} {
|
|
|
|
|
t.Run(failure, func(t *testing.T) {
|
|
|
|
|
fixture := newStateRunFixture(t)
|
|
|
|
|
flags := []string{"--debug", "--cache", "bypass", "--llm-transient-retries=0"}
|
|
|
|
|
fragment := ""
|
|
|
|
|
switch failure {
|
|
|
|
|
case "resolution":
|
|
|
|
|
flags = append(flags, "--lane", "missing")
|
|
|
|
|
fragment = "missing"
|
|
|
|
|
case "provider":
|
|
|
|
|
fixture.options.LLMClient = stateFailureClient{cause: errors.New("native provider failure")}
|
|
|
|
|
fragment = "native provider failure"
|
|
|
|
|
case "publication":
|
|
|
|
|
if err := os.WriteFile(fixture.outputRoot, []byte("not a directory"), 0600); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
fragment = fixture.outputRoot
|
2026-07-18 14:13:27 +00:00
|
|
|
}
|
2026-11-22 05:24:30 +00:00
|
|
|
result := fixture.run(t, flags...)
|
|
|
|
|
if result.status == 0 || !strings.Contains(result.stderr, fragment) || result.stdout != "" {
|
|
|
|
|
t.Fatalf("failed run=%#v", result)
|
2026-07-18 14:13:27 +00:00
|
|
|
}
|
2026-11-22 05:24:30 +00:00
|
|
|
bundle := onlyChildDir(t, fixture.debugRoot)
|
|
|
|
|
if !strings.Contains(result.stderr, bundle) || !strings.Contains(readAllFiles(t, bundle), fragment) {
|
|
|
|
|
t.Fatalf("failure evidence or path missing: %#v", result)
|
|
|
|
|
}
|
|
|
|
|
assertFile(t, filepath.Join(bundle, "outcome.json"))
|
|
|
|
|
if failure != "publication" {
|
|
|
|
|
assertAbsent(t, fixture.outputRoot)
|
|
|
|
|
} else {
|
|
|
|
|
data, err := os.ReadFile(fixture.outputRoot)
|
|
|
|
|
if err != nil || string(data) != "not a directory" {
|
|
|
|
|
t.Fatalf("blocked output parent changed: %s %v", data, err)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
})
|
2026-07-18 14:13:27 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-22 05:24:30 +00:00
|
|
|
func TestRunCaptureContainsSelectedApplicationDataNotUnrelatedHostData(t *testing.T) {
|
|
|
|
|
fixture := newStateRunFixture(t)
|
|
|
|
|
t.Setenv("NOTARIUS_TEST_UNRELATED_ENV", "HOST_ONLY_SENTINEL")
|
|
|
|
|
if err := os.WriteFile(filepath.Join(filepath.Dir(fixture.configPath), "unrelated.txt"), []byte("HOST_ONLY_FILE_SENTINEL"), 0600); err != nil {
|
2026-07-18 14:21:26 +00:00
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
2026-11-22 05:24:30 +00:00
|
|
|
result := fixture.run(t, "--debug", "--cache", "bypass")
|
|
|
|
|
if result.status != 0 {
|
|
|
|
|
t.Fatalf("run=%#v", result)
|
2026-07-18 14:21:26 +00:00
|
|
|
}
|
2026-11-22 05:24:30 +00:00
|
|
|
evidence := readAllFiles(t, result.report.DebugDirectory)
|
|
|
|
|
if !strings.Contains(evidence, "Aria raises her holy symbol and casts Cure Wounds.") {
|
|
|
|
|
t.Fatal("capture omitted selected source")
|
2026-07-18 14:21:26 +00:00
|
|
|
}
|
2026-11-22 05:24:30 +00:00
|
|
|
for _, unrelated := range []string{"HOST_ONLY_SENTINEL", "HOST_ONLY_FILE_SENTINEL"} {
|
|
|
|
|
if strings.Contains(evidence, unrelated) {
|
|
|
|
|
t.Fatalf("capture included unrelated host data: %s", unrelated)
|
|
|
|
|
}
|
2026-07-18 14:31:46 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-22 05:24:30 +00:00
|
|
|
func TestRepeatedRunIdentityCannotOverwriteFirstOutput(t *testing.T) {
|
|
|
|
|
fixture := newStateRunFixture(t)
|
|
|
|
|
fixture.options.RunIDGenerator = func(time.Time) (string, error) { return "fixed-run", nil }
|
|
|
|
|
first := fixture.run(t, "--cache", "bypass")
|
|
|
|
|
if first.status != 0 {
|
|
|
|
|
t.Fatalf("first=%#v", first)
|
2026-08-03 19:09:49 +00:00
|
|
|
}
|
2026-11-22 05:24:30 +00:00
|
|
|
before := readTree(t, first.report.OutputDirectory)
|
|
|
|
|
second := fixture.run(t, "--cache", "bypass")
|
|
|
|
|
if second.status != 1 || !strings.Contains(second.stderr, "already exists") {
|
|
|
|
|
t.Fatalf("second=%#v", second)
|
2026-08-03 19:09:49 +00:00
|
|
|
}
|
2026-11-22 05:24:30 +00:00
|
|
|
if !sameFiles(before, readTree(t, first.report.OutputDirectory)) {
|
|
|
|
|
t.Fatal("output collision changed first run")
|
2026-08-03 19:09:49 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-22 05:24:30 +00:00
|
|
|
func TestRunRefusesExistingOutputAndDebugDirectoriesWithoutChangingThem(t *testing.T) {
|
|
|
|
|
for _, debug := range []bool{false, true} {
|
|
|
|
|
t.Run(fmt.Sprintf("debug=%t", debug), func(t *testing.T) {
|
|
|
|
|
fixture := newStateRunFixture(t)
|
|
|
|
|
fixture.options.RunIDGenerator = func(time.Time) (string, error) { return "fixed-run", nil }
|
|
|
|
|
selected := filepath.Join(fixture.outputRoot, "fixed-run")
|
|
|
|
|
if debug {
|
|
|
|
|
selected = filepath.Join(fixture.debugRoot, "fixed-run")
|
|
|
|
|
}
|
|
|
|
|
if err := os.MkdirAll(filepath.Join(selected, "nested"), 0700); err != nil {
|
2026-07-18 14:31:46 +00:00
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
2026-11-22 05:24:30 +00:00
|
|
|
if err := os.WriteFile(filepath.Join(selected, "nested", "sentinel"), []byte("preserve me"), 0600); err != nil {
|
|
|
|
|
t.Fatal(err)
|
2026-07-18 14:31:46 +00:00
|
|
|
}
|
2026-11-22 05:24:30 +00:00
|
|
|
before := readTree(t, selected)
|
|
|
|
|
result := fixture.run(t, "--cache", "bypass", fmt.Sprintf("--debug=%t", debug))
|
|
|
|
|
if result.status != 1 || !strings.Contains(result.stderr, selected) {
|
|
|
|
|
t.Fatalf("collision=%#v", result)
|
2026-07-18 14:31:46 +00:00
|
|
|
}
|
2026-11-22 05:24:30 +00:00
|
|
|
if !reflect.DeepEqual(before, readTree(t, selected)) {
|
|
|
|
|
t.Fatal("existing directory changed")
|
2026-07-18 14:31:46 +00:00
|
|
|
}
|
2026-11-22 05:24:30 +00:00
|
|
|
if debug {
|
|
|
|
|
assertAbsent(t, fixture.outputRoot)
|
|
|
|
|
if len(fixture.client.requestsSnapshot()) != 0 {
|
|
|
|
|
t.Fatal("debug allocation failure reached provider")
|
|
|
|
|
}
|
2026-07-18 14:31:46 +00:00
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-22 05:24:30 +00:00
|
|
|
func TestRunIdentityFailuresPrecedePathUse(t *testing.T) {
|
|
|
|
|
for _, test := range []struct {
|
|
|
|
|
name, id string
|
|
|
|
|
err error
|
2026-07-18 14:31:46 +00:00
|
|
|
}{
|
2026-11-22 05:24:30 +00:00
|
|
|
{name: "identity generation", err: errors.New("random source unavailable")},
|
|
|
|
|
{name: "unsafe identity", id: "../outside"},
|
2026-07-18 14:31:46 +00:00
|
|
|
} {
|
2026-11-22 05:24:30 +00:00
|
|
|
t.Run(test.name, func(t *testing.T) {
|
|
|
|
|
fixture := newStateRunFixture(t)
|
|
|
|
|
fixture.options.RunIDGenerator = func(time.Time) (string, error) { return test.id, test.err }
|
|
|
|
|
result := fixture.run(t, "--debug")
|
|
|
|
|
if result.status != 1 || result.stderr == "" || len(fixture.client.requestsSnapshot()) != 0 {
|
|
|
|
|
t.Fatalf("identity failure=%#v", result)
|
2026-07-18 14:31:46 +00:00
|
|
|
}
|
2026-11-22 05:24:30 +00:00
|
|
|
assertAbsent(t, fixture.outputRoot)
|
|
|
|
|
assertAbsent(t, fixture.debugRoot)
|
|
|
|
|
assertAbsent(t, fixture.cacheRoot)
|
2026-07-18 14:31:46 +00:00
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-22 05:24:30 +00:00
|
|
|
type stateFailureClient struct{ cause error }
|
2026-07-18 14:31:46 +00:00
|
|
|
|
2026-11-22 05:24:30 +00:00
|
|
|
func (client stateFailureClient) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
|
|
|
return nil, client.cause
|
2026-07-18 13:00:04 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func onlyChildDir(t *testing.T, root string) string {
|
|
|
|
|
t.Helper()
|
|
|
|
|
entries, err := os.ReadDir(root)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
var dirs []string
|
|
|
|
|
for _, entry := range entries {
|
|
|
|
|
if entry.IsDir() {
|
|
|
|
|
dirs = append(dirs, filepath.Join(root, entry.Name()))
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if len(dirs) != 1 {
|
|
|
|
|
t.Fatalf("directories in %q = %v, want one", root, dirs)
|
|
|
|
|
}
|
|
|
|
|
return dirs[0]
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func assertFile(t *testing.T, path string) {
|
|
|
|
|
t.Helper()
|
|
|
|
|
if info, err := os.Stat(path); err != nil || info.IsDir() {
|
|
|
|
|
t.Fatalf("file %q: %v", path, err)
|
|
|
|
|
}
|
|
|
|
|
}
|
2026-11-22 05:24:30 +00:00
|
|
|
|
2026-07-18 13:00:04 +00:00
|
|
|
func assertAbsent(t *testing.T, path string) {
|
|
|
|
|
t.Helper()
|
|
|
|
|
if _, err := os.Stat(path); !os.IsNotExist(err) {
|
|
|
|
|
t.Fatalf("%q exists or stat failed: %v", path, err)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func readAllFiles(t *testing.T, root string) string {
|
|
|
|
|
t.Helper()
|
|
|
|
|
var content strings.Builder
|
|
|
|
|
if err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error {
|
|
|
|
|
if err != nil {
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
if entry.IsDir() {
|
|
|
|
|
return nil
|
|
|
|
|
}
|
|
|
|
|
data, err := os.ReadFile(path)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
content.Write(data)
|
|
|
|
|
return nil
|
|
|
|
|
}); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
return content.String()
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func readTree(t *testing.T, root string) map[string][]byte {
|
|
|
|
|
t.Helper()
|
|
|
|
|
files := map[string][]byte{}
|
|
|
|
|
if err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error {
|
|
|
|
|
if err != nil {
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
if entry.IsDir() {
|
|
|
|
|
return nil
|
|
|
|
|
}
|
|
|
|
|
data, err := os.ReadFile(path)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
relative, err := filepath.Rel(root, path)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
files[relative] = data
|
|
|
|
|
return nil
|
|
|
|
|
}); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
return files
|
|
|
|
|
}
|
2026-11-22 05:24:30 +00:00
|
|
|
|
2026-07-18 13:00:04 +00:00
|
|
|
func sameFiles(left, right map[string][]byte) bool {
|
|
|
|
|
if len(left) != len(right) {
|
|
|
|
|
return false
|
|
|
|
|
}
|
|
|
|
|
for path, data := range left {
|
|
|
|
|
if !bytes.Equal(data, right[path]) {
|
|
|
|
|
return false
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return true
|
|
|
|
|
}
|