Files
notarius/internal/cli/state_hardening_test.go

373 lines
12 KiB
Go

package cli
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"time"
"go.maximumdirect.net/promptkit"
)
type stateRunFixture struct {
configPath, inputPath, outputRoot, cacheRoot, debugRoot string
client *productionFakeLLMClient
options Options
}
func newStateRunFixture(t *testing.T) *stateRunFixture {
t.Helper()
root := t.TempDir()
fixture := &stateRunFixture{
configPath: filepath.Join(root, "config.yml"),
inputPath: repositoryPath("examples", "dnd-minimal", "transcript.json"),
outputRoot: filepath.Join(root, "output"), cacheRoot: filepath.Join(root, "cache"), debugRoot: filepath.Join(root, "debug"),
client: &productionFakeLLMClient{},
}
contents := fmt.Sprintf(`version: 6
output: {directory: %q}
cache_directory: %q
debug: {directory: %q}
pipelines:
dnd-session:
input: {module: seriatim}
chunk: {module: generic}
artifacts:
spells:
extract: {module: dnd/spells}
`, fixture.outputRoot, fixture.cacheRoot, fixture.debugRoot) + productionProviderConfiguration
if err := os.WriteFile(fixture.configPath, []byte(contents), 0600); err != nil {
t.Fatal(err)
}
fixture.options = productionRunOptions(t, fixture.client)
nextID := 0
fixture.options.RunIDGenerator = func(time.Time) (string, error) { nextID++; return fmt.Sprintf("run-%d", nextID), nil }
return fixture
}
type stateRunResult struct {
status int
stdout, stderr string
report runResult
}
func (fixture *stateRunFixture) run(t *testing.T, flags ...string) stateRunResult {
t.Helper()
args := append([]string{"run", "dnd-session", "--config", fixture.configPath, "--input-file", fixture.inputPath, "--format", "json"}, flags...)
var stdout, stderr bytes.Buffer
result := stateRunResult{status: RunWithOptions(args, &stdout, &stderr, fixture.options), stdout: stdout.String(), stderr: stderr.String()}
if result.status == 0 {
if err := json.Unmarshal(stdout.Bytes(), &result.report); err != nil {
t.Fatalf("completion=%q: %v", stdout.String(), err)
}
}
return result
}
func TestRunStateSurfacesFollowDebugAndNativeCacheModes(t *testing.T) {
for _, debug := range []bool{false, true} {
for _, mode := range []string{"auto", "bypass", "refresh"} {
t.Run(fmt.Sprintf("debug=%t/cache=%s", debug, mode), func(t *testing.T) {
fixture := newStateRunFixture(t)
result := fixture.run(t, "--cache", mode, fmt.Sprintf("--debug=%t", debug))
if result.status != 0 || result.report.ArtifactCount != 1 || result.report.FallbackCount != 0 {
t.Fatalf("run=%#v", result)
}
assertFile(t, filepath.Join(result.report.OutputDirectory, "lanes", "spells.json"))
if !strings.HasPrefix(result.report.OutputDirectory, fixture.outputRoot+string(os.PathSeparator)) {
t.Fatalf("output escaped selected root: %#v", result.report)
}
if mode == "bypass" {
assertAbsent(t, fixture.cacheRoot)
} else {
files := readTree(t, fixture.cacheRoot)
if len(files) != 1 {
t.Fatalf("accepted cache files=%#v", files)
}
for name, content := range files {
if !strings.HasSuffix(name, ".json") || !json.Valid(content) {
t.Fatalf("invalid accepted cache entry %s", name)
}
}
}
if debug {
if result.report.DebugDirectory != filepath.Join(fixture.debugRoot, filepath.Base(result.report.OutputDirectory)) {
t.Fatalf("run identity differs between roots: %#v", result.report)
}
assertFile(t, filepath.Join(result.report.DebugDirectory, "outcome.json"))
} else {
assertAbsent(t, fixture.debugRoot)
if result.report.DebugDirectory != "" {
t.Fatalf("disabled debug path reported: %#v", result.report)
}
}
})
}
}
}
func TestRunKeepsStateRootsIndependentWhenReusingAcceptedArtifacts(t *testing.T) {
fixture := newStateRunFixture(t)
first := fixture.run(t, "--debug", "--cache", "auto")
if first.status != 0 {
t.Fatalf("first=%#v", first)
}
entries := readTree(t, fixture.cacheRoot)
calls := len(fixture.client.requestsSnapshot())
second := fixture.run(t, "--cache", "auto")
if second.status != 0 || len(fixture.client.requestsSnapshot()) != calls {
t.Fatalf("reused=%#v provider calls=%d/%d", second, len(fixture.client.requestsSnapshot()), calls)
}
if !sameFiles(entries, readTree(t, fixture.cacheRoot)) {
t.Fatal("accepted hit rewrote cached artifact")
}
if first.report.OutputDirectory == second.report.OutputDirectory {
t.Fatal("rerun did not get its own output directory")
}
assertFile(t, filepath.Join(first.report.OutputDirectory, "lanes", "spells.json"))
assertFile(t, filepath.Join(second.report.OutputDirectory, "lanes", "spells.json"))
assertFile(t, filepath.Join(first.report.DebugDirectory, "outcome.json"))
target, err := os.Readlink(filepath.Join(fixture.debugRoot, "latest"))
if err != nil || target != filepath.Base(first.report.DebugDirectory) {
t.Fatalf("disabled capture changed latest: %q %v", target, err)
}
}
func TestRunRetainsCaptureAcrossResolutionProviderAndPublicationFailures(t *testing.T) {
for _, failure := range []string{"resolution", "provider", "publication"} {
t.Run(failure, func(t *testing.T) {
fixture := newStateRunFixture(t)
flags := []string{"--debug", "--cache", "bypass", "--llm-transient-retries=0"}
fragment := ""
switch failure {
case "resolution":
flags = append(flags, "--lane", "missing")
fragment = "missing"
case "provider":
fixture.options.LLMClient = stateFailureClient{cause: errors.New("native provider failure")}
fragment = "native provider failure"
case "publication":
if err := os.WriteFile(fixture.outputRoot, []byte("not a directory"), 0600); err != nil {
t.Fatal(err)
}
fragment = fixture.outputRoot
}
result := fixture.run(t, flags...)
if result.status == 0 || !strings.Contains(result.stderr, fragment) || result.stdout != "" {
t.Fatalf("failed run=%#v", result)
}
bundle := onlyChildDir(t, fixture.debugRoot)
if !strings.Contains(result.stderr, bundle) || !strings.Contains(readAllFiles(t, bundle), fragment) {
t.Fatalf("failure evidence or path missing: %#v", result)
}
assertFile(t, filepath.Join(bundle, "outcome.json"))
if failure != "publication" {
assertAbsent(t, fixture.outputRoot)
} else {
data, err := os.ReadFile(fixture.outputRoot)
if err != nil || string(data) != "not a directory" {
t.Fatalf("blocked output parent changed: %s %v", data, err)
}
}
})
}
}
func TestRunCaptureContainsSelectedApplicationDataNotUnrelatedHostData(t *testing.T) {
fixture := newStateRunFixture(t)
t.Setenv("NOTARIUS_TEST_UNRELATED_ENV", "HOST_ONLY_SENTINEL")
if err := os.WriteFile(filepath.Join(filepath.Dir(fixture.configPath), "unrelated.txt"), []byte("HOST_ONLY_FILE_SENTINEL"), 0600); err != nil {
t.Fatal(err)
}
result := fixture.run(t, "--debug", "--cache", "bypass")
if result.status != 0 {
t.Fatalf("run=%#v", result)
}
evidence := readAllFiles(t, result.report.DebugDirectory)
if !strings.Contains(evidence, "Aria raises her holy symbol and casts Cure Wounds.") {
t.Fatal("capture omitted selected source")
}
for _, unrelated := range []string{"HOST_ONLY_SENTINEL", "HOST_ONLY_FILE_SENTINEL"} {
if strings.Contains(evidence, unrelated) {
t.Fatalf("capture included unrelated host data: %s", unrelated)
}
}
}
func TestRepeatedRunIdentityCannotOverwriteFirstOutput(t *testing.T) {
fixture := newStateRunFixture(t)
fixture.options.RunIDGenerator = func(time.Time) (string, error) { return "fixed-run", nil }
first := fixture.run(t, "--cache", "bypass")
if first.status != 0 {
t.Fatalf("first=%#v", first)
}
before := readTree(t, first.report.OutputDirectory)
second := fixture.run(t, "--cache", "bypass")
if second.status != 1 || !strings.Contains(second.stderr, "already exists") {
t.Fatalf("second=%#v", second)
}
if !sameFiles(before, readTree(t, first.report.OutputDirectory)) {
t.Fatal("output collision changed first run")
}
}
func TestRunRefusesExistingOutputAndDebugDirectoriesWithoutChangingThem(t *testing.T) {
for _, debug := range []bool{false, true} {
t.Run(fmt.Sprintf("debug=%t", debug), func(t *testing.T) {
fixture := newStateRunFixture(t)
fixture.options.RunIDGenerator = func(time.Time) (string, error) { return "fixed-run", nil }
selected := filepath.Join(fixture.outputRoot, "fixed-run")
if debug {
selected = filepath.Join(fixture.debugRoot, "fixed-run")
}
if err := os.MkdirAll(filepath.Join(selected, "nested"), 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(selected, "nested", "sentinel"), []byte("preserve me"), 0600); err != nil {
t.Fatal(err)
}
before := readTree(t, selected)
result := fixture.run(t, "--cache", "bypass", fmt.Sprintf("--debug=%t", debug))
if result.status != 1 || !strings.Contains(result.stderr, selected) {
t.Fatalf("collision=%#v", result)
}
if !reflect.DeepEqual(before, readTree(t, selected)) {
t.Fatal("existing directory changed")
}
if debug {
assertAbsent(t, fixture.outputRoot)
if len(fixture.client.requestsSnapshot()) != 0 {
t.Fatal("debug allocation failure reached provider")
}
}
})
}
}
func TestRunIdentityFailuresPrecedePathUse(t *testing.T) {
for _, test := range []struct {
name, id string
err error
}{
{name: "identity generation", err: errors.New("random source unavailable")},
{name: "unsafe identity", id: "../outside"},
} {
t.Run(test.name, func(t *testing.T) {
fixture := newStateRunFixture(t)
fixture.options.RunIDGenerator = func(time.Time) (string, error) { return test.id, test.err }
result := fixture.run(t, "--debug")
if result.status != 1 || result.stderr == "" || len(fixture.client.requestsSnapshot()) != 0 {
t.Fatalf("identity failure=%#v", result)
}
assertAbsent(t, fixture.outputRoot)
assertAbsent(t, fixture.debugRoot)
assertAbsent(t, fixture.cacheRoot)
})
}
}
type stateFailureClient struct{ cause error }
func (client stateFailureClient) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
return nil, client.cause
}
func onlyChildDir(t *testing.T, root string) string {
t.Helper()
entries, err := os.ReadDir(root)
if err != nil {
t.Fatal(err)
}
var dirs []string
for _, entry := range entries {
if entry.IsDir() {
dirs = append(dirs, filepath.Join(root, entry.Name()))
}
}
if len(dirs) != 1 {
t.Fatalf("directories in %q = %v, want one", root, dirs)
}
return dirs[0]
}
func assertFile(t *testing.T, path string) {
t.Helper()
if info, err := os.Stat(path); err != nil || info.IsDir() {
t.Fatalf("file %q: %v", path, err)
}
}
func assertAbsent(t *testing.T, path string) {
t.Helper()
if _, err := os.Stat(path); !os.IsNotExist(err) {
t.Fatalf("%q exists or stat failed: %v", path, err)
}
}
func readAllFiles(t *testing.T, root string) string {
t.Helper()
var content strings.Builder
if err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error {
if err != nil {
return err
}
if entry.IsDir() {
return nil
}
data, err := os.ReadFile(path)
if err != nil {
return err
}
content.Write(data)
return nil
}); err != nil {
t.Fatal(err)
}
return content.String()
}
func readTree(t *testing.T, root string) map[string][]byte {
t.Helper()
files := map[string][]byte{}
if err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error {
if err != nil {
return err
}
if entry.IsDir() {
return nil
}
data, err := os.ReadFile(path)
if err != nil {
return err
}
relative, err := filepath.Rel(root, path)
if err != nil {
return err
}
files[relative] = data
return nil
}); err != nil {
t.Fatal(err)
}
return files
}
func sameFiles(left, right map[string][]byte) bool {
if len(left) != len(right) {
return false
}
for path, data := range left {
if !bytes.Equal(data, right[path]) {
return false
}
}
return true
}