Unify chunk-plan caching and remove checkpoint execution paths
This commit is contained in:
@@ -3,331 +3,11 @@ package cli
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkplan"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
func TestRunChunkPlanModePrecedenceAndValidation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
fileMode string
|
||||
envMode string
|
||||
cliMode string
|
||||
wantStores int
|
||||
}{
|
||||
{name: "default", wantStores: 1},
|
||||
{name: "file", fileMode: "bypass"},
|
||||
{name: "retired environment has no effect", envMode: "bypass", wantStores: 1},
|
||||
{name: "cli", envMode: "refresh", cliMode: "bypass"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
if tt.name == "default" {
|
||||
removeStateTestConfigLine(t, roots.config, " mode: auto\n")
|
||||
} else if tt.fileMode != "" {
|
||||
replaceStateTestConfigLine(t, roots.config, " mode: auto\n", " mode: "+tt.fileMode+"\n")
|
||||
}
|
||||
|
||||
var stores []string
|
||||
opts := newStateTestHarness().options()
|
||||
opts.LookupEnv = func(name string) (string, bool) {
|
||||
if name == "NOTARIUS_CACHE_CHUNK_PLANS_MODE" && tt.envMode != "" {
|
||||
return tt.envMode, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
opts.ChunkPlanStoreFactory = func(root string) (pipeline.ChunkPlanStore, error) {
|
||||
stores = append(stores, root)
|
||||
return chunkplan.NewFilesystemStore(root)
|
||||
}
|
||||
|
||||
args := []string{"run", "sample", "--config", roots.config, "--input", roots.input}
|
||||
if tt.cliMode != "" {
|
||||
args = append(args, "--chunk_cache", tt.cliMode)
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
if code := RunWithOptions(args, &stdout, &stderr, opts); code != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
assertStateTestOutput(t, roots.output)
|
||||
if len(stores) != tt.wantStores {
|
||||
t.Fatalf("chunk plan store roots = %v, want %d stores", stores, tt.wantStores)
|
||||
}
|
||||
if tt.wantStores == 1 && stores[0] != roots.plans {
|
||||
t.Fatalf("chunk plan store root = %q, want %q", stores[0], roots.plans)
|
||||
}
|
||||
if tt.wantStores == 0 {
|
||||
assertAbsent(t, roots.plans)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("invalid cli syntax is a usage error", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "invalid"}, &stdout, &stderr, newStateTestHarness().options())
|
||||
if code != 2 || stdout.Len() != 0 || stderr.Len() == 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
assertNoRunState(t, roots)
|
||||
})
|
||||
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
}{
|
||||
{name: "invalid file mode"},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
opts := newStateTestHarness().options()
|
||||
replaceStateTestConfigLine(t, roots.config, " mode: auto\n", " mode: invalid\n")
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input}, &stdout, &stderr, opts)
|
||||
if code != 1 || stdout.Len() != 0 || stderr.Len() == 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
assertNoRunState(t, roots)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunChunkPlanRootSelectionAndFailures(t *testing.T) {
|
||||
t.Run("empty configured root uses the per-user cache root", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
removeStateTestConfigLine(t, roots.config, fmt.Sprintf(" directory: %q\n", roots.plans))
|
||||
userCache := filepath.Join(t.TempDir(), "user-cache")
|
||||
var stores []string
|
||||
opts := newStateTestHarness().options()
|
||||
opts.UserCacheDir = func() (string, error) { return userCache, nil }
|
||||
opts.ChunkPlanStoreFactory = func(root string) (pipeline.ChunkPlanStore, error) {
|
||||
stores = append(stores, root)
|
||||
return chunkplan.NewFilesystemStore(root)
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input}, &stdout, &stderr, opts)
|
||||
if code != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
wantRoot := filepath.Join(userCache, "notarius", "chunk-plans")
|
||||
if len(stores) != 1 || stores[0] != wantRoot {
|
||||
t.Fatalf("chunk plan store roots = %v, want [%q]", stores, wantRoot)
|
||||
}
|
||||
assertFile(t, filepath.Join(wantRoot, strings.TrimPrefix(stateTestDigest, "sha256:"), "plan.json"))
|
||||
assertAbsent(t, roots.plans)
|
||||
})
|
||||
|
||||
t.Run("bypass avoids default cache dependencies", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
removeStateTestConfigLine(t, roots.config, fmt.Sprintf(" directory: %q\n", roots.plans))
|
||||
userCacheCalls := 0
|
||||
storeCalls := 0
|
||||
opts := newStateTestHarness().options()
|
||||
opts.UserCacheDir = func() (string, error) {
|
||||
userCacheCalls++
|
||||
return "", errors.New("user cache must not be resolved")
|
||||
}
|
||||
opts.ChunkPlanStoreFactory = func(string) (pipeline.ChunkPlanStore, error) {
|
||||
storeCalls++
|
||||
return nil, errors.New("chunk plan store must not be constructed")
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}, &stdout, &stderr, opts)
|
||||
if code != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
if userCacheCalls != 0 || storeCalls != 0 {
|
||||
t.Fatalf("user cache calls=%d store calls=%d, want none", userCacheCalls, storeCalls)
|
||||
}
|
||||
assertStateTestOutput(t, roots.output)
|
||||
assertAbsent(t, roots.plans)
|
||||
})
|
||||
|
||||
t.Run("user cache resolution failure has context and no output", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
removeStateTestConfigLine(t, roots.config, fmt.Sprintf(" directory: %q\n", roots.plans))
|
||||
opts := newStateTestHarness().options()
|
||||
opts.UserCacheDir = func() (string, error) { return "", errors.New("cache home unavailable") }
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input}, &stdout, &stderr, opts)
|
||||
if code != 1 || !strings.Contains(stderr.String(), "resolve chunk plan root") || !strings.Contains(stderr.String(), "cache home unavailable") {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
assertNoRunState(t, roots)
|
||||
})
|
||||
|
||||
t.Run("store construction failure has context and no output", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
opts := newStateTestHarness().options()
|
||||
opts.ChunkPlanStoreFactory = func(root string) (pipeline.ChunkPlanStore, error) {
|
||||
return nil, fmt.Errorf("store unavailable")
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input}, &stdout, &stderr, opts)
|
||||
want := fmt.Sprintf("create chunk plan store at %q", roots.plans)
|
||||
if code != 1 || !strings.Contains(stderr.String(), want) || !strings.Contains(stderr.String(), "store unavailable") {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
assertNoRunState(t, roots)
|
||||
})
|
||||
|
||||
t.Run("checkpoint root resolution failure has context and no output", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
removeStateTestConfigLine(t, roots.config, fmt.Sprintf(" directory: %q\n", roots.checkpoints))
|
||||
opts := newStateTestHarness().options()
|
||||
opts.UserCacheDir = func() (string, error) { return "", errors.New("checkpoint cache unavailable") }
|
||||
result := runStateTest(t, roots, opts, false, true, "bypass")
|
||||
if result.code != 1 || !strings.Contains(result.stderr, "resolve checkpoint root") || !strings.Contains(result.stderr, "checkpoint cache unavailable") {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr)
|
||||
}
|
||||
assertNoRunState(t, roots)
|
||||
})
|
||||
}
|
||||
|
||||
func TestRunAutoReusesPlanWhenRunInputsChange(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
data, err := os.ReadFile(roots.config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
configText := replaceRequiredOnce(t, string(data), " chunk:\n module: test/chunk\n", ` chunk:
|
||||
module: test/chunk
|
||||
options:
|
||||
strategy: first
|
||||
`)
|
||||
configText = replaceRequiredOnce(t, configText, " output:\n module: test/output\n", ` other:
|
||||
extract:
|
||||
module: test/extract
|
||||
merge:
|
||||
module: test/merge
|
||||
normalize:
|
||||
module: test/normalize
|
||||
output:
|
||||
module: test/output
|
||||
`)
|
||||
if err := os.WriteFile(roots.config, []byte(configText), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
referencePath := filepath.Join(filepath.Dir(roots.input), "reference.txt")
|
||||
if err := os.WriteFile(referencePath, []byte("reference content"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
harness := newStateTestHarness()
|
||||
var firstStdout, firstStderr bytes.Buffer
|
||||
first := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input}, &firstStdout, &firstStderr, harness.options())
|
||||
if first != 0 {
|
||||
t.Fatalf("first run code=%d stdout=%q stderr=%q", first, firstStdout.String(), firstStderr.String())
|
||||
}
|
||||
configData, err := os.ReadFile(roots.config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
configText = replaceRequiredOnce(t, string(configData), "strategy: first", "strategy: second")
|
||||
if err := os.WriteFile(roots.config, []byte(configText), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
second := RunWithOptions([]string{
|
||||
"run", "sample", "--config", roots.config, "--input", roots.input,
|
||||
"--only", "items", "--reference", "chunk.cache-reference=" + referencePath,
|
||||
}, &stdout, &stderr, harness.options())
|
||||
if second != 0 {
|
||||
t.Fatalf("second run code=%d stdout=%q stderr=%q", second, stdout.String(), stderr.String())
|
||||
}
|
||||
harness.mu.Lock()
|
||||
chunkCalls := harness.chunkCalls
|
||||
sessions := append([]string(nil), harness.sessionIDs...)
|
||||
harness.mu.Unlock()
|
||||
if chunkCalls != 1 {
|
||||
t.Fatalf("chunk calls across changed run inputs = %d, want 1", chunkCalls)
|
||||
}
|
||||
rawInput, err := os.ReadFile(roots.input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantSessionID, err := resolvePromptSessionID("", "test/input", rawInput)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, sessionID := range sessions {
|
||||
if sessionID != wantSessionID {
|
||||
t.Fatalf("session IDs across reference changes = %#v, want %q", sessions, wantSessionID)
|
||||
}
|
||||
}
|
||||
assertFile(t, filepath.Join(roots.plans, strings.TrimPrefix(stateTestDigest, "sha256:"), "plan.json"))
|
||||
assertAnyFile(t, roots.output)
|
||||
}
|
||||
|
||||
func TestRunResumeSelectsConfiguredOrPerUserCheckpointRoot(t *testing.T) {
|
||||
for _, configured := range []bool{true, false} {
|
||||
name := "per-user root"
|
||||
if configured {
|
||||
name = "configured root"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
if !configured {
|
||||
removeStateTestConfigLine(t, roots.config, fmt.Sprintf(" directory: %q\n", roots.checkpoints))
|
||||
}
|
||||
userCache := filepath.Join(t.TempDir(), "user-cache")
|
||||
userCacheCalls := 0
|
||||
opts := newStateTestHarness().options()
|
||||
opts.UserCacheDir = func() (string, error) {
|
||||
userCacheCalls++
|
||||
return userCache, nil
|
||||
}
|
||||
result := runStateTest(t, roots, opts, false, true, "bypass")
|
||||
if result.code != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr)
|
||||
}
|
||||
wantRoot := roots.checkpoints
|
||||
wantCalls := 0
|
||||
if !configured {
|
||||
wantRoot = filepath.Join(userCache, "notarius", "checkpoints")
|
||||
wantCalls = 1
|
||||
}
|
||||
if userCacheCalls != wantCalls {
|
||||
t.Fatalf("user cache calls = %d, want %d", userCacheCalls, wantCalls)
|
||||
}
|
||||
assertAnyFile(t, wantRoot)
|
||||
if !configured {
|
||||
assertAbsent(t, roots.checkpoints)
|
||||
}
|
||||
assertStateTestOutput(t, roots.output)
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("disabled avoids checkpoint root resolution", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
replaceStateTestConfigLine(t, roots.config, " enabled: true\n", " enabled: false\n")
|
||||
removeStateTestConfigLine(t, roots.config, fmt.Sprintf(" directory: %q\n", roots.checkpoints))
|
||||
opts := newStateTestHarness().options()
|
||||
opts.UserCacheDir = func() (string, error) { return "", errors.New("checkpoint cache must not be resolved") }
|
||||
result := runStateTest(t, roots, opts, false, false, "bypass")
|
||||
if result.code != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr)
|
||||
}
|
||||
assertStateTestOutput(t, roots.output)
|
||||
assertAbsent(t, roots.checkpoints)
|
||||
})
|
||||
|
||||
t.Run("resume requires enabled checkpoint recording", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
replaceStateTestConfigLine(t, roots.config, " enabled: true\n", " enabled: false\n")
|
||||
result := runStateTest(t, roots, newStateTestHarness().options(), true, true, "bypass")
|
||||
if result.code != 1 || !strings.Contains(result.stderr, "--resume requires cache.checkpoints.enabled: true") {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr)
|
||||
}
|
||||
assertNoRunState(t, roots)
|
||||
})
|
||||
}
|
||||
|
||||
func TestConfigCommandsDoNotResolveRunState(t *testing.T) {
|
||||
for _, args := range [][]string{
|
||||
{"config", "validate", "--config"},
|
||||
@@ -338,9 +18,6 @@ func TestConfigCommandsDoNotResolveRunState(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
opts := newStateTestHarness().options()
|
||||
opts.UserCacheDir = func() (string, error) { return "", errors.New("state root must not be resolved") }
|
||||
opts.ChunkPlanStoreFactory = func(string) (pipeline.ChunkPlanStore, error) {
|
||||
return nil, errors.New("chunk plan store must not be constructed")
|
||||
}
|
||||
command := append([]string(nil), args...)
|
||||
command = append(command, roots.config)
|
||||
var stdout, stderr bytes.Buffer
|
||||
|
||||
Reference in New Issue
Block a user