Unify chunk-plan caching and remove checkpoint execution paths
This commit is contained in:
@@ -3,331 +3,11 @@ package cli
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkplan"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
func TestRunChunkPlanModePrecedenceAndValidation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
fileMode string
|
||||
envMode string
|
||||
cliMode string
|
||||
wantStores int
|
||||
}{
|
||||
{name: "default", wantStores: 1},
|
||||
{name: "file", fileMode: "bypass"},
|
||||
{name: "retired environment has no effect", envMode: "bypass", wantStores: 1},
|
||||
{name: "cli", envMode: "refresh", cliMode: "bypass"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
if tt.name == "default" {
|
||||
removeStateTestConfigLine(t, roots.config, " mode: auto\n")
|
||||
} else if tt.fileMode != "" {
|
||||
replaceStateTestConfigLine(t, roots.config, " mode: auto\n", " mode: "+tt.fileMode+"\n")
|
||||
}
|
||||
|
||||
var stores []string
|
||||
opts := newStateTestHarness().options()
|
||||
opts.LookupEnv = func(name string) (string, bool) {
|
||||
if name == "NOTARIUS_CACHE_CHUNK_PLANS_MODE" && tt.envMode != "" {
|
||||
return tt.envMode, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
opts.ChunkPlanStoreFactory = func(root string) (pipeline.ChunkPlanStore, error) {
|
||||
stores = append(stores, root)
|
||||
return chunkplan.NewFilesystemStore(root)
|
||||
}
|
||||
|
||||
args := []string{"run", "sample", "--config", roots.config, "--input", roots.input}
|
||||
if tt.cliMode != "" {
|
||||
args = append(args, "--chunk_cache", tt.cliMode)
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
if code := RunWithOptions(args, &stdout, &stderr, opts); code != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
assertStateTestOutput(t, roots.output)
|
||||
if len(stores) != tt.wantStores {
|
||||
t.Fatalf("chunk plan store roots = %v, want %d stores", stores, tt.wantStores)
|
||||
}
|
||||
if tt.wantStores == 1 && stores[0] != roots.plans {
|
||||
t.Fatalf("chunk plan store root = %q, want %q", stores[0], roots.plans)
|
||||
}
|
||||
if tt.wantStores == 0 {
|
||||
assertAbsent(t, roots.plans)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("invalid cli syntax is a usage error", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "invalid"}, &stdout, &stderr, newStateTestHarness().options())
|
||||
if code != 2 || stdout.Len() != 0 || stderr.Len() == 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
assertNoRunState(t, roots)
|
||||
})
|
||||
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
}{
|
||||
{name: "invalid file mode"},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
opts := newStateTestHarness().options()
|
||||
replaceStateTestConfigLine(t, roots.config, " mode: auto\n", " mode: invalid\n")
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input}, &stdout, &stderr, opts)
|
||||
if code != 1 || stdout.Len() != 0 || stderr.Len() == 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
assertNoRunState(t, roots)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunChunkPlanRootSelectionAndFailures(t *testing.T) {
|
||||
t.Run("empty configured root uses the per-user cache root", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
removeStateTestConfigLine(t, roots.config, fmt.Sprintf(" directory: %q\n", roots.plans))
|
||||
userCache := filepath.Join(t.TempDir(), "user-cache")
|
||||
var stores []string
|
||||
opts := newStateTestHarness().options()
|
||||
opts.UserCacheDir = func() (string, error) { return userCache, nil }
|
||||
opts.ChunkPlanStoreFactory = func(root string) (pipeline.ChunkPlanStore, error) {
|
||||
stores = append(stores, root)
|
||||
return chunkplan.NewFilesystemStore(root)
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input}, &stdout, &stderr, opts)
|
||||
if code != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
wantRoot := filepath.Join(userCache, "notarius", "chunk-plans")
|
||||
if len(stores) != 1 || stores[0] != wantRoot {
|
||||
t.Fatalf("chunk plan store roots = %v, want [%q]", stores, wantRoot)
|
||||
}
|
||||
assertFile(t, filepath.Join(wantRoot, strings.TrimPrefix(stateTestDigest, "sha256:"), "plan.json"))
|
||||
assertAbsent(t, roots.plans)
|
||||
})
|
||||
|
||||
t.Run("bypass avoids default cache dependencies", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
removeStateTestConfigLine(t, roots.config, fmt.Sprintf(" directory: %q\n", roots.plans))
|
||||
userCacheCalls := 0
|
||||
storeCalls := 0
|
||||
opts := newStateTestHarness().options()
|
||||
opts.UserCacheDir = func() (string, error) {
|
||||
userCacheCalls++
|
||||
return "", errors.New("user cache must not be resolved")
|
||||
}
|
||||
opts.ChunkPlanStoreFactory = func(string) (pipeline.ChunkPlanStore, error) {
|
||||
storeCalls++
|
||||
return nil, errors.New("chunk plan store must not be constructed")
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}, &stdout, &stderr, opts)
|
||||
if code != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
if userCacheCalls != 0 || storeCalls != 0 {
|
||||
t.Fatalf("user cache calls=%d store calls=%d, want none", userCacheCalls, storeCalls)
|
||||
}
|
||||
assertStateTestOutput(t, roots.output)
|
||||
assertAbsent(t, roots.plans)
|
||||
})
|
||||
|
||||
t.Run("user cache resolution failure has context and no output", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
removeStateTestConfigLine(t, roots.config, fmt.Sprintf(" directory: %q\n", roots.plans))
|
||||
opts := newStateTestHarness().options()
|
||||
opts.UserCacheDir = func() (string, error) { return "", errors.New("cache home unavailable") }
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input}, &stdout, &stderr, opts)
|
||||
if code != 1 || !strings.Contains(stderr.String(), "resolve chunk plan root") || !strings.Contains(stderr.String(), "cache home unavailable") {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
assertNoRunState(t, roots)
|
||||
})
|
||||
|
||||
t.Run("store construction failure has context and no output", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
opts := newStateTestHarness().options()
|
||||
opts.ChunkPlanStoreFactory = func(root string) (pipeline.ChunkPlanStore, error) {
|
||||
return nil, fmt.Errorf("store unavailable")
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input}, &stdout, &stderr, opts)
|
||||
want := fmt.Sprintf("create chunk plan store at %q", roots.plans)
|
||||
if code != 1 || !strings.Contains(stderr.String(), want) || !strings.Contains(stderr.String(), "store unavailable") {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
assertNoRunState(t, roots)
|
||||
})
|
||||
|
||||
t.Run("checkpoint root resolution failure has context and no output", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
removeStateTestConfigLine(t, roots.config, fmt.Sprintf(" directory: %q\n", roots.checkpoints))
|
||||
opts := newStateTestHarness().options()
|
||||
opts.UserCacheDir = func() (string, error) { return "", errors.New("checkpoint cache unavailable") }
|
||||
result := runStateTest(t, roots, opts, false, true, "bypass")
|
||||
if result.code != 1 || !strings.Contains(result.stderr, "resolve checkpoint root") || !strings.Contains(result.stderr, "checkpoint cache unavailable") {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr)
|
||||
}
|
||||
assertNoRunState(t, roots)
|
||||
})
|
||||
}
|
||||
|
||||
func TestRunAutoReusesPlanWhenRunInputsChange(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
data, err := os.ReadFile(roots.config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
configText := replaceRequiredOnce(t, string(data), " chunk:\n module: test/chunk\n", ` chunk:
|
||||
module: test/chunk
|
||||
options:
|
||||
strategy: first
|
||||
`)
|
||||
configText = replaceRequiredOnce(t, configText, " output:\n module: test/output\n", ` other:
|
||||
extract:
|
||||
module: test/extract
|
||||
merge:
|
||||
module: test/merge
|
||||
normalize:
|
||||
module: test/normalize
|
||||
output:
|
||||
module: test/output
|
||||
`)
|
||||
if err := os.WriteFile(roots.config, []byte(configText), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
referencePath := filepath.Join(filepath.Dir(roots.input), "reference.txt")
|
||||
if err := os.WriteFile(referencePath, []byte("reference content"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
harness := newStateTestHarness()
|
||||
var firstStdout, firstStderr bytes.Buffer
|
||||
first := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input}, &firstStdout, &firstStderr, harness.options())
|
||||
if first != 0 {
|
||||
t.Fatalf("first run code=%d stdout=%q stderr=%q", first, firstStdout.String(), firstStderr.String())
|
||||
}
|
||||
configData, err := os.ReadFile(roots.config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
configText = replaceRequiredOnce(t, string(configData), "strategy: first", "strategy: second")
|
||||
if err := os.WriteFile(roots.config, []byte(configText), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
second := RunWithOptions([]string{
|
||||
"run", "sample", "--config", roots.config, "--input", roots.input,
|
||||
"--only", "items", "--reference", "chunk.cache-reference=" + referencePath,
|
||||
}, &stdout, &stderr, harness.options())
|
||||
if second != 0 {
|
||||
t.Fatalf("second run code=%d stdout=%q stderr=%q", second, stdout.String(), stderr.String())
|
||||
}
|
||||
harness.mu.Lock()
|
||||
chunkCalls := harness.chunkCalls
|
||||
sessions := append([]string(nil), harness.sessionIDs...)
|
||||
harness.mu.Unlock()
|
||||
if chunkCalls != 1 {
|
||||
t.Fatalf("chunk calls across changed run inputs = %d, want 1", chunkCalls)
|
||||
}
|
||||
rawInput, err := os.ReadFile(roots.input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantSessionID, err := resolvePromptSessionID("", "test/input", rawInput)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, sessionID := range sessions {
|
||||
if sessionID != wantSessionID {
|
||||
t.Fatalf("session IDs across reference changes = %#v, want %q", sessions, wantSessionID)
|
||||
}
|
||||
}
|
||||
assertFile(t, filepath.Join(roots.plans, strings.TrimPrefix(stateTestDigest, "sha256:"), "plan.json"))
|
||||
assertAnyFile(t, roots.output)
|
||||
}
|
||||
|
||||
func TestRunResumeSelectsConfiguredOrPerUserCheckpointRoot(t *testing.T) {
|
||||
for _, configured := range []bool{true, false} {
|
||||
name := "per-user root"
|
||||
if configured {
|
||||
name = "configured root"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
if !configured {
|
||||
removeStateTestConfigLine(t, roots.config, fmt.Sprintf(" directory: %q\n", roots.checkpoints))
|
||||
}
|
||||
userCache := filepath.Join(t.TempDir(), "user-cache")
|
||||
userCacheCalls := 0
|
||||
opts := newStateTestHarness().options()
|
||||
opts.UserCacheDir = func() (string, error) {
|
||||
userCacheCalls++
|
||||
return userCache, nil
|
||||
}
|
||||
result := runStateTest(t, roots, opts, false, true, "bypass")
|
||||
if result.code != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr)
|
||||
}
|
||||
wantRoot := roots.checkpoints
|
||||
wantCalls := 0
|
||||
if !configured {
|
||||
wantRoot = filepath.Join(userCache, "notarius", "checkpoints")
|
||||
wantCalls = 1
|
||||
}
|
||||
if userCacheCalls != wantCalls {
|
||||
t.Fatalf("user cache calls = %d, want %d", userCacheCalls, wantCalls)
|
||||
}
|
||||
assertAnyFile(t, wantRoot)
|
||||
if !configured {
|
||||
assertAbsent(t, roots.checkpoints)
|
||||
}
|
||||
assertStateTestOutput(t, roots.output)
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("disabled avoids checkpoint root resolution", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
replaceStateTestConfigLine(t, roots.config, " enabled: true\n", " enabled: false\n")
|
||||
removeStateTestConfigLine(t, roots.config, fmt.Sprintf(" directory: %q\n", roots.checkpoints))
|
||||
opts := newStateTestHarness().options()
|
||||
opts.UserCacheDir = func() (string, error) { return "", errors.New("checkpoint cache must not be resolved") }
|
||||
result := runStateTest(t, roots, opts, false, false, "bypass")
|
||||
if result.code != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr)
|
||||
}
|
||||
assertStateTestOutput(t, roots.output)
|
||||
assertAbsent(t, roots.checkpoints)
|
||||
})
|
||||
|
||||
t.Run("resume requires enabled checkpoint recording", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
replaceStateTestConfigLine(t, roots.config, " enabled: true\n", " enabled: false\n")
|
||||
result := runStateTest(t, roots, newStateTestHarness().options(), true, true, "bypass")
|
||||
if result.code != 1 || !strings.Contains(result.stderr, "--resume requires cache.checkpoints.enabled: true") {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr)
|
||||
}
|
||||
assertNoRunState(t, roots)
|
||||
})
|
||||
}
|
||||
|
||||
func TestConfigCommandsDoNotResolveRunState(t *testing.T) {
|
||||
for _, args := range [][]string{
|
||||
{"config", "validate", "--config"},
|
||||
@@ -338,9 +18,6 @@ func TestConfigCommandsDoNotResolveRunState(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
opts := newStateTestHarness().options()
|
||||
opts.UserCacheDir = func() (string, error) { return "", errors.New("state root must not be resolved") }
|
||||
opts.ChunkPlanStoreFactory = func(string) (pipeline.ChunkPlanStore, error) {
|
||||
return nil, errors.New("chunk plan store must not be constructed")
|
||||
}
|
||||
command := append([]string(nil), args...)
|
||||
command = append(command, roots.config)
|
||||
var stdout, stderr bytes.Buffer
|
||||
|
||||
@@ -1,338 +0,0 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
func TestRecomputeStepRecoversThroughFilesystemCheckpoints(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
invalidateOutput bool
|
||||
wantCode int
|
||||
}{
|
||||
{name: "accepted producer is hydrated", wantCode: 0},
|
||||
{name: "invalid producer stops dependents", invalidateOutput: true, wantCode: 1},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
roots := newRecomputeTestRoots(t)
|
||||
harness := newRecomputeTestHarness()
|
||||
fresh := runRecomputeCommand(roots, harness.options(), false)
|
||||
if fresh.code != 0 {
|
||||
t.Fatalf("fresh run code=%d stderr=%q", fresh.code, fresh.stderr)
|
||||
}
|
||||
removeCheckpointLaneStage(t, roots.checkpoints, "extract", "first", "producer")
|
||||
removeCheckpointLaneStage(t, roots.checkpoints, "merge", "first", "producer")
|
||||
if tt.invalidateOutput {
|
||||
path := findCheckpointFile(t, roots.checkpoints, "normalize", "first", "producer", "output.json")
|
||||
if err := os.WriteFile(path, []byte("{"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
harness.resetCalls()
|
||||
|
||||
resumed := runRecomputeCommand(roots, harness.options(), true)
|
||||
if resumed.code != tt.wantCode {
|
||||
t.Fatalf("resumed code=%d stdout=%q stderr=%q", resumed.code, resumed.stdout, resumed.stderr)
|
||||
}
|
||||
events := readLatestCheckpointEvents(t, roots.debug)
|
||||
if tt.invalidateOutput {
|
||||
if harness.callsFor("test/extract/middle") != 0 || harness.callsFor("test/extract/dependent") != 0 {
|
||||
t.Fatalf("dependent calls after invalid producer = %#v", harness.callsSnapshot())
|
||||
}
|
||||
if !strings.Contains(resumed.stderr, string(pipeline.CheckpointReasonDecodeFailed)) {
|
||||
t.Fatalf("stderr=%q, want stable checkpoint reason", resumed.stderr)
|
||||
}
|
||||
assertNormalizeDecisionSequence(t, events, []checkpointDecisionExpectation{{"first", "producer", pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonDecodeFailed}})
|
||||
return
|
||||
}
|
||||
|
||||
if got := harness.callsSnapshot(); !reflect.DeepEqual(got, map[string]int{"test/extract/dependent": 1, "test/extract/middle": 1}) {
|
||||
t.Fatalf("resumed extractor calls = %#v", got)
|
||||
}
|
||||
outputPath := filepath.Join(latestChildDir(t, roots.output), "result.json")
|
||||
data, err := os.ReadFile(outputPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(data) != "[\"producer\",\"unrelated\",\"middle\",\"dependent\"]\n" {
|
||||
t.Fatalf("ordered output = %q", data)
|
||||
}
|
||||
assertNormalizeDecisionSequence(t, events, []checkpointDecisionExpectation{
|
||||
{"first", "producer", pipeline.CheckpointDecisionReused, pipeline.CheckpointReasonAcceptedArtifactReused},
|
||||
{"first", "unrelated", pipeline.CheckpointDecisionReused, pipeline.CheckpointReasonReused},
|
||||
{"second", "middle", pipeline.CheckpointDecisionForcedRecompute, pipeline.CheckpointReasonRecomputeStep},
|
||||
{"third", "dependent", pipeline.CheckpointDecisionForcedRecompute, pipeline.CheckpointReasonRecomputeStep},
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type checkpointDecisionExpectation struct {
|
||||
step, lane string
|
||||
action pipeline.CheckpointDecisionCategory
|
||||
reason pipeline.CheckpointReasonCode
|
||||
}
|
||||
|
||||
func assertNormalizeDecisionSequence(t *testing.T, events []pipeline.CheckpointEvent, want []checkpointDecisionExpectation) {
|
||||
t.Helper()
|
||||
var got []checkpointDecisionExpectation
|
||||
for _, event := range events {
|
||||
if event.Stage == string(pipeline.StageNormalize) {
|
||||
got = append(got, checkpointDecisionExpectation{event.StepID, event.LaneID, event.Action, event.ReasonCode})
|
||||
}
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("normalize decisions = %#v, want %#v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
type recomputeTestHarness struct {
|
||||
base *stateTestHarness
|
||||
mu sync.Mutex
|
||||
calls map[string]int
|
||||
}
|
||||
|
||||
func newRecomputeTestHarness() *recomputeTestHarness {
|
||||
return &recomputeTestHarness{base: newStateTestHarness(), calls: make(map[string]int)}
|
||||
}
|
||||
|
||||
func (h *recomputeTestHarness) options() Options {
|
||||
opts := h.base.options()
|
||||
for _, key := range []string{"test/extract/producer", "test/extract/unrelated", "test/extract/middle", "test/extract/dependent"} {
|
||||
moduleKey := key
|
||||
spec := pipeline.ModuleSpec{
|
||||
Key: moduleKey, Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: stateTestArtifactKind,
|
||||
ReferenceSlots: []contracts.ReferenceSlot{{Name: "upstream", AcceptedMediaTypes: []string{"application/json"}, AcceptedArtifactKinds: []contracts.ArtifactKind{stateTestArtifactKind}}},
|
||||
}
|
||||
if err := pipeline.RegisterExtractor(opts.Registries.Extractors, spec, func() (contracts.Extractor[stateTestArtifact], error) {
|
||||
return recomputeTestExtractor{key: moduleKey, harness: h}, nil
|
||||
}); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
if err := opts.Registries.Outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "test/recompute-output", Stage: pipeline.StageOutput, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"normalized"}, Provides: []string{"output"}}, func() (contracts.OutputEncoder, error) {
|
||||
return recomputeTestOutput{}, nil
|
||||
}); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
opts.Catalog = catalogFromRegistries(opts.Registries)
|
||||
return opts
|
||||
}
|
||||
|
||||
func (h *recomputeTestHarness) record(key string) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.calls[key]++
|
||||
}
|
||||
|
||||
func (h *recomputeTestHarness) resetCalls() {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.calls = make(map[string]int)
|
||||
}
|
||||
|
||||
func (h *recomputeTestHarness) callsFor(key string) int {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
return h.calls[key]
|
||||
}
|
||||
|
||||
func (h *recomputeTestHarness) callsSnapshot() map[string]int {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
result := make(map[string]int, len(h.calls))
|
||||
for key, value := range h.calls {
|
||||
result[key] = value
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
type recomputeTestExtractor struct {
|
||||
key string
|
||||
harness *recomputeTestHarness
|
||||
}
|
||||
|
||||
func (e recomputeTestExtractor) Key() string { return e.key }
|
||||
func (e recomputeTestExtractor) ReferenceSlots() []contracts.ReferenceSlot {
|
||||
return []contracts.ReferenceSlot{{Name: "upstream", AcceptedMediaTypes: []string{"application/json"}, AcceptedArtifactKinds: []contracts.ArtifactKind{stateTestArtifactKind}}}
|
||||
}
|
||||
func (e recomputeTestExtractor) Extract(context.Context, contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[stateTestArtifact], error) {
|
||||
e.harness.record(e.key)
|
||||
return contracts.TypedExtractionResult[stateTestArtifact]{Value: stateTestArtifact{Value: e.key}}, nil
|
||||
}
|
||||
|
||||
type recomputeTestOutput struct{}
|
||||
|
||||
func (recomputeTestOutput) Key() string { return "test/recompute-output" }
|
||||
func (recomputeTestOutput) Encode(_ context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
|
||||
lanes := make([]string, 0, len(req.NormalizeOutputs))
|
||||
for _, output := range req.NormalizeOutputs {
|
||||
lanes = append(lanes, output.LaneID)
|
||||
}
|
||||
data, err := json.Marshal(lanes)
|
||||
if err != nil {
|
||||
return contracts.OutputResult{}, err
|
||||
}
|
||||
return contracts.OutputResult{Files: []contracts.OutputFile{{Name: "result.json", Bytes: append(data, '\n')}}}, nil
|
||||
}
|
||||
|
||||
func newRecomputeTestRoots(t *testing.T) stateTestRoots {
|
||||
t.Helper()
|
||||
roots := newStateTestRoots(t)
|
||||
config := fmt.Sprintf(`version: 5
|
||||
output:
|
||||
directory: %q
|
||||
cache:
|
||||
chunk_plans:
|
||||
directory: %q
|
||||
mode: bypass
|
||||
checkpoints:
|
||||
enabled: true
|
||||
directory: %q
|
||||
debug:
|
||||
directory: %q
|
||||
pipelines:
|
||||
sample:
|
||||
input:
|
||||
module: test/input
|
||||
chunk:
|
||||
module: test/chunk
|
||||
steps:
|
||||
- id: first
|
||||
artifacts:
|
||||
producer:
|
||||
extract:
|
||||
module: test/extract/producer
|
||||
merge:
|
||||
module: test/merge
|
||||
normalize:
|
||||
module: test/normalize
|
||||
unrelated:
|
||||
extract:
|
||||
module: test/extract/unrelated
|
||||
merge:
|
||||
module: test/merge
|
||||
normalize:
|
||||
module: test/normalize
|
||||
- id: second
|
||||
references:
|
||||
upstream:
|
||||
artifact:
|
||||
step: first
|
||||
lane: producer
|
||||
artifacts:
|
||||
middle:
|
||||
extract:
|
||||
module: test/extract/middle
|
||||
merge:
|
||||
module: test/merge
|
||||
normalize:
|
||||
module: test/normalize
|
||||
- id: third
|
||||
references:
|
||||
upstream:
|
||||
artifact:
|
||||
step: second
|
||||
lane: middle
|
||||
artifacts:
|
||||
dependent:
|
||||
extract:
|
||||
module: test/extract/dependent
|
||||
merge:
|
||||
module: test/merge
|
||||
normalize:
|
||||
module: test/normalize
|
||||
output:
|
||||
module: test/recompute-output
|
||||
`, roots.output, roots.plans, roots.checkpoints, roots.debug)
|
||||
if err := os.WriteFile(roots.config, []byte(config), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return roots
|
||||
}
|
||||
|
||||
func runRecomputeCommand(roots stateTestRoots, opts Options, recompute bool) stateTestResult {
|
||||
args := []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}
|
||||
if recompute {
|
||||
args = append(args, "--resume", "--recompute-step", "second", "--debug")
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
return stateTestResult{code: RunWithOptions(args, &stdout, &stderr, opts), stdout: stdout.String(), stderr: stderr.String()}
|
||||
}
|
||||
|
||||
func removeCheckpointLaneStage(t *testing.T, root, stage, step, lane string) {
|
||||
t.Helper()
|
||||
dir := filepath.Dir(findCheckpointFile(t, root, stage, step, lane, "manifest.json"))
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func findCheckpointFile(t *testing.T, root, stage, step, lane, name string) string {
|
||||
t.Helper()
|
||||
want := filepath.Join(stage, step, lane, name)
|
||||
var matches []string
|
||||
err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !entry.IsDir() && strings.HasSuffix(path, want) {
|
||||
matches = append(matches, path)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(matches) != 1 {
|
||||
t.Fatalf("checkpoint files ending in %q = %v", want, matches)
|
||||
}
|
||||
return matches[0]
|
||||
}
|
||||
|
||||
func latestChildDir(t *testing.T, root string) string {
|
||||
t.Helper()
|
||||
entries, err := os.ReadDir(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var dirs []string
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
dirs = append(dirs, filepath.Join(root, entry.Name()))
|
||||
}
|
||||
}
|
||||
if len(dirs) == 0 {
|
||||
t.Fatal("no child directory")
|
||||
}
|
||||
sort.Strings(dirs)
|
||||
return dirs[len(dirs)-1]
|
||||
}
|
||||
|
||||
func readLatestCheckpointEvents(t *testing.T, root string) []pipeline.CheckpointEvent {
|
||||
t.Helper()
|
||||
var events []pipeline.CheckpointEvent
|
||||
data, err := os.ReadFile(filepath.Join(latestChildDir(t, root), "summary", "checkpoint-events.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := json.Unmarshal(data, &events); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return events
|
||||
}
|
||||
@@ -1,56 +0,0 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
func TestRecomputePolicyIncludesDependentLanesAndReusablePredecessors(t *testing.T) {
|
||||
producer := pipeline.ResolvedArtifactLane{StepID: "first", ID: "producer"}
|
||||
unrelated := pipeline.ResolvedArtifactLane{StepID: "first", ID: "unrelated"}
|
||||
consumer := pipeline.ResolvedArtifactLane{
|
||||
StepID: "second", ID: "consumer",
|
||||
ExtractReferences: pipeline.ResolvedReferenceTarget{Bindings: []pipeline.ReferenceBinding{{Artifact: &pipeline.ArtifactReference{Step: "first", Lane: "producer"}}}},
|
||||
}
|
||||
downstream := pipeline.ResolvedArtifactLane{
|
||||
StepID: "third", ID: "downstream",
|
||||
ExtractReferences: pipeline.ResolvedReferenceTarget{Bindings: []pipeline.ReferenceBinding{{Artifact: &pipeline.ArtifactReference{Step: "second", Lane: "consumer"}}}},
|
||||
}
|
||||
independent := pipeline.ResolvedArtifactLane{StepID: "third", ID: "independent"}
|
||||
resolved := pipeline.ResolvedPipeline{Steps: []pipeline.ResolvedPipelineStep{
|
||||
{ID: "first", ArtifactLanes: []pipeline.ResolvedArtifactLane{producer, unrelated}},
|
||||
{ID: "second", ArtifactLanes: []pipeline.ResolvedArtifactLane{consumer}},
|
||||
{ID: "third", ArtifactLanes: []pipeline.ResolvedArtifactLane{downstream, independent}},
|
||||
}}
|
||||
|
||||
policy, err := recomputePolicy(resolved, "second")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := policy.ForcedLanes[pipeline.CheckpointLaneKey("second", "consumer")]; !ok {
|
||||
t.Fatal("selected lane was not forced")
|
||||
}
|
||||
if _, ok := policy.ForcedLanes[pipeline.CheckpointLaneKey("third", "downstream")]; !ok {
|
||||
t.Fatal("transitive dependent lane was not forced")
|
||||
}
|
||||
if _, ok := policy.ForcedLanes[pipeline.CheckpointLaneKey("first", "producer")]; ok {
|
||||
t.Fatal("predecessor was implicitly forced")
|
||||
}
|
||||
if _, ok := policy.RequireReusableLanes[pipeline.CheckpointLaneKey("first", "producer")]; !ok {
|
||||
t.Fatal("required predecessor was not marked reusable")
|
||||
}
|
||||
if _, ok := policy.ForcedLanes[pipeline.CheckpointLaneKey("first", "unrelated")]; ok {
|
||||
t.Fatal("unrelated lane was forced")
|
||||
}
|
||||
if _, ok := policy.ForcedLanes[pipeline.CheckpointLaneKey("third", "independent")]; ok {
|
||||
t.Fatal("unrelated later lane was forced")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecomputePolicyRejectsUnknownStep(t *testing.T) {
|
||||
_, err := recomputePolicy(pipeline.ResolvedPipeline{Steps: []pipeline.ResolvedPipelineStep{{ID: "known"}}}, "missing")
|
||||
if err == nil {
|
||||
t.Fatal("unknown step was accepted")
|
||||
}
|
||||
}
|
||||
@@ -20,8 +20,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/fileio"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/checkpoint"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkplan"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/artifactcache"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
frameworkdebug "gitea.maximumdirect.net/eric/notarius/internal/framework/debug"
|
||||
frameworkllm "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
@@ -32,19 +31,18 @@ import (
|
||||
)
|
||||
|
||||
type Options struct {
|
||||
LLMClient promptkit.LLMClient
|
||||
LLMRuntime promptkit.RuntimeConfig
|
||||
Catalog pipeline.ModuleCatalog
|
||||
Registries pipeline.Registries
|
||||
LLMClientFactory LLMClientFactory
|
||||
RunIDGenerator RunIDGenerator
|
||||
LookupEnv func(string) (string, bool)
|
||||
Now func() time.Time
|
||||
UserCacheDir func() (string, error)
|
||||
ChunkPlanStoreFactory pipeline.ChunkPlanStoreFactory
|
||||
DebugRecorderFactory func(string) (pipeline.DebugRecorder, error)
|
||||
DebugTerminalFactory func(*debugbundle.SummaryWriter) DebugTerminalWriter
|
||||
promptKitAssets *frameworkllm.AssetRegistry
|
||||
LLMClient promptkit.LLMClient
|
||||
LLMRuntime promptkit.RuntimeConfig
|
||||
Catalog pipeline.ModuleCatalog
|
||||
Registries pipeline.Registries
|
||||
LLMClientFactory LLMClientFactory
|
||||
RunIDGenerator RunIDGenerator
|
||||
LookupEnv func(string) (string, bool)
|
||||
Now func() time.Time
|
||||
UserCacheDir func() (string, error)
|
||||
DebugRecorderFactory func(string) (pipeline.DebugRecorder, error)
|
||||
DebugTerminalFactory func(*debugbundle.SummaryWriter) DebugTerminalWriter
|
||||
promptKitAssets *frameworkllm.AssetRegistry
|
||||
}
|
||||
|
||||
type LLMRuntimeOverrides struct {
|
||||
@@ -85,9 +83,6 @@ func normalizeOptions(opts Options) (Options, error) {
|
||||
if opts.UserCacheDir == nil {
|
||||
opts.UserCacheDir = os.UserCacheDir
|
||||
}
|
||||
if opts.ChunkPlanStoreFactory == nil {
|
||||
opts.ChunkPlanStoreFactory = chunkplan.NewFilesystemStore
|
||||
}
|
||||
if opts.DebugRecorderFactory == nil {
|
||||
opts.DebugRecorderFactory = frameworkdebug.NewFilesystemRecorder
|
||||
}
|
||||
@@ -130,19 +125,14 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
llmProfile := singleValueFlag{name: "--llm-profile"}
|
||||
reasoningEffort := singleValueFlag{name: "--reasoning-effort"}
|
||||
clearReasoningEffort := fs.Bool("clear-reasoning-effort", false, "clear the LLM profile reasoning effort")
|
||||
resume := fs.Bool("resume", false, "reuse compatible recorded checkpoints")
|
||||
recomputeStep := singleValueFlag{name: "--recompute-step"}
|
||||
chunkCache := chunkCacheFlag{}
|
||||
requestedSessionID := sessionIDFlag{}
|
||||
referenceFlags := stringListFlag{}
|
||||
withoutReferenceFlags := stringListFlag{}
|
||||
fs.Var(&requestedSessionID, "session-id", "prompt session identifier")
|
||||
fs.Var(&llmProfile, "llm-profile", "LLM profile override")
|
||||
fs.Var(&reasoningEffort, "reasoning-effort", "reasoning effort override")
|
||||
fs.Var(&chunkCache, "chunk_cache", "chunk plan cache mode: auto, bypass, or refresh")
|
||||
fs.Var(&referenceFlags, "reference", "reference binding, as slot=path, chunk.slot=path, lane.slot=path, lane.extract.slot=path, lane.merge.slot=path, or lane.normalize.slot=path")
|
||||
fs.Var(&withoutReferenceFlags, "without-reference", "unbind a reference, using the same selector forms as --reference")
|
||||
fs.Var(&recomputeStep, "recompute-step", "recompute one ordered pipeline step and dependent lanes")
|
||||
if err := validateRunFlagValues(args); err != nil {
|
||||
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||
return 2
|
||||
@@ -209,14 +199,6 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||
return 2
|
||||
}
|
||||
if recomputeStep.set && strings.TrimSpace(recomputeStep.value) == "" {
|
||||
fmt.Fprintln(stderr, "notarius: --recompute-step must not be empty")
|
||||
return 2
|
||||
}
|
||||
if recomputeStep.set && len(only) > 0 {
|
||||
fmt.Fprintln(stderr, "notarius: --recompute-step cannot be combined with --only")
|
||||
return 2
|
||||
}
|
||||
referenceRequests, err := parseReferenceFlags(referenceFlags)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||
@@ -228,7 +210,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
return 2
|
||||
}
|
||||
|
||||
loaded, loadedConfigPath, err := loadConfigKit(*configPath, *configOverlay, runConfigurationOverrides(chunkCache, *outputDir, *debugDir), opts)
|
||||
loaded, loadedConfigPath, err := loadConfigKit(*configPath, *configOverlay, runConfigurationOverrides(*outputDir, *debugDir), opts)
|
||||
if err != nil {
|
||||
writeConfigLoadError(stderr, err)
|
||||
return 1
|
||||
@@ -238,19 +220,6 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
if *resume && !cfg.Cache.Checkpoints.Enabled {
|
||||
fmt.Fprintln(stderr, "notarius: --resume requires cache.checkpoints.enabled: true")
|
||||
return 1
|
||||
}
|
||||
if recomputeStep.set && !*resume {
|
||||
fmt.Fprintln(stderr, "notarius: --recompute-step requires --resume")
|
||||
return 2
|
||||
}
|
||||
if recomputeStep.set && !cfg.Cache.Checkpoints.Enabled {
|
||||
fmt.Fprintln(stderr, "notarius: --recompute-step requires cache.checkpoints.enabled: true")
|
||||
return 1
|
||||
}
|
||||
|
||||
startedAt := opts.Now().UTC()
|
||||
runID, err := opts.RunIDGenerator(startedAt)
|
||||
if err != nil {
|
||||
@@ -293,10 +262,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
ConfigSource: configSource(*configPath),
|
||||
ConfigOverlay: effectiveConfigOverlay(*configOverlay),
|
||||
OnlyLanes: append([]string(nil), only...),
|
||||
ChunkCacheOverride: chunkCache.explicitValue(),
|
||||
ReasoningEffortOverride: runtimeOverrides.ReasoningEffort,
|
||||
Resume: *resume,
|
||||
RecomputeStep: strings.TrimSpace(recomputeStep.value),
|
||||
RunID: runID,
|
||||
StartedAt: startedAt,
|
||||
}
|
||||
@@ -339,13 +305,6 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, err)
|
||||
}
|
||||
effective.ResolvedPipeline = materialized
|
||||
checkpointPolicy := pipeline.CheckpointExecutionPolicy{}
|
||||
if recomputeStep.set {
|
||||
checkpointPolicy, err = recomputePolicy(effective.ResolvedPipeline, recomputeStep.value)
|
||||
if err != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, err)
|
||||
}
|
||||
}
|
||||
invocation.PipelineDigest = effective.ResolvedPipeline.Digest
|
||||
if err := writeSummary(summary, func() error { return summary.WriteInvocation(invocation) }); err != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug invocation metadata: %w", err))
|
||||
@@ -376,13 +335,6 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
if err != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("create LLM client for profile %q: %w", factoryProfileID, err))
|
||||
}
|
||||
var llmFingerprints []checkpoint.Fingerprint
|
||||
if effective.Config.Cache.Checkpoints.Enabled {
|
||||
llmFingerprints, err = llmCheckpointFingerprints(llmClient)
|
||||
if err != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("prepare LLM checkpoint identity: %w", err))
|
||||
}
|
||||
}
|
||||
llmClient = pipeline.WithDebugLLMRecording(llmClient, debugRecorder)
|
||||
prepared, err := pipeline.Prepare(effective.ResolvedPipeline, registries, pipeline.ModuleDependencies{LLM: llmClient})
|
||||
if err != nil {
|
||||
@@ -400,32 +352,24 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
if err := writeSummary(summary, func() error { return summary.WriteInvocation(invocation) }); err != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug invocation metadata: %w", err))
|
||||
}
|
||||
chunkPlans, err := chunkPlanStoreForRun(effective.Config.Cache.ChunkPlans, opts)
|
||||
if err != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, err)
|
||||
}
|
||||
checkpointRecorder, checkpointLoader, err := checkpointHandlersForRun(effective.Config.Cache.Checkpoints, opts, effective.ResolvedPipeline, prepared.CheckpointFingerprints(), llmFingerprints, rawInput, only, llmProfiles, strings.TrimSpace(llmProfile.value), effectiveSessionID, runtimeOverrides, *resume)
|
||||
cache, err := artifactcache.NewFilesystemStore(effective.Config.CacheDirectory)
|
||||
if err != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, err)
|
||||
}
|
||||
|
||||
output, err := pipeline.New().Run(ctx, pipeline.RunInput{
|
||||
Prepared: prepared,
|
||||
Path: strings.TrimSpace(*inputPath),
|
||||
RawInput: rawInput,
|
||||
SessionID: effectiveSessionID,
|
||||
RunID: runID,
|
||||
StartedAt: startedAt,
|
||||
LLMProfiles: llmProfiles,
|
||||
Metadata: runMetadata(effective.Config.Output.Directory, debugPath),
|
||||
Diagnostics: referenceDiagnostics,
|
||||
ChunkCacheMode: effective.Config.Cache.ChunkPlans.Mode,
|
||||
ChunkPlans: chunkPlans,
|
||||
Checkpoints: checkpointRecorder,
|
||||
Checkpoint: checkpointLoader,
|
||||
CheckpointPolicy: checkpointPolicy,
|
||||
Debug: debugRecorder,
|
||||
Workers: cfg.Concurrency.Workers,
|
||||
Prepared: prepared,
|
||||
Path: strings.TrimSpace(*inputPath),
|
||||
RawInput: rawInput,
|
||||
SessionID: effectiveSessionID,
|
||||
RunID: runID,
|
||||
StartedAt: startedAt,
|
||||
LLMProfiles: llmProfiles,
|
||||
Metadata: runMetadata(effective.Config.Output.Directory, debugPath),
|
||||
Diagnostics: referenceDiagnostics,
|
||||
Cache: cache,
|
||||
Debug: debugRecorder,
|
||||
Workers: cfg.Concurrency.Workers,
|
||||
})
|
||||
diagnosticProjection, diagnosticErr := contracts.ProjectDiagnosticCollection(output.Diagnostics)
|
||||
if diagnosticErr == nil {
|
||||
@@ -511,196 +455,10 @@ func writePartialSummary(summary *debugbundle.SummaryWriter, output pipeline.Run
|
||||
if err := summary.WriteRunManifest(output.Manifest); err != nil {
|
||||
return err
|
||||
}
|
||||
if output.ChunkPlan != nil {
|
||||
if err := summary.WriteChunkPlan(*output.ChunkPlan); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := summary.WriteDiagnostics(output.Diagnostics); err != nil {
|
||||
return err
|
||||
}
|
||||
return summary.WriteCheckpointEvents(output.CheckpointEvents)
|
||||
}
|
||||
|
||||
func checkpointHandlersForRun(
|
||||
settings config.CheckpointCacheConfig,
|
||||
opts Options,
|
||||
resolved pipeline.ResolvedPipeline,
|
||||
componentFingerprints []pipeline.CheckpointFingerprint,
|
||||
llmFingerprints []checkpoint.Fingerprint,
|
||||
rawInput []byte,
|
||||
only []string,
|
||||
llmProfiles []artifacts.LLMProfileManifest,
|
||||
llmProfileOverride string,
|
||||
sessionID string,
|
||||
runtimeOverrides LLMRuntimeOverrides,
|
||||
resume bool,
|
||||
) (pipeline.CheckpointRecorder, pipeline.CheckpointLoader, error) {
|
||||
if !settings.Enabled {
|
||||
if resume {
|
||||
return nil, nil, fmt.Errorf("--resume requires cache.checkpoints.enabled: true")
|
||||
}
|
||||
return pipeline.NoopCheckpointRecorder(), pipeline.NoopCheckpointLoader(), nil
|
||||
}
|
||||
identity, err := checkpoint.NewIdentity(checkpoint.IdentityInput{
|
||||
Pipeline: resolved,
|
||||
InputKey: resolved.Input.Module,
|
||||
RawInputDigest: rawInputDigest(rawInput),
|
||||
SelectedLanes: only,
|
||||
RuntimeOverrides: runtimeOverrideFingerprints(llmProfileOverride, sessionID, runtimeOverrides),
|
||||
References: pipeline.ReferenceProvenance(resolved),
|
||||
ProvenanceFingerprints: combineCheckpointFingerprints(
|
||||
llmProfileFingerprints(llmProfiles),
|
||||
llmFingerprints,
|
||||
checkpointIdentityFingerprints(componentFingerprints),
|
||||
),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("create checkpoint identity: %w", err)
|
||||
}
|
||||
checkpointRoot := strings.TrimSpace(settings.Directory)
|
||||
if checkpointRoot == "" {
|
||||
checkpointRoot, err = config.DefaultCheckpointRoot(opts.UserCacheDir)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("resolve checkpoint root: %w", err)
|
||||
}
|
||||
}
|
||||
recorder, err := checkpoint.NewFilesystemRecorder(checkpointRoot, identity)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("create checkpoint recorder: %w", err)
|
||||
}
|
||||
loader := pipeline.NoopCheckpointLoader()
|
||||
if resume {
|
||||
loader, err = checkpoint.NewFilesystemLoader(checkpointRoot, identity)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("create checkpoint loader: %w", err)
|
||||
}
|
||||
}
|
||||
return recorder, loader, nil
|
||||
}
|
||||
|
||||
func llmCheckpointFingerprints(client contracts.StructuredLLMClient) ([]checkpoint.Fingerprint, error) {
|
||||
provider, ok := client.(frameworkllm.CheckpointFingerprintProvider)
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
values, err := provider.LLMCheckpointFingerprints()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]checkpoint.Fingerprint, 0, len(values))
|
||||
for _, value := range values {
|
||||
out = append(out, checkpoint.Fingerprint{Name: value.Name, Value: value.Value})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func combineCheckpointFingerprints(sources ...[]checkpoint.Fingerprint) []checkpoint.Fingerprint {
|
||||
var out []checkpoint.Fingerprint
|
||||
for _, source := range sources {
|
||||
out = append(out, source...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func recomputePolicy(resolved pipeline.ResolvedPipeline, requestedStep string) (pipeline.CheckpointExecutionPolicy, error) {
|
||||
requestedStep = strings.TrimSpace(requestedStep)
|
||||
if requestedStep == "" {
|
||||
return pipeline.CheckpointExecutionPolicy{}, fmt.Errorf("--recompute-step must not be empty")
|
||||
}
|
||||
var selected *pipeline.ResolvedPipelineStep
|
||||
for index := range resolved.Steps {
|
||||
if strings.TrimSpace(resolved.Steps[index].ID) == requestedStep {
|
||||
selected = &resolved.Steps[index]
|
||||
break
|
||||
}
|
||||
}
|
||||
if selected == nil {
|
||||
return pipeline.CheckpointExecutionPolicy{}, fmt.Errorf("unknown pipeline step %q", requestedStep)
|
||||
}
|
||||
|
||||
policy := pipeline.CheckpointExecutionPolicy{ForcedLanes: make(map[string]struct{}), RequireReusableLanes: make(map[string]struct{})}
|
||||
var forced []pipeline.ResolvedArtifactLane
|
||||
for _, lane := range selected.ArtifactLanes {
|
||||
lane.StepID = selected.ID
|
||||
policy.ForcedLanes[pipeline.CheckpointLaneKey(selected.ID, lane.ID)] = struct{}{}
|
||||
forced = append(forced, lane)
|
||||
}
|
||||
|
||||
lanes := make(map[string]pipeline.ResolvedArtifactLane)
|
||||
dependents := make(map[string][]pipeline.ResolvedArtifactLane)
|
||||
for _, step := range resolved.Steps {
|
||||
for _, lane := range step.ArtifactLanes {
|
||||
lane.StepID = step.ID
|
||||
lanes[pipeline.CheckpointLaneKey(step.ID, lane.ID)] = lane
|
||||
for _, target := range []pipeline.ResolvedReferenceTarget{lane.ExtractReferences, lane.MergeReferences, lane.NormalizeReferences} {
|
||||
for _, binding := range target.Bindings {
|
||||
if binding.Artifact != nil {
|
||||
producerKey := pipeline.CheckpointLaneKey(binding.Artifact.Step, binding.Artifact.Lane)
|
||||
dependents[producerKey] = append(dependents[producerKey], lane)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
processed := make(map[string]struct{})
|
||||
queue := append([]pipeline.ResolvedArtifactLane(nil), forced...)
|
||||
for len(queue) > 0 {
|
||||
lane := queue[0]
|
||||
queue = queue[1:]
|
||||
key := pipeline.CheckpointLaneKey(lane.StepID, lane.ID)
|
||||
if _, ok := processed[key]; ok {
|
||||
continue
|
||||
}
|
||||
processed[key] = struct{}{}
|
||||
for _, dependent := range dependents[key] {
|
||||
dependentKey := pipeline.CheckpointLaneKey(dependent.StepID, dependent.ID)
|
||||
if _, alreadyForced := policy.ForcedLanes[dependentKey]; alreadyForced {
|
||||
continue
|
||||
}
|
||||
policy.ForcedLanes[dependentKey] = struct{}{}
|
||||
forced = append(forced, dependent)
|
||||
queue = append(queue, dependent)
|
||||
}
|
||||
}
|
||||
|
||||
processed = make(map[string]struct{})
|
||||
queue = append([]pipeline.ResolvedArtifactLane(nil), forced...)
|
||||
for len(queue) > 0 {
|
||||
lane := queue[0]
|
||||
queue = queue[1:]
|
||||
key := pipeline.CheckpointLaneKey(lane.StepID, lane.ID)
|
||||
if _, ok := processed[key]; ok {
|
||||
continue
|
||||
}
|
||||
processed[key] = struct{}{}
|
||||
for _, target := range []pipeline.ResolvedReferenceTarget{lane.ExtractReferences, lane.MergeReferences, lane.NormalizeReferences} {
|
||||
for _, binding := range target.Bindings {
|
||||
if binding.Artifact == nil {
|
||||
continue
|
||||
}
|
||||
producerKey := pipeline.CheckpointLaneKey(binding.Artifact.Step, binding.Artifact.Lane)
|
||||
if _, forcedAlready := policy.ForcedLanes[producerKey]; !forcedAlready {
|
||||
policy.RequireReusableLanes[producerKey] = struct{}{}
|
||||
}
|
||||
if producer, ok := lanes[producerKey]; ok {
|
||||
queue = append(queue, producer)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return policy, nil
|
||||
}
|
||||
|
||||
func checkpointIdentityFingerprints(values []pipeline.CheckpointFingerprint) []checkpoint.Fingerprint {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]checkpoint.Fingerprint, len(values))
|
||||
for index, value := range values {
|
||||
out[index] = checkpoint.Fingerprint{Name: "component:" + value.Name, Value: value.Value}
|
||||
}
|
||||
return out
|
||||
return nil
|
||||
}
|
||||
|
||||
func rawInputDigest(data []byte) string {
|
||||
@@ -708,42 +466,6 @@ func rawInputDigest(data []byte) string {
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func runtimeOverrideFingerprints(llmProfileOverride string, sessionID string, runtimeOverrides LLMRuntimeOverrides) []checkpoint.Fingerprint {
|
||||
var values []checkpoint.Fingerprint
|
||||
if strings.TrimSpace(llmProfileOverride) != "" {
|
||||
values = append(values, checkpoint.Fingerprint{Name: "llm_profile_override", Value: strings.TrimSpace(llmProfileOverride)})
|
||||
}
|
||||
if strings.TrimSpace(sessionID) != "" {
|
||||
values = append(values, checkpoint.Fingerprint{Name: "session_id", Value: strings.TrimSpace(sessionID)})
|
||||
}
|
||||
if runtimeOverrides.ReasoningEffort != nil {
|
||||
value := strings.TrimSpace(*runtimeOverrides.ReasoningEffort)
|
||||
if value == "" {
|
||||
value = "<cleared>"
|
||||
}
|
||||
values = append(values, checkpoint.Fingerprint{Name: "reasoning_effort_override", Value: value})
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
func llmProfileFingerprints(profiles []artifacts.LLMProfileManifest) []checkpoint.Fingerprint {
|
||||
if len(profiles) == 0 {
|
||||
return nil
|
||||
}
|
||||
values := make([]checkpoint.Fingerprint, 0, len(profiles))
|
||||
for _, profile := range profiles {
|
||||
id := strings.TrimSpace(profile.ID)
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
values = append(values, checkpoint.Fingerprint{
|
||||
Name: "llm_profile:" + id,
|
||||
Value: strings.TrimSpace(profile.Provider) + ":" + strings.TrimSpace(profile.Model),
|
||||
})
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
func configSource(configPath string) string {
|
||||
if strings.TrimSpace(configPath) != "" {
|
||||
return "flag"
|
||||
@@ -837,22 +559,15 @@ func reorderRunArgs(args []string) []string {
|
||||
|
||||
func runFlagTakesValue(arg string) bool {
|
||||
switch arg {
|
||||
case "--config", "--config-overlay", "--input", "--only", "--output-dir", "--debug-dir", "--llm-profile", "--session-id", "--reasoning-effort", "--chunk_cache", "--reference", "--without-reference", "--recompute-step":
|
||||
case "--config", "--config-overlay", "--input", "--only", "--output-dir", "--debug-dir", "--llm-profile", "--session-id", "--reasoning-effort", "--reference", "--without-reference":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func runConfigurationOverrides(chunkCache chunkCacheFlag, outputDir, debugDir string) []configkit.Override {
|
||||
func runConfigurationOverrides(outputDir, debugDir string) []configkit.Override {
|
||||
overrides := make([]configkit.Override, 0, 3)
|
||||
if chunkCache.set {
|
||||
overrides = append(overrides, configkit.Override{
|
||||
Path: configkit.Keys("cache", "chunk_plans", "mode"),
|
||||
Value: string(chunkCache.value),
|
||||
Flag: "--chunk_cache",
|
||||
})
|
||||
}
|
||||
if outputDir = strings.TrimSpace(outputDir); outputDir != "" {
|
||||
overrides = append(overrides, configkit.Override{
|
||||
Path: configkit.Keys("output", "directory"),
|
||||
@@ -870,57 +585,6 @@ func runConfigurationOverrides(chunkCache chunkCacheFlag, outputDir, debugDir st
|
||||
return overrides
|
||||
}
|
||||
|
||||
type chunkCacheFlag struct {
|
||||
value pipeline.ChunkCacheMode
|
||||
set bool
|
||||
}
|
||||
|
||||
func (f *chunkCacheFlag) String() string {
|
||||
if f == nil {
|
||||
return ""
|
||||
}
|
||||
return string(f.value)
|
||||
}
|
||||
|
||||
func (f *chunkCacheFlag) Set(raw string) error {
|
||||
mode, err := pipeline.ParseChunkCacheMode(raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f.value = mode
|
||||
f.set = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f chunkCacheFlag) explicitValue() string {
|
||||
if !f.set {
|
||||
return ""
|
||||
}
|
||||
return string(f.value)
|
||||
}
|
||||
|
||||
func chunkPlanStoreForRun(cfg config.ChunkPlanCacheConfig, opts Options) (pipeline.ChunkPlanStore, error) {
|
||||
if cfg.Mode == pipeline.ChunkCacheBypass {
|
||||
return nil, nil
|
||||
}
|
||||
root := strings.TrimSpace(cfg.Directory)
|
||||
if root == "" {
|
||||
var err error
|
||||
root, err = config.DefaultChunkPlanRoot(opts.UserCacheDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve chunk plan root: %w", err)
|
||||
}
|
||||
}
|
||||
store, err := opts.ChunkPlanStoreFactory(root)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create chunk plan store at %q: %w", root, err)
|
||||
}
|
||||
if store == nil {
|
||||
return nil, fmt.Errorf("create chunk plan store at %q: factory returned nil", root)
|
||||
}
|
||||
return store, nil
|
||||
}
|
||||
|
||||
func validateRunFlagValues(args []string) error {
|
||||
for i, arg := range args {
|
||||
if arg != "--config-overlay" && arg != "--session-id" && arg != "--reasoning-effort" {
|
||||
|
||||
Reference in New Issue
Block a user