Unify chunk-plan caching and remove checkpoint execution paths

This commit is contained in:
2026-11-22 02:24:45 +00:00
parent 36906cb3d1
commit 3b1b4c9bcf
26 changed files with 717 additions and 4258 deletions

View File

@@ -1288,6 +1288,29 @@ native prepared details retained on a hit without an execution event.
## Stage 28: Unify Chunk-Plan Reuse And Remove Checkpoint Execution Paths
**Status:** Complete. Native chunk producers now use the same accepted-result
cache as typed artifacts, storing only the final domain plan under the initial
native message-content hash after materialization and complete assessment.
Hits decode and rematerialize against the current source without semantic
reassessment; unusable entries miss without entering conversation history.
Deterministic chunkers remain uncached. Removed source-digest plan lookup,
checkpoint collaborators/transitions, dependency fingerprint collection,
required-lane hydration, resume barriers, and recomputation policy from runner,
preparation, handoffs, and CLI cache composition. The CLI cache collaborator now
uses the resolved unified root; full native run wiring remains Stage 33.
Replaced canonical-partition/checkpoint tests with native-provider/filesystem
coverage for final correction, content/reference changes, settings-only reuse,
current-source rematerialization, unusable ranges/data, cache modes, terminal
exclusions, failed refresh, and unchanged legacy files. Retained current chunk-map
handoff/schema and deterministic provenance checks. Focused pipeline tests/vet/
repeated race checks, cache/contracts/source/chunk-map/generic/Seriatim/
reconciliation tests, production pipeline/cache/config builds, and
`git diff --check` pass. Full pipeline tests still stop at retired fixtures
(including checkpoint helpers and NormalizeRetry); CLI setup still stops at old
ConfigKit's removed YAML plugin/limit import. Those suites are not claimed
passing. Stage 35 owns remaining obsolete stores, module fingerprint declarations,
and fixture cleanup; final assembled verification remains required.
**Goal:** LLM chunk plans use the same accepted-result cache and no second store.
**Inspect/change:** `runner_chunk_plan.go`, `chunk_cache.go`, `chunk_plan_store.go`,

View File

@@ -3,331 +3,11 @@ package cli
import (
"bytes"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkplan"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
func TestRunChunkPlanModePrecedenceAndValidation(t *testing.T) {
tests := []struct {
name string
fileMode string
envMode string
cliMode string
wantStores int
}{
{name: "default", wantStores: 1},
{name: "file", fileMode: "bypass"},
{name: "retired environment has no effect", envMode: "bypass", wantStores: 1},
{name: "cli", envMode: "refresh", cliMode: "bypass"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
roots := newStateTestRoots(t)
if tt.name == "default" {
removeStateTestConfigLine(t, roots.config, " mode: auto\n")
} else if tt.fileMode != "" {
replaceStateTestConfigLine(t, roots.config, " mode: auto\n", " mode: "+tt.fileMode+"\n")
}
var stores []string
opts := newStateTestHarness().options()
opts.LookupEnv = func(name string) (string, bool) {
if name == "NOTARIUS_CACHE_CHUNK_PLANS_MODE" && tt.envMode != "" {
return tt.envMode, true
}
return "", false
}
opts.ChunkPlanStoreFactory = func(root string) (pipeline.ChunkPlanStore, error) {
stores = append(stores, root)
return chunkplan.NewFilesystemStore(root)
}
args := []string{"run", "sample", "--config", roots.config, "--input", roots.input}
if tt.cliMode != "" {
args = append(args, "--chunk_cache", tt.cliMode)
}
var stdout, stderr bytes.Buffer
if code := RunWithOptions(args, &stdout, &stderr, opts); code != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
assertStateTestOutput(t, roots.output)
if len(stores) != tt.wantStores {
t.Fatalf("chunk plan store roots = %v, want %d stores", stores, tt.wantStores)
}
if tt.wantStores == 1 && stores[0] != roots.plans {
t.Fatalf("chunk plan store root = %q, want %q", stores[0], roots.plans)
}
if tt.wantStores == 0 {
assertAbsent(t, roots.plans)
}
})
}
t.Run("invalid cli syntax is a usage error", func(t *testing.T) {
roots := newStateTestRoots(t)
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "invalid"}, &stdout, &stderr, newStateTestHarness().options())
if code != 2 || stdout.Len() != 0 || stderr.Len() == 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
assertNoRunState(t, roots)
})
for _, tt := range []struct {
name string
}{
{name: "invalid file mode"},
} {
t.Run(tt.name, func(t *testing.T) {
roots := newStateTestRoots(t)
opts := newStateTestHarness().options()
replaceStateTestConfigLine(t, roots.config, " mode: auto\n", " mode: invalid\n")
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input}, &stdout, &stderr, opts)
if code != 1 || stdout.Len() != 0 || stderr.Len() == 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
assertNoRunState(t, roots)
})
}
}
func TestRunChunkPlanRootSelectionAndFailures(t *testing.T) {
t.Run("empty configured root uses the per-user cache root", func(t *testing.T) {
roots := newStateTestRoots(t)
removeStateTestConfigLine(t, roots.config, fmt.Sprintf(" directory: %q\n", roots.plans))
userCache := filepath.Join(t.TempDir(), "user-cache")
var stores []string
opts := newStateTestHarness().options()
opts.UserCacheDir = func() (string, error) { return userCache, nil }
opts.ChunkPlanStoreFactory = func(root string) (pipeline.ChunkPlanStore, error) {
stores = append(stores, root)
return chunkplan.NewFilesystemStore(root)
}
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input}, &stdout, &stderr, opts)
if code != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
wantRoot := filepath.Join(userCache, "notarius", "chunk-plans")
if len(stores) != 1 || stores[0] != wantRoot {
t.Fatalf("chunk plan store roots = %v, want [%q]", stores, wantRoot)
}
assertFile(t, filepath.Join(wantRoot, strings.TrimPrefix(stateTestDigest, "sha256:"), "plan.json"))
assertAbsent(t, roots.plans)
})
t.Run("bypass avoids default cache dependencies", func(t *testing.T) {
roots := newStateTestRoots(t)
removeStateTestConfigLine(t, roots.config, fmt.Sprintf(" directory: %q\n", roots.plans))
userCacheCalls := 0
storeCalls := 0
opts := newStateTestHarness().options()
opts.UserCacheDir = func() (string, error) {
userCacheCalls++
return "", errors.New("user cache must not be resolved")
}
opts.ChunkPlanStoreFactory = func(string) (pipeline.ChunkPlanStore, error) {
storeCalls++
return nil, errors.New("chunk plan store must not be constructed")
}
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}, &stdout, &stderr, opts)
if code != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
if userCacheCalls != 0 || storeCalls != 0 {
t.Fatalf("user cache calls=%d store calls=%d, want none", userCacheCalls, storeCalls)
}
assertStateTestOutput(t, roots.output)
assertAbsent(t, roots.plans)
})
t.Run("user cache resolution failure has context and no output", func(t *testing.T) {
roots := newStateTestRoots(t)
removeStateTestConfigLine(t, roots.config, fmt.Sprintf(" directory: %q\n", roots.plans))
opts := newStateTestHarness().options()
opts.UserCacheDir = func() (string, error) { return "", errors.New("cache home unavailable") }
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input}, &stdout, &stderr, opts)
if code != 1 || !strings.Contains(stderr.String(), "resolve chunk plan root") || !strings.Contains(stderr.String(), "cache home unavailable") {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
assertNoRunState(t, roots)
})
t.Run("store construction failure has context and no output", func(t *testing.T) {
roots := newStateTestRoots(t)
opts := newStateTestHarness().options()
opts.ChunkPlanStoreFactory = func(root string) (pipeline.ChunkPlanStore, error) {
return nil, fmt.Errorf("store unavailable")
}
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input}, &stdout, &stderr, opts)
want := fmt.Sprintf("create chunk plan store at %q", roots.plans)
if code != 1 || !strings.Contains(stderr.String(), want) || !strings.Contains(stderr.String(), "store unavailable") {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
assertNoRunState(t, roots)
})
t.Run("checkpoint root resolution failure has context and no output", func(t *testing.T) {
roots := newStateTestRoots(t)
removeStateTestConfigLine(t, roots.config, fmt.Sprintf(" directory: %q\n", roots.checkpoints))
opts := newStateTestHarness().options()
opts.UserCacheDir = func() (string, error) { return "", errors.New("checkpoint cache unavailable") }
result := runStateTest(t, roots, opts, false, true, "bypass")
if result.code != 1 || !strings.Contains(result.stderr, "resolve checkpoint root") || !strings.Contains(result.stderr, "checkpoint cache unavailable") {
t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr)
}
assertNoRunState(t, roots)
})
}
func TestRunAutoReusesPlanWhenRunInputsChange(t *testing.T) {
roots := newStateTestRoots(t)
data, err := os.ReadFile(roots.config)
if err != nil {
t.Fatal(err)
}
configText := replaceRequiredOnce(t, string(data), " chunk:\n module: test/chunk\n", ` chunk:
module: test/chunk
options:
strategy: first
`)
configText = replaceRequiredOnce(t, configText, " output:\n module: test/output\n", ` other:
extract:
module: test/extract
merge:
module: test/merge
normalize:
module: test/normalize
output:
module: test/output
`)
if err := os.WriteFile(roots.config, []byte(configText), 0o600); err != nil {
t.Fatal(err)
}
referencePath := filepath.Join(filepath.Dir(roots.input), "reference.txt")
if err := os.WriteFile(referencePath, []byte("reference content"), 0o600); err != nil {
t.Fatal(err)
}
harness := newStateTestHarness()
var firstStdout, firstStderr bytes.Buffer
first := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input}, &firstStdout, &firstStderr, harness.options())
if first != 0 {
t.Fatalf("first run code=%d stdout=%q stderr=%q", first, firstStdout.String(), firstStderr.String())
}
configData, err := os.ReadFile(roots.config)
if err != nil {
t.Fatal(err)
}
configText = replaceRequiredOnce(t, string(configData), "strategy: first", "strategy: second")
if err := os.WriteFile(roots.config, []byte(configText), 0o600); err != nil {
t.Fatal(err)
}
var stdout, stderr bytes.Buffer
second := RunWithOptions([]string{
"run", "sample", "--config", roots.config, "--input", roots.input,
"--only", "items", "--reference", "chunk.cache-reference=" + referencePath,
}, &stdout, &stderr, harness.options())
if second != 0 {
t.Fatalf("second run code=%d stdout=%q stderr=%q", second, stdout.String(), stderr.String())
}
harness.mu.Lock()
chunkCalls := harness.chunkCalls
sessions := append([]string(nil), harness.sessionIDs...)
harness.mu.Unlock()
if chunkCalls != 1 {
t.Fatalf("chunk calls across changed run inputs = %d, want 1", chunkCalls)
}
rawInput, err := os.ReadFile(roots.input)
if err != nil {
t.Fatal(err)
}
wantSessionID, err := resolvePromptSessionID("", "test/input", rawInput)
if err != nil {
t.Fatal(err)
}
for _, sessionID := range sessions {
if sessionID != wantSessionID {
t.Fatalf("session IDs across reference changes = %#v, want %q", sessions, wantSessionID)
}
}
assertFile(t, filepath.Join(roots.plans, strings.TrimPrefix(stateTestDigest, "sha256:"), "plan.json"))
assertAnyFile(t, roots.output)
}
func TestRunResumeSelectsConfiguredOrPerUserCheckpointRoot(t *testing.T) {
for _, configured := range []bool{true, false} {
name := "per-user root"
if configured {
name = "configured root"
}
t.Run(name, func(t *testing.T) {
roots := newStateTestRoots(t)
if !configured {
removeStateTestConfigLine(t, roots.config, fmt.Sprintf(" directory: %q\n", roots.checkpoints))
}
userCache := filepath.Join(t.TempDir(), "user-cache")
userCacheCalls := 0
opts := newStateTestHarness().options()
opts.UserCacheDir = func() (string, error) {
userCacheCalls++
return userCache, nil
}
result := runStateTest(t, roots, opts, false, true, "bypass")
if result.code != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr)
}
wantRoot := roots.checkpoints
wantCalls := 0
if !configured {
wantRoot = filepath.Join(userCache, "notarius", "checkpoints")
wantCalls = 1
}
if userCacheCalls != wantCalls {
t.Fatalf("user cache calls = %d, want %d", userCacheCalls, wantCalls)
}
assertAnyFile(t, wantRoot)
if !configured {
assertAbsent(t, roots.checkpoints)
}
assertStateTestOutput(t, roots.output)
})
}
t.Run("disabled avoids checkpoint root resolution", func(t *testing.T) {
roots := newStateTestRoots(t)
replaceStateTestConfigLine(t, roots.config, " enabled: true\n", " enabled: false\n")
removeStateTestConfigLine(t, roots.config, fmt.Sprintf(" directory: %q\n", roots.checkpoints))
opts := newStateTestHarness().options()
opts.UserCacheDir = func() (string, error) { return "", errors.New("checkpoint cache must not be resolved") }
result := runStateTest(t, roots, opts, false, false, "bypass")
if result.code != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr)
}
assertStateTestOutput(t, roots.output)
assertAbsent(t, roots.checkpoints)
})
t.Run("resume requires enabled checkpoint recording", func(t *testing.T) {
roots := newStateTestRoots(t)
replaceStateTestConfigLine(t, roots.config, " enabled: true\n", " enabled: false\n")
result := runStateTest(t, roots, newStateTestHarness().options(), true, true, "bypass")
if result.code != 1 || !strings.Contains(result.stderr, "--resume requires cache.checkpoints.enabled: true") {
t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr)
}
assertNoRunState(t, roots)
})
}
func TestConfigCommandsDoNotResolveRunState(t *testing.T) {
for _, args := range [][]string{
{"config", "validate", "--config"},
@@ -338,9 +18,6 @@ func TestConfigCommandsDoNotResolveRunState(t *testing.T) {
roots := newStateTestRoots(t)
opts := newStateTestHarness().options()
opts.UserCacheDir = func() (string, error) { return "", errors.New("state root must not be resolved") }
opts.ChunkPlanStoreFactory = func(string) (pipeline.ChunkPlanStore, error) {
return nil, errors.New("chunk plan store must not be constructed")
}
command := append([]string(nil), args...)
command = append(command, roots.config)
var stdout, stderr bytes.Buffer

View File

@@ -1,338 +0,0 @@
package cli
import (
"bytes"
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"reflect"
"sort"
"strings"
"sync"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
func TestRecomputeStepRecoversThroughFilesystemCheckpoints(t *testing.T) {
tests := []struct {
name string
invalidateOutput bool
wantCode int
}{
{name: "accepted producer is hydrated", wantCode: 0},
{name: "invalid producer stops dependents", invalidateOutput: true, wantCode: 1},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
roots := newRecomputeTestRoots(t)
harness := newRecomputeTestHarness()
fresh := runRecomputeCommand(roots, harness.options(), false)
if fresh.code != 0 {
t.Fatalf("fresh run code=%d stderr=%q", fresh.code, fresh.stderr)
}
removeCheckpointLaneStage(t, roots.checkpoints, "extract", "first", "producer")
removeCheckpointLaneStage(t, roots.checkpoints, "merge", "first", "producer")
if tt.invalidateOutput {
path := findCheckpointFile(t, roots.checkpoints, "normalize", "first", "producer", "output.json")
if err := os.WriteFile(path, []byte("{"), 0o600); err != nil {
t.Fatal(err)
}
}
harness.resetCalls()
resumed := runRecomputeCommand(roots, harness.options(), true)
if resumed.code != tt.wantCode {
t.Fatalf("resumed code=%d stdout=%q stderr=%q", resumed.code, resumed.stdout, resumed.stderr)
}
events := readLatestCheckpointEvents(t, roots.debug)
if tt.invalidateOutput {
if harness.callsFor("test/extract/middle") != 0 || harness.callsFor("test/extract/dependent") != 0 {
t.Fatalf("dependent calls after invalid producer = %#v", harness.callsSnapshot())
}
if !strings.Contains(resumed.stderr, string(pipeline.CheckpointReasonDecodeFailed)) {
t.Fatalf("stderr=%q, want stable checkpoint reason", resumed.stderr)
}
assertNormalizeDecisionSequence(t, events, []checkpointDecisionExpectation{{"first", "producer", pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonDecodeFailed}})
return
}
if got := harness.callsSnapshot(); !reflect.DeepEqual(got, map[string]int{"test/extract/dependent": 1, "test/extract/middle": 1}) {
t.Fatalf("resumed extractor calls = %#v", got)
}
outputPath := filepath.Join(latestChildDir(t, roots.output), "result.json")
data, err := os.ReadFile(outputPath)
if err != nil {
t.Fatal(err)
}
if string(data) != "[\"producer\",\"unrelated\",\"middle\",\"dependent\"]\n" {
t.Fatalf("ordered output = %q", data)
}
assertNormalizeDecisionSequence(t, events, []checkpointDecisionExpectation{
{"first", "producer", pipeline.CheckpointDecisionReused, pipeline.CheckpointReasonAcceptedArtifactReused},
{"first", "unrelated", pipeline.CheckpointDecisionReused, pipeline.CheckpointReasonReused},
{"second", "middle", pipeline.CheckpointDecisionForcedRecompute, pipeline.CheckpointReasonRecomputeStep},
{"third", "dependent", pipeline.CheckpointDecisionForcedRecompute, pipeline.CheckpointReasonRecomputeStep},
})
})
}
}
type checkpointDecisionExpectation struct {
step, lane string
action pipeline.CheckpointDecisionCategory
reason pipeline.CheckpointReasonCode
}
func assertNormalizeDecisionSequence(t *testing.T, events []pipeline.CheckpointEvent, want []checkpointDecisionExpectation) {
t.Helper()
var got []checkpointDecisionExpectation
for _, event := range events {
if event.Stage == string(pipeline.StageNormalize) {
got = append(got, checkpointDecisionExpectation{event.StepID, event.LaneID, event.Action, event.ReasonCode})
}
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("normalize decisions = %#v, want %#v", got, want)
}
}
type recomputeTestHarness struct {
base *stateTestHarness
mu sync.Mutex
calls map[string]int
}
func newRecomputeTestHarness() *recomputeTestHarness {
return &recomputeTestHarness{base: newStateTestHarness(), calls: make(map[string]int)}
}
func (h *recomputeTestHarness) options() Options {
opts := h.base.options()
for _, key := range []string{"test/extract/producer", "test/extract/unrelated", "test/extract/middle", "test/extract/dependent"} {
moduleKey := key
spec := pipeline.ModuleSpec{
Key: moduleKey, Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: stateTestArtifactKind,
ReferenceSlots: []contracts.ReferenceSlot{{Name: "upstream", AcceptedMediaTypes: []string{"application/json"}, AcceptedArtifactKinds: []contracts.ArtifactKind{stateTestArtifactKind}}},
}
if err := pipeline.RegisterExtractor(opts.Registries.Extractors, spec, func() (contracts.Extractor[stateTestArtifact], error) {
return recomputeTestExtractor{key: moduleKey, harness: h}, nil
}); err != nil {
panic(err)
}
}
if err := opts.Registries.Outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "test/recompute-output", Stage: pipeline.StageOutput, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"normalized"}, Provides: []string{"output"}}, func() (contracts.OutputEncoder, error) {
return recomputeTestOutput{}, nil
}); err != nil {
panic(err)
}
opts.Catalog = catalogFromRegistries(opts.Registries)
return opts
}
func (h *recomputeTestHarness) record(key string) {
h.mu.Lock()
defer h.mu.Unlock()
h.calls[key]++
}
func (h *recomputeTestHarness) resetCalls() {
h.mu.Lock()
defer h.mu.Unlock()
h.calls = make(map[string]int)
}
func (h *recomputeTestHarness) callsFor(key string) int {
h.mu.Lock()
defer h.mu.Unlock()
return h.calls[key]
}
func (h *recomputeTestHarness) callsSnapshot() map[string]int {
h.mu.Lock()
defer h.mu.Unlock()
result := make(map[string]int, len(h.calls))
for key, value := range h.calls {
result[key] = value
}
return result
}
type recomputeTestExtractor struct {
key string
harness *recomputeTestHarness
}
func (e recomputeTestExtractor) Key() string { return e.key }
func (e recomputeTestExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return []contracts.ReferenceSlot{{Name: "upstream", AcceptedMediaTypes: []string{"application/json"}, AcceptedArtifactKinds: []contracts.ArtifactKind{stateTestArtifactKind}}}
}
func (e recomputeTestExtractor) Extract(context.Context, contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[stateTestArtifact], error) {
e.harness.record(e.key)
return contracts.TypedExtractionResult[stateTestArtifact]{Value: stateTestArtifact{Value: e.key}}, nil
}
type recomputeTestOutput struct{}
func (recomputeTestOutput) Key() string { return "test/recompute-output" }
func (recomputeTestOutput) Encode(_ context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
lanes := make([]string, 0, len(req.NormalizeOutputs))
for _, output := range req.NormalizeOutputs {
lanes = append(lanes, output.LaneID)
}
data, err := json.Marshal(lanes)
if err != nil {
return contracts.OutputResult{}, err
}
return contracts.OutputResult{Files: []contracts.OutputFile{{Name: "result.json", Bytes: append(data, '\n')}}}, nil
}
func newRecomputeTestRoots(t *testing.T) stateTestRoots {
t.Helper()
roots := newStateTestRoots(t)
config := fmt.Sprintf(`version: 5
output:
directory: %q
cache:
chunk_plans:
directory: %q
mode: bypass
checkpoints:
enabled: true
directory: %q
debug:
directory: %q
pipelines:
sample:
input:
module: test/input
chunk:
module: test/chunk
steps:
- id: first
artifacts:
producer:
extract:
module: test/extract/producer
merge:
module: test/merge
normalize:
module: test/normalize
unrelated:
extract:
module: test/extract/unrelated
merge:
module: test/merge
normalize:
module: test/normalize
- id: second
references:
upstream:
artifact:
step: first
lane: producer
artifacts:
middle:
extract:
module: test/extract/middle
merge:
module: test/merge
normalize:
module: test/normalize
- id: third
references:
upstream:
artifact:
step: second
lane: middle
artifacts:
dependent:
extract:
module: test/extract/dependent
merge:
module: test/merge
normalize:
module: test/normalize
output:
module: test/recompute-output
`, roots.output, roots.plans, roots.checkpoints, roots.debug)
if err := os.WriteFile(roots.config, []byte(config), 0o600); err != nil {
t.Fatal(err)
}
return roots
}
func runRecomputeCommand(roots stateTestRoots, opts Options, recompute bool) stateTestResult {
args := []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}
if recompute {
args = append(args, "--resume", "--recompute-step", "second", "--debug")
}
var stdout, stderr bytes.Buffer
return stateTestResult{code: RunWithOptions(args, &stdout, &stderr, opts), stdout: stdout.String(), stderr: stderr.String()}
}
func removeCheckpointLaneStage(t *testing.T, root, stage, step, lane string) {
t.Helper()
dir := filepath.Dir(findCheckpointFile(t, root, stage, step, lane, "manifest.json"))
if err := os.RemoveAll(dir); err != nil {
t.Fatal(err)
}
}
func findCheckpointFile(t *testing.T, root, stage, step, lane, name string) string {
t.Helper()
want := filepath.Join(stage, step, lane, name)
var matches []string
err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error {
if err != nil {
return err
}
if !entry.IsDir() && strings.HasSuffix(path, want) {
matches = append(matches, path)
}
return nil
})
if err != nil {
t.Fatal(err)
}
if len(matches) != 1 {
t.Fatalf("checkpoint files ending in %q = %v", want, matches)
}
return matches[0]
}
func latestChildDir(t *testing.T, root string) string {
t.Helper()
entries, err := os.ReadDir(root)
if err != nil {
t.Fatal(err)
}
var dirs []string
for _, entry := range entries {
if entry.IsDir() {
dirs = append(dirs, filepath.Join(root, entry.Name()))
}
}
if len(dirs) == 0 {
t.Fatal("no child directory")
}
sort.Strings(dirs)
return dirs[len(dirs)-1]
}
func readLatestCheckpointEvents(t *testing.T, root string) []pipeline.CheckpointEvent {
t.Helper()
var events []pipeline.CheckpointEvent
data, err := os.ReadFile(filepath.Join(latestChildDir(t, root), "summary", "checkpoint-events.json"))
if err != nil {
t.Fatal(err)
}
if err := json.Unmarshal(data, &events); err != nil {
t.Fatal(err)
}
return events
}

View File

@@ -1,56 +0,0 @@
package cli
import (
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
func TestRecomputePolicyIncludesDependentLanesAndReusablePredecessors(t *testing.T) {
producer := pipeline.ResolvedArtifactLane{StepID: "first", ID: "producer"}
unrelated := pipeline.ResolvedArtifactLane{StepID: "first", ID: "unrelated"}
consumer := pipeline.ResolvedArtifactLane{
StepID: "second", ID: "consumer",
ExtractReferences: pipeline.ResolvedReferenceTarget{Bindings: []pipeline.ReferenceBinding{{Artifact: &pipeline.ArtifactReference{Step: "first", Lane: "producer"}}}},
}
downstream := pipeline.ResolvedArtifactLane{
StepID: "third", ID: "downstream",
ExtractReferences: pipeline.ResolvedReferenceTarget{Bindings: []pipeline.ReferenceBinding{{Artifact: &pipeline.ArtifactReference{Step: "second", Lane: "consumer"}}}},
}
independent := pipeline.ResolvedArtifactLane{StepID: "third", ID: "independent"}
resolved := pipeline.ResolvedPipeline{Steps: []pipeline.ResolvedPipelineStep{
{ID: "first", ArtifactLanes: []pipeline.ResolvedArtifactLane{producer, unrelated}},
{ID: "second", ArtifactLanes: []pipeline.ResolvedArtifactLane{consumer}},
{ID: "third", ArtifactLanes: []pipeline.ResolvedArtifactLane{downstream, independent}},
}}
policy, err := recomputePolicy(resolved, "second")
if err != nil {
t.Fatal(err)
}
if _, ok := policy.ForcedLanes[pipeline.CheckpointLaneKey("second", "consumer")]; !ok {
t.Fatal("selected lane was not forced")
}
if _, ok := policy.ForcedLanes[pipeline.CheckpointLaneKey("third", "downstream")]; !ok {
t.Fatal("transitive dependent lane was not forced")
}
if _, ok := policy.ForcedLanes[pipeline.CheckpointLaneKey("first", "producer")]; ok {
t.Fatal("predecessor was implicitly forced")
}
if _, ok := policy.RequireReusableLanes[pipeline.CheckpointLaneKey("first", "producer")]; !ok {
t.Fatal("required predecessor was not marked reusable")
}
if _, ok := policy.ForcedLanes[pipeline.CheckpointLaneKey("first", "unrelated")]; ok {
t.Fatal("unrelated lane was forced")
}
if _, ok := policy.ForcedLanes[pipeline.CheckpointLaneKey("third", "independent")]; ok {
t.Fatal("unrelated later lane was forced")
}
}
func TestRecomputePolicyRejectsUnknownStep(t *testing.T) {
_, err := recomputePolicy(pipeline.ResolvedPipeline{Steps: []pipeline.ResolvedPipelineStep{{ID: "known"}}}, "missing")
if err == nil {
t.Fatal("unknown step was accepted")
}
}

View File

@@ -20,8 +20,7 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle"
"gitea.maximumdirect.net/eric/notarius/internal/core/fileio"
"gitea.maximumdirect.net/eric/notarius/internal/framework/checkpoint"
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkplan"
"gitea.maximumdirect.net/eric/notarius/internal/framework/artifactcache"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
frameworkdebug "gitea.maximumdirect.net/eric/notarius/internal/framework/debug"
frameworkllm "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
@@ -32,19 +31,18 @@ import (
)
type Options struct {
LLMClient promptkit.LLMClient
LLMRuntime promptkit.RuntimeConfig
Catalog pipeline.ModuleCatalog
Registries pipeline.Registries
LLMClientFactory LLMClientFactory
RunIDGenerator RunIDGenerator
LookupEnv func(string) (string, bool)
Now func() time.Time
UserCacheDir func() (string, error)
ChunkPlanStoreFactory pipeline.ChunkPlanStoreFactory
DebugRecorderFactory func(string) (pipeline.DebugRecorder, error)
DebugTerminalFactory func(*debugbundle.SummaryWriter) DebugTerminalWriter
promptKitAssets *frameworkllm.AssetRegistry
LLMClient promptkit.LLMClient
LLMRuntime promptkit.RuntimeConfig
Catalog pipeline.ModuleCatalog
Registries pipeline.Registries
LLMClientFactory LLMClientFactory
RunIDGenerator RunIDGenerator
LookupEnv func(string) (string, bool)
Now func() time.Time
UserCacheDir func() (string, error)
DebugRecorderFactory func(string) (pipeline.DebugRecorder, error)
DebugTerminalFactory func(*debugbundle.SummaryWriter) DebugTerminalWriter
promptKitAssets *frameworkllm.AssetRegistry
}
type LLMRuntimeOverrides struct {
@@ -85,9 +83,6 @@ func normalizeOptions(opts Options) (Options, error) {
if opts.UserCacheDir == nil {
opts.UserCacheDir = os.UserCacheDir
}
if opts.ChunkPlanStoreFactory == nil {
opts.ChunkPlanStoreFactory = chunkplan.NewFilesystemStore
}
if opts.DebugRecorderFactory == nil {
opts.DebugRecorderFactory = frameworkdebug.NewFilesystemRecorder
}
@@ -130,19 +125,14 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
llmProfile := singleValueFlag{name: "--llm-profile"}
reasoningEffort := singleValueFlag{name: "--reasoning-effort"}
clearReasoningEffort := fs.Bool("clear-reasoning-effort", false, "clear the LLM profile reasoning effort")
resume := fs.Bool("resume", false, "reuse compatible recorded checkpoints")
recomputeStep := singleValueFlag{name: "--recompute-step"}
chunkCache := chunkCacheFlag{}
requestedSessionID := sessionIDFlag{}
referenceFlags := stringListFlag{}
withoutReferenceFlags := stringListFlag{}
fs.Var(&requestedSessionID, "session-id", "prompt session identifier")
fs.Var(&llmProfile, "llm-profile", "LLM profile override")
fs.Var(&reasoningEffort, "reasoning-effort", "reasoning effort override")
fs.Var(&chunkCache, "chunk_cache", "chunk plan cache mode: auto, bypass, or refresh")
fs.Var(&referenceFlags, "reference", "reference binding, as slot=path, chunk.slot=path, lane.slot=path, lane.extract.slot=path, lane.merge.slot=path, or lane.normalize.slot=path")
fs.Var(&withoutReferenceFlags, "without-reference", "unbind a reference, using the same selector forms as --reference")
fs.Var(&recomputeStep, "recompute-step", "recompute one ordered pipeline step and dependent lanes")
if err := validateRunFlagValues(args); err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 2
@@ -209,14 +199,6 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 2
}
if recomputeStep.set && strings.TrimSpace(recomputeStep.value) == "" {
fmt.Fprintln(stderr, "notarius: --recompute-step must not be empty")
return 2
}
if recomputeStep.set && len(only) > 0 {
fmt.Fprintln(stderr, "notarius: --recompute-step cannot be combined with --only")
return 2
}
referenceRequests, err := parseReferenceFlags(referenceFlags)
if err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
@@ -228,7 +210,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
return 2
}
loaded, loadedConfigPath, err := loadConfigKit(*configPath, *configOverlay, runConfigurationOverrides(chunkCache, *outputDir, *debugDir), opts)
loaded, loadedConfigPath, err := loadConfigKit(*configPath, *configOverlay, runConfigurationOverrides(*outputDir, *debugDir), opts)
if err != nil {
writeConfigLoadError(stderr, err)
return 1
@@ -238,19 +220,6 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 1
}
if *resume && !cfg.Cache.Checkpoints.Enabled {
fmt.Fprintln(stderr, "notarius: --resume requires cache.checkpoints.enabled: true")
return 1
}
if recomputeStep.set && !*resume {
fmt.Fprintln(stderr, "notarius: --recompute-step requires --resume")
return 2
}
if recomputeStep.set && !cfg.Cache.Checkpoints.Enabled {
fmt.Fprintln(stderr, "notarius: --recompute-step requires cache.checkpoints.enabled: true")
return 1
}
startedAt := opts.Now().UTC()
runID, err := opts.RunIDGenerator(startedAt)
if err != nil {
@@ -293,10 +262,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
ConfigSource: configSource(*configPath),
ConfigOverlay: effectiveConfigOverlay(*configOverlay),
OnlyLanes: append([]string(nil), only...),
ChunkCacheOverride: chunkCache.explicitValue(),
ReasoningEffortOverride: runtimeOverrides.ReasoningEffort,
Resume: *resume,
RecomputeStep: strings.TrimSpace(recomputeStep.value),
RunID: runID,
StartedAt: startedAt,
}
@@ -339,13 +305,6 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
return failPipelineCommand(stderr, commandState, terminalWriter, err)
}
effective.ResolvedPipeline = materialized
checkpointPolicy := pipeline.CheckpointExecutionPolicy{}
if recomputeStep.set {
checkpointPolicy, err = recomputePolicy(effective.ResolvedPipeline, recomputeStep.value)
if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, err)
}
}
invocation.PipelineDigest = effective.ResolvedPipeline.Digest
if err := writeSummary(summary, func() error { return summary.WriteInvocation(invocation) }); err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug invocation metadata: %w", err))
@@ -376,13 +335,6 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("create LLM client for profile %q: %w", factoryProfileID, err))
}
var llmFingerprints []checkpoint.Fingerprint
if effective.Config.Cache.Checkpoints.Enabled {
llmFingerprints, err = llmCheckpointFingerprints(llmClient)
if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("prepare LLM checkpoint identity: %w", err))
}
}
llmClient = pipeline.WithDebugLLMRecording(llmClient, debugRecorder)
prepared, err := pipeline.Prepare(effective.ResolvedPipeline, registries, pipeline.ModuleDependencies{LLM: llmClient})
if err != nil {
@@ -400,32 +352,24 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
if err := writeSummary(summary, func() error { return summary.WriteInvocation(invocation) }); err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug invocation metadata: %w", err))
}
chunkPlans, err := chunkPlanStoreForRun(effective.Config.Cache.ChunkPlans, opts)
if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, err)
}
checkpointRecorder, checkpointLoader, err := checkpointHandlersForRun(effective.Config.Cache.Checkpoints, opts, effective.ResolvedPipeline, prepared.CheckpointFingerprints(), llmFingerprints, rawInput, only, llmProfiles, strings.TrimSpace(llmProfile.value), effectiveSessionID, runtimeOverrides, *resume)
cache, err := artifactcache.NewFilesystemStore(effective.Config.CacheDirectory)
if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, err)
}
output, err := pipeline.New().Run(ctx, pipeline.RunInput{
Prepared: prepared,
Path: strings.TrimSpace(*inputPath),
RawInput: rawInput,
SessionID: effectiveSessionID,
RunID: runID,
StartedAt: startedAt,
LLMProfiles: llmProfiles,
Metadata: runMetadata(effective.Config.Output.Directory, debugPath),
Diagnostics: referenceDiagnostics,
ChunkCacheMode: effective.Config.Cache.ChunkPlans.Mode,
ChunkPlans: chunkPlans,
Checkpoints: checkpointRecorder,
Checkpoint: checkpointLoader,
CheckpointPolicy: checkpointPolicy,
Debug: debugRecorder,
Workers: cfg.Concurrency.Workers,
Prepared: prepared,
Path: strings.TrimSpace(*inputPath),
RawInput: rawInput,
SessionID: effectiveSessionID,
RunID: runID,
StartedAt: startedAt,
LLMProfiles: llmProfiles,
Metadata: runMetadata(effective.Config.Output.Directory, debugPath),
Diagnostics: referenceDiagnostics,
Cache: cache,
Debug: debugRecorder,
Workers: cfg.Concurrency.Workers,
})
diagnosticProjection, diagnosticErr := contracts.ProjectDiagnosticCollection(output.Diagnostics)
if diagnosticErr == nil {
@@ -511,196 +455,10 @@ func writePartialSummary(summary *debugbundle.SummaryWriter, output pipeline.Run
if err := summary.WriteRunManifest(output.Manifest); err != nil {
return err
}
if output.ChunkPlan != nil {
if err := summary.WriteChunkPlan(*output.ChunkPlan); err != nil {
return err
}
}
if err := summary.WriteDiagnostics(output.Diagnostics); err != nil {
return err
}
return summary.WriteCheckpointEvents(output.CheckpointEvents)
}
func checkpointHandlersForRun(
settings config.CheckpointCacheConfig,
opts Options,
resolved pipeline.ResolvedPipeline,
componentFingerprints []pipeline.CheckpointFingerprint,
llmFingerprints []checkpoint.Fingerprint,
rawInput []byte,
only []string,
llmProfiles []artifacts.LLMProfileManifest,
llmProfileOverride string,
sessionID string,
runtimeOverrides LLMRuntimeOverrides,
resume bool,
) (pipeline.CheckpointRecorder, pipeline.CheckpointLoader, error) {
if !settings.Enabled {
if resume {
return nil, nil, fmt.Errorf("--resume requires cache.checkpoints.enabled: true")
}
return pipeline.NoopCheckpointRecorder(), pipeline.NoopCheckpointLoader(), nil
}
identity, err := checkpoint.NewIdentity(checkpoint.IdentityInput{
Pipeline: resolved,
InputKey: resolved.Input.Module,
RawInputDigest: rawInputDigest(rawInput),
SelectedLanes: only,
RuntimeOverrides: runtimeOverrideFingerprints(llmProfileOverride, sessionID, runtimeOverrides),
References: pipeline.ReferenceProvenance(resolved),
ProvenanceFingerprints: combineCheckpointFingerprints(
llmProfileFingerprints(llmProfiles),
llmFingerprints,
checkpointIdentityFingerprints(componentFingerprints),
),
})
if err != nil {
return nil, nil, fmt.Errorf("create checkpoint identity: %w", err)
}
checkpointRoot := strings.TrimSpace(settings.Directory)
if checkpointRoot == "" {
checkpointRoot, err = config.DefaultCheckpointRoot(opts.UserCacheDir)
if err != nil {
return nil, nil, fmt.Errorf("resolve checkpoint root: %w", err)
}
}
recorder, err := checkpoint.NewFilesystemRecorder(checkpointRoot, identity)
if err != nil {
return nil, nil, fmt.Errorf("create checkpoint recorder: %w", err)
}
loader := pipeline.NoopCheckpointLoader()
if resume {
loader, err = checkpoint.NewFilesystemLoader(checkpointRoot, identity)
if err != nil {
return nil, nil, fmt.Errorf("create checkpoint loader: %w", err)
}
}
return recorder, loader, nil
}
func llmCheckpointFingerprints(client contracts.StructuredLLMClient) ([]checkpoint.Fingerprint, error) {
provider, ok := client.(frameworkllm.CheckpointFingerprintProvider)
if !ok {
return nil, nil
}
values, err := provider.LLMCheckpointFingerprints()
if err != nil {
return nil, err
}
out := make([]checkpoint.Fingerprint, 0, len(values))
for _, value := range values {
out = append(out, checkpoint.Fingerprint{Name: value.Name, Value: value.Value})
}
return out, nil
}
func combineCheckpointFingerprints(sources ...[]checkpoint.Fingerprint) []checkpoint.Fingerprint {
var out []checkpoint.Fingerprint
for _, source := range sources {
out = append(out, source...)
}
return out
}
func recomputePolicy(resolved pipeline.ResolvedPipeline, requestedStep string) (pipeline.CheckpointExecutionPolicy, error) {
requestedStep = strings.TrimSpace(requestedStep)
if requestedStep == "" {
return pipeline.CheckpointExecutionPolicy{}, fmt.Errorf("--recompute-step must not be empty")
}
var selected *pipeline.ResolvedPipelineStep
for index := range resolved.Steps {
if strings.TrimSpace(resolved.Steps[index].ID) == requestedStep {
selected = &resolved.Steps[index]
break
}
}
if selected == nil {
return pipeline.CheckpointExecutionPolicy{}, fmt.Errorf("unknown pipeline step %q", requestedStep)
}
policy := pipeline.CheckpointExecutionPolicy{ForcedLanes: make(map[string]struct{}), RequireReusableLanes: make(map[string]struct{})}
var forced []pipeline.ResolvedArtifactLane
for _, lane := range selected.ArtifactLanes {
lane.StepID = selected.ID
policy.ForcedLanes[pipeline.CheckpointLaneKey(selected.ID, lane.ID)] = struct{}{}
forced = append(forced, lane)
}
lanes := make(map[string]pipeline.ResolvedArtifactLane)
dependents := make(map[string][]pipeline.ResolvedArtifactLane)
for _, step := range resolved.Steps {
for _, lane := range step.ArtifactLanes {
lane.StepID = step.ID
lanes[pipeline.CheckpointLaneKey(step.ID, lane.ID)] = lane
for _, target := range []pipeline.ResolvedReferenceTarget{lane.ExtractReferences, lane.MergeReferences, lane.NormalizeReferences} {
for _, binding := range target.Bindings {
if binding.Artifact != nil {
producerKey := pipeline.CheckpointLaneKey(binding.Artifact.Step, binding.Artifact.Lane)
dependents[producerKey] = append(dependents[producerKey], lane)
}
}
}
}
}
processed := make(map[string]struct{})
queue := append([]pipeline.ResolvedArtifactLane(nil), forced...)
for len(queue) > 0 {
lane := queue[0]
queue = queue[1:]
key := pipeline.CheckpointLaneKey(lane.StepID, lane.ID)
if _, ok := processed[key]; ok {
continue
}
processed[key] = struct{}{}
for _, dependent := range dependents[key] {
dependentKey := pipeline.CheckpointLaneKey(dependent.StepID, dependent.ID)
if _, alreadyForced := policy.ForcedLanes[dependentKey]; alreadyForced {
continue
}
policy.ForcedLanes[dependentKey] = struct{}{}
forced = append(forced, dependent)
queue = append(queue, dependent)
}
}
processed = make(map[string]struct{})
queue = append([]pipeline.ResolvedArtifactLane(nil), forced...)
for len(queue) > 0 {
lane := queue[0]
queue = queue[1:]
key := pipeline.CheckpointLaneKey(lane.StepID, lane.ID)
if _, ok := processed[key]; ok {
continue
}
processed[key] = struct{}{}
for _, target := range []pipeline.ResolvedReferenceTarget{lane.ExtractReferences, lane.MergeReferences, lane.NormalizeReferences} {
for _, binding := range target.Bindings {
if binding.Artifact == nil {
continue
}
producerKey := pipeline.CheckpointLaneKey(binding.Artifact.Step, binding.Artifact.Lane)
if _, forcedAlready := policy.ForcedLanes[producerKey]; !forcedAlready {
policy.RequireReusableLanes[producerKey] = struct{}{}
}
if producer, ok := lanes[producerKey]; ok {
queue = append(queue, producer)
}
}
}
}
return policy, nil
}
func checkpointIdentityFingerprints(values []pipeline.CheckpointFingerprint) []checkpoint.Fingerprint {
if len(values) == 0 {
return nil
}
out := make([]checkpoint.Fingerprint, len(values))
for index, value := range values {
out[index] = checkpoint.Fingerprint{Name: "component:" + value.Name, Value: value.Value}
}
return out
return nil
}
func rawInputDigest(data []byte) string {
@@ -708,42 +466,6 @@ func rawInputDigest(data []byte) string {
return "sha256:" + hex.EncodeToString(sum[:])
}
func runtimeOverrideFingerprints(llmProfileOverride string, sessionID string, runtimeOverrides LLMRuntimeOverrides) []checkpoint.Fingerprint {
var values []checkpoint.Fingerprint
if strings.TrimSpace(llmProfileOverride) != "" {
values = append(values, checkpoint.Fingerprint{Name: "llm_profile_override", Value: strings.TrimSpace(llmProfileOverride)})
}
if strings.TrimSpace(sessionID) != "" {
values = append(values, checkpoint.Fingerprint{Name: "session_id", Value: strings.TrimSpace(sessionID)})
}
if runtimeOverrides.ReasoningEffort != nil {
value := strings.TrimSpace(*runtimeOverrides.ReasoningEffort)
if value == "" {
value = "<cleared>"
}
values = append(values, checkpoint.Fingerprint{Name: "reasoning_effort_override", Value: value})
}
return values
}
func llmProfileFingerprints(profiles []artifacts.LLMProfileManifest) []checkpoint.Fingerprint {
if len(profiles) == 0 {
return nil
}
values := make([]checkpoint.Fingerprint, 0, len(profiles))
for _, profile := range profiles {
id := strings.TrimSpace(profile.ID)
if id == "" {
continue
}
values = append(values, checkpoint.Fingerprint{
Name: "llm_profile:" + id,
Value: strings.TrimSpace(profile.Provider) + ":" + strings.TrimSpace(profile.Model),
})
}
return values
}
func configSource(configPath string) string {
if strings.TrimSpace(configPath) != "" {
return "flag"
@@ -837,22 +559,15 @@ func reorderRunArgs(args []string) []string {
func runFlagTakesValue(arg string) bool {
switch arg {
case "--config", "--config-overlay", "--input", "--only", "--output-dir", "--debug-dir", "--llm-profile", "--session-id", "--reasoning-effort", "--chunk_cache", "--reference", "--without-reference", "--recompute-step":
case "--config", "--config-overlay", "--input", "--only", "--output-dir", "--debug-dir", "--llm-profile", "--session-id", "--reasoning-effort", "--reference", "--without-reference":
return true
default:
return false
}
}
func runConfigurationOverrides(chunkCache chunkCacheFlag, outputDir, debugDir string) []configkit.Override {
func runConfigurationOverrides(outputDir, debugDir string) []configkit.Override {
overrides := make([]configkit.Override, 0, 3)
if chunkCache.set {
overrides = append(overrides, configkit.Override{
Path: configkit.Keys("cache", "chunk_plans", "mode"),
Value: string(chunkCache.value),
Flag: "--chunk_cache",
})
}
if outputDir = strings.TrimSpace(outputDir); outputDir != "" {
overrides = append(overrides, configkit.Override{
Path: configkit.Keys("output", "directory"),
@@ -870,57 +585,6 @@ func runConfigurationOverrides(chunkCache chunkCacheFlag, outputDir, debugDir st
return overrides
}
type chunkCacheFlag struct {
value pipeline.ChunkCacheMode
set bool
}
func (f *chunkCacheFlag) String() string {
if f == nil {
return ""
}
return string(f.value)
}
func (f *chunkCacheFlag) Set(raw string) error {
mode, err := pipeline.ParseChunkCacheMode(raw)
if err != nil {
return err
}
f.value = mode
f.set = true
return nil
}
func (f chunkCacheFlag) explicitValue() string {
if !f.set {
return ""
}
return string(f.value)
}
func chunkPlanStoreForRun(cfg config.ChunkPlanCacheConfig, opts Options) (pipeline.ChunkPlanStore, error) {
if cfg.Mode == pipeline.ChunkCacheBypass {
return nil, nil
}
root := strings.TrimSpace(cfg.Directory)
if root == "" {
var err error
root, err = config.DefaultChunkPlanRoot(opts.UserCacheDir)
if err != nil {
return nil, fmt.Errorf("resolve chunk plan root: %w", err)
}
}
store, err := opts.ChunkPlanStoreFactory(root)
if err != nil {
return nil, fmt.Errorf("create chunk plan store at %q: %w", root, err)
}
if store == nil {
return nil, fmt.Errorf("create chunk plan store at %q: factory returned nil", root)
}
return store, nil
}
func validateRunFlagValues(args []string) error {
for i, arg := range args {
if arg != "--config-overlay" && arg != "--session-id" && arg != "--reasoning-effort" {

View File

@@ -1,504 +0,0 @@
package pipeline
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"sort"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
type CheckpointFingerprint struct {
Name string `json:"name"`
Value string `json:"value"`
}
// CheckpointFingerprintProvider supplies stable, non-secret semantic identity
// for a prepared module or validator. Values must not contain source content,
// credentials, local paths, timestamps, or other invocation-specific data.
type CheckpointFingerprintProvider interface {
CheckpointFingerprints() []CheckpointFingerprint
}
type CheckpointRecorder interface {
SourceRunning(moduleKey string) error
SourceSucceeded(moduleKey string, doc *source.SourceDocument) error
SourceFailed(moduleKey string, err error) error
ExtractRunning(laneID string, moduleKey string, dependencies []CheckpointFingerprint) error
ExtractSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, outputs []CheckpointArtifact, rejected []contracts.RejectedOutput) error
ExtractFailed(laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error
MergeRunning(laneID string, moduleKey string, dependencies []CheckpointFingerprint) error
MergeSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, output CheckpointArtifact) error
MergeRejected(laneID string, moduleKey string, dependencies []CheckpointFingerprint, rejected contracts.RejectedOutput) error
MergeFailed(laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error
NormalizeRunning(laneID string, moduleKey string, dependencies []CheckpointFingerprint) error
NormalizeSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, output CheckpointArtifact) error
NormalizeRejected(laneID string, moduleKey string, dependencies []CheckpointFingerprint, rejected contracts.RejectedOutput) error
NormalizeFailed(laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error
}
// StepCheckpointRecorder is implemented by checkpoint stores that isolate
// lane artifacts by their ordered pipeline step. The legacy recorder methods
// remain available for callers that do not have step context.
type StepCheckpointRecorder interface {
ExtractRunningForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint) error
ExtractSucceededForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, outputs []CheckpointArtifact, rejected []contracts.RejectedOutput) error
ExtractFailedForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error
MergeRunningForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint) error
MergeSucceededForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, output CheckpointArtifact) error
MergeRejectedForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, rejected contracts.RejectedOutput) error
MergeFailedForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error
NormalizeRunningForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint) error
NormalizeSucceededForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, output CheckpointArtifact) error
NormalizeRejectedForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, rejected contracts.RejectedOutput) error
NormalizeFailedForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error
}
type CheckpointDecisionCategory string
const (
CheckpointDecisionExecuted CheckpointDecisionCategory = "executed"
CheckpointDecisionReused CheckpointDecisionCategory = "reused"
CheckpointDecisionForcedRecompute CheckpointDecisionCategory = "forced_recompute"
CheckpointDecisionDependencyInvalidated CheckpointDecisionCategory = "dependency_invalidated"
)
type CheckpointReasonCode string
const (
CheckpointReasonLoadingDisabled CheckpointReasonCode = "loading_disabled"
CheckpointReasonMissing CheckpointReasonCode = "checkpoint_missing"
CheckpointReasonPathInvalid CheckpointReasonCode = "checkpoint_path_invalid"
CheckpointReasonReadFailed CheckpointReasonCode = "checkpoint_read_failed"
CheckpointReasonDecodeFailed CheckpointReasonCode = "checkpoint_decode_failed"
CheckpointReasonWorkspaceSchemaIncompatible CheckpointReasonCode = "workspace_schema_incompatible"
CheckpointReasonIdentityMismatch CheckpointReasonCode = "identity_mismatch"
CheckpointReasonStageMismatch CheckpointReasonCode = "stage_mismatch"
CheckpointReasonStepMismatch CheckpointReasonCode = "step_mismatch"
CheckpointReasonLaneMismatch CheckpointReasonCode = "lane_mismatch"
CheckpointReasonModuleMismatch CheckpointReasonCode = "module_mismatch"
CheckpointReasonStatusNotReusable CheckpointReasonCode = "status_not_reusable"
CheckpointReasonDependencyMismatch CheckpointReasonCode = "dependency_mismatch"
CheckpointReasonArtifactPayloadInvalid CheckpointReasonCode = "artifact_payload_invalid"
CheckpointReasonArtifactDigestMismatch CheckpointReasonCode = "artifact_digest_mismatch"
CheckpointReasonArtifactCodecIncompatible CheckpointReasonCode = "artifact_codec_incompatible"
CheckpointReasonArtifactNotCanonical CheckpointReasonCode = "artifact_not_canonical"
CheckpointReasonReused CheckpointReasonCode = "checkpoint_reused"
CheckpointReasonAcceptedArtifactReused CheckpointReasonCode = "accepted_artifact_reused"
CheckpointReasonRecomputeStep CheckpointReasonCode = "recompute_step"
CheckpointReasonValidationIncompleteLineage CheckpointReasonCode = "validation_incomplete_lineage"
)
type CheckpointDecision struct {
Reused bool `json:"reused"`
Category CheckpointDecisionCategory `json:"category,omitempty"`
ReasonCode CheckpointReasonCode `json:"reason_code,omitempty"`
Detail string `json:"detail,omitempty"`
// Reason is retained as a compatibility/debug field for existing callers.
// Detail and Reason are derived from the stable reason code.
Reason string `json:"reason,omitempty"`
}
const checkpointDecisionDetailLimit = 512
func NewCheckpointDecision(category CheckpointDecisionCategory, reasonCode CheckpointReasonCode) CheckpointDecision {
return checkpointDecision(category, reasonCode)
}
func checkpointDecision(category CheckpointDecisionCategory, reasonCode CheckpointReasonCode) CheckpointDecision {
detail := normalizeCheckpointDecisionDetail(checkpointDecisionDetail(reasonCode))
return CheckpointDecision{
Reused: category == CheckpointDecisionReused,
Category: category,
ReasonCode: reasonCode,
Detail: detail,
Reason: detail,
}
}
func checkpointDecisionDetail(reasonCode CheckpointReasonCode) string {
switch reasonCode {
case CheckpointReasonLoadingDisabled:
return "checkpoint loading is disabled"
case CheckpointReasonMissing:
return "checkpoint artifact is missing"
case CheckpointReasonPathInvalid:
return "checkpoint location is invalid"
case CheckpointReasonReadFailed:
return "checkpoint artifact could not be read"
case CheckpointReasonDecodeFailed:
return "checkpoint artifact could not be decoded"
case CheckpointReasonWorkspaceSchemaIncompatible:
return "checkpoint workspace schema is incompatible"
case CheckpointReasonIdentityMismatch:
return "checkpoint identity does not match the current invocation"
case CheckpointReasonStageMismatch:
return "checkpoint stage does not match"
case CheckpointReasonStepMismatch:
return "checkpoint step does not match"
case CheckpointReasonLaneMismatch:
return "checkpoint lane does not match"
case CheckpointReasonModuleMismatch:
return "checkpoint module does not match"
case CheckpointReasonStatusNotReusable:
return "checkpoint status is not reusable"
case CheckpointReasonDependencyMismatch:
return "checkpoint dependencies do not match"
case CheckpointReasonArtifactPayloadInvalid:
return "checkpoint artifact payload is invalid"
case CheckpointReasonArtifactDigestMismatch:
return "checkpoint artifact digest does not match"
case CheckpointReasonArtifactCodecIncompatible:
return "checkpoint artifact is incompatible with the registered codec"
case CheckpointReasonArtifactNotCanonical:
return "checkpoint artifact is not canonical"
case CheckpointReasonReused:
return "checkpoint is reusable"
case CheckpointReasonAcceptedArtifactReused:
return "accepted normalized artifact is reusable"
case CheckpointReasonRecomputeStep:
return "selected step requires execution"
case CheckpointReasonValidationIncompleteLineage:
return "checkpoint reuse is disabled by validation-incomplete input lineage"
default:
return "checkpoint decision"
}
}
func normalizeCheckpointDecisionDetail(detail string) string {
detail = strings.TrimSpace(strings.ToValidUTF8(detail, "?"))
var b strings.Builder
for _, r := range detail {
if r < 0x20 || r == 0x7f {
r = ' '
}
if b.Len()+len(string(r)) > checkpointDecisionDetailLimit {
break
}
b.WriteRune(r)
}
return strings.Join(strings.Fields(b.String()), " ")
}
type CheckpointEvent struct {
Stage string `json:"stage"`
StepID string `json:"step_id,omitempty"`
LaneID string `json:"lane_id,omitempty"`
ModuleKey string `json:"module_key,omitempty"`
Action CheckpointDecisionCategory `json:"action"`
Category CheckpointDecisionCategory `json:"category,omitempty"`
ReasonCode CheckpointReasonCode `json:"reason_code,omitempty"`
Detail string `json:"detail,omitempty"`
Reason string `json:"reason,omitempty"`
}
type CheckpointExecutionPolicy struct {
ForcedLanes map[string]struct{}
RequireReusableLanes map[string]struct{}
}
func CheckpointLaneKey(stepID, laneID string) string {
return strings.TrimSpace(stepID) + "\x00" + strings.TrimSpace(laneID)
}
func (policy CheckpointExecutionPolicy) forced(stepID, laneID string) bool {
_, ok := policy.ForcedLanes[CheckpointLaneKey(stepID, laneID)]
return ok
}
func (policy CheckpointExecutionPolicy) requiresReusable(stepID, laneID string) bool {
_, ok := policy.RequireReusableLanes[CheckpointLaneKey(stepID, laneID)]
return ok
}
func forceCheckpointDecision(policy CheckpointExecutionPolicy, stepID, laneID string, decision CheckpointDecision) CheckpointDecision {
if policy.forced(stepID, laneID) {
return checkpointDecision(CheckpointDecisionForcedRecompute, CheckpointReasonRecomputeStep)
}
return decision
}
func requireReusableCheckpoint(policy CheckpointExecutionPolicy, stepID, laneID string, decision CheckpointDecision) error {
if policy.requiresReusable(stepID, laneID) && !policy.forced(stepID, laneID) && !decision.Reused {
return fmt.Errorf("required reusable checkpoint unavailable for step %q lane %q (%s)", strings.TrimSpace(stepID), strings.TrimSpace(laneID), decision.ReasonCode)
}
return nil
}
// resolveCheckpointDecision applies runner policy and canonical payload
// validation at the single point where a stage's observable decision is made.
type checkpointResolution struct {
decision CheckpointDecision
values []any
artifacts []CheckpointArtifact
}
func resolveCheckpointDecision(output *RunOutput, loader CheckpointLoader, policy CheckpointExecutionPolicy, stage ModuleStage, stepID, laneID, moduleKey string, decision CheckpointDecision, codec artifactCodecEntry, artifacts []CheckpointArtifact) (checkpointResolution, error) {
decision = forceCheckpointDecision(policy, stepID, laneID, decision)
resolution := checkpointResolution{decision: decision}
if decision.Reused {
resolution.values = make([]any, 0, len(artifacts))
resolution.artifacts = make([]CheckpointArtifact, 0, len(artifacts))
for _, artifact := range artifacts {
value, hydrated, err := decodeCanonicalCheckpointArtifact(codec, artifact)
if err != nil {
decision = checkpointDecision(CheckpointDecisionExecuted, checkpointArtifactReasonCode(err))
resolution.values = nil
resolution.artifacts = nil
break
}
resolution.values = append(resolution.values, value)
resolution.artifacts = append(resolution.artifacts, hydrated)
}
}
resolution.decision = decision
if output != nil {
recordCheckpointEvent(output, loader, string(stage), stepID, laneID, moduleKey, decision)
}
if err := requireReusableCheckpoint(policy, stepID, laneID, decision); err != nil {
return resolution, err
}
return resolution, nil
}
type SourceCheckpoint struct {
Document *source.SourceDocument
}
// CheckpointArtifact is the durable, domain-neutral value stored at a lane
// checkpoint boundary.
type CheckpointArtifact struct {
LaneID string
ModuleKey string
SourceID string
ChunkID string
ChunkIndex int
ChunkRef source.SourceRef
Artifact contracts.SerializedArtifact
SchemaDigest string
Diagnostics []CheckpointDiagnostic
}
// CheckpointDiagnostic stores a producer-local diagnostic alongside a
// reusable artifact. The runner supplies the current run's origin when it
// promotes this value into a diagnostic group.
type CheckpointDiagnostic struct {
Diagnostic contracts.ProducerDiagnostic `json:"diagnostic"`
ValidatorKey string `json:"validator_key,omitempty"`
}
type ExtractCheckpoint struct {
Outputs []CheckpointArtifact
Rejected []contracts.RejectedOutput
}
type MergeCheckpoint struct {
Output CheckpointArtifact
}
type NormalizeCheckpoint struct {
Output CheckpointArtifact
}
type CheckpointLoader interface {
Enabled() bool
Source(moduleKey string) (SourceCheckpoint, CheckpointDecision)
Extract(laneID string, moduleKey string, dependencies []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision)
Merge(laneID string, moduleKey string, dependencies []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision)
Normalize(laneID string, moduleKey string, dependencies []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision)
AcceptedNormalize(stepID, laneID, moduleKey string) (NormalizeCheckpoint, CheckpointDecision)
}
// StepCheckpointLoader is the step-aware counterpart used by the persistent
// checkpoint implementation. Loaders without this optional interface remain
// usable by framework callers and test doubles through the legacy methods.
type StepCheckpointLoader interface {
ExtractForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision)
MergeForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision)
NormalizeForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision)
}
type noopCheckpointRecorder struct{}
type noopCheckpointLoader struct{}
func NoopCheckpointRecorder() CheckpointRecorder { return noopCheckpointRecorder{} }
func NoopCheckpointLoader() CheckpointLoader { return noopCheckpointLoader{} }
func (noopCheckpointRecorder) SourceRunning(string) error { return nil }
func (noopCheckpointRecorder) SourceSucceeded(string, *source.SourceDocument) error { return nil }
func (noopCheckpointRecorder) SourceFailed(string, error) error { return nil }
func (noopCheckpointRecorder) ExtractRunning(string, string, []CheckpointFingerprint) error {
return nil
}
func (noopCheckpointRecorder) ExtractSucceeded(string, string, []CheckpointFingerprint, []CheckpointArtifact, []contracts.RejectedOutput) error {
return nil
}
func (noopCheckpointRecorder) ExtractFailed(string, string, []CheckpointFingerprint, error) error {
return nil
}
func (noopCheckpointRecorder) MergeRunning(string, string, []CheckpointFingerprint) error { return nil }
func (noopCheckpointRecorder) MergeSucceeded(string, string, []CheckpointFingerprint, CheckpointArtifact) error {
return nil
}
func (noopCheckpointRecorder) MergeRejected(string, string, []CheckpointFingerprint, contracts.RejectedOutput) error {
return nil
}
func (noopCheckpointRecorder) MergeFailed(string, string, []CheckpointFingerprint, error) error {
return nil
}
func (noopCheckpointRecorder) NormalizeRunning(string, string, []CheckpointFingerprint) error {
return nil
}
func (noopCheckpointRecorder) NormalizeSucceeded(string, string, []CheckpointFingerprint, CheckpointArtifact) error {
return nil
}
func (noopCheckpointRecorder) NormalizeRejected(string, string, []CheckpointFingerprint, contracts.RejectedOutput) error {
return nil
}
func (noopCheckpointRecorder) NormalizeFailed(string, string, []CheckpointFingerprint, error) error {
return nil
}
func (noopCheckpointLoader) Enabled() bool { return false }
func (noopCheckpointLoader) Source(string) (SourceCheckpoint, CheckpointDecision) {
return SourceCheckpoint{}, checkpointDecision(CheckpointDecisionExecuted, CheckpointReasonLoadingDisabled)
}
func (noopCheckpointLoader) Extract(string, string, []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
return ExtractCheckpoint{}, checkpointDecision(CheckpointDecisionExecuted, CheckpointReasonLoadingDisabled)
}
func (noopCheckpointLoader) Merge(string, string, []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision) {
return MergeCheckpoint{}, checkpointDecision(CheckpointDecisionExecuted, CheckpointReasonLoadingDisabled)
}
func (noopCheckpointLoader) Normalize(string, string, []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision) {
return NormalizeCheckpoint{}, checkpointDecision(CheckpointDecisionExecuted, CheckpointReasonLoadingDisabled)
}
func (noopCheckpointLoader) AcceptedNormalize(string, string, string) (NormalizeCheckpoint, CheckpointDecision) {
return NormalizeCheckpoint{}, checkpointDecision(CheckpointDecisionExecuted, CheckpointReasonLoadingDisabled)
}
func checkpointExtractRunning(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint) error {
if stepAware, ok := recorder.(StepCheckpointRecorder); ok {
return stepAware.ExtractRunningForStep(stepID, laneID, moduleKey, deps)
}
return recorder.ExtractRunning(laneID, moduleKey, deps)
}
func checkpointExtractSucceeded(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, outputs []CheckpointArtifact, rejected []contracts.RejectedOutput) error {
if stepAware, ok := recorder.(StepCheckpointRecorder); ok {
return stepAware.ExtractSucceededForStep(stepID, laneID, moduleKey, deps, outputs, rejected)
}
return recorder.ExtractSucceeded(laneID, moduleKey, deps, outputs, rejected)
}
func checkpointExtractFailed(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, err error) error {
if stepAware, ok := recorder.(StepCheckpointRecorder); ok {
return stepAware.ExtractFailedForStep(stepID, laneID, moduleKey, deps, err)
}
return recorder.ExtractFailed(laneID, moduleKey, deps, err)
}
func checkpointMergeRunning(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint) error {
if stepAware, ok := recorder.(StepCheckpointRecorder); ok {
return stepAware.MergeRunningForStep(stepID, laneID, moduleKey, deps)
}
return recorder.MergeRunning(laneID, moduleKey, deps)
}
func checkpointMergeSucceeded(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, output CheckpointArtifact) error {
if stepAware, ok := recorder.(StepCheckpointRecorder); ok {
return stepAware.MergeSucceededForStep(stepID, laneID, moduleKey, deps, output)
}
return recorder.MergeSucceeded(laneID, moduleKey, deps, output)
}
func checkpointMergeRejected(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, rejected contracts.RejectedOutput) error {
if stepAware, ok := recorder.(StepCheckpointRecorder); ok {
return stepAware.MergeRejectedForStep(stepID, laneID, moduleKey, deps, rejected)
}
return recorder.MergeRejected(laneID, moduleKey, deps, rejected)
}
func checkpointMergeFailed(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, err error) error {
if stepAware, ok := recorder.(StepCheckpointRecorder); ok {
return stepAware.MergeFailedForStep(stepID, laneID, moduleKey, deps, err)
}
return recorder.MergeFailed(laneID, moduleKey, deps, err)
}
func checkpointNormalizeRunning(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint) error {
if stepAware, ok := recorder.(StepCheckpointRecorder); ok {
return stepAware.NormalizeRunningForStep(stepID, laneID, moduleKey, deps)
}
return recorder.NormalizeRunning(laneID, moduleKey, deps)
}
func checkpointNormalizeSucceeded(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, output CheckpointArtifact) error {
if stepAware, ok := recorder.(StepCheckpointRecorder); ok {
return stepAware.NormalizeSucceededForStep(stepID, laneID, moduleKey, deps, output)
}
return recorder.NormalizeSucceeded(laneID, moduleKey, deps, output)
}
func checkpointNormalizeRejected(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, rejected contracts.RejectedOutput) error {
if stepAware, ok := recorder.(StepCheckpointRecorder); ok {
return stepAware.NormalizeRejectedForStep(stepID, laneID, moduleKey, deps, rejected)
}
return recorder.NormalizeRejected(laneID, moduleKey, deps, rejected)
}
func checkpointNormalizeFailed(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, err error) error {
if stepAware, ok := recorder.(StepCheckpointRecorder); ok {
return stepAware.NormalizeFailedForStep(stepID, laneID, moduleKey, deps, err)
}
return recorder.NormalizeFailed(laneID, moduleKey, deps, err)
}
func digestFingerprints(name string, digest string) []CheckpointFingerprint {
digest = strings.TrimSpace(digest)
if digest == "" {
return nil
}
return []CheckpointFingerprint{{Name: name, Value: digest}}
}
func joinedChunkDigest(chunks []source.Chunk) (string, error) {
if len(chunks) == 0 {
return "", nil
}
values := make([]string, 0, len(chunks))
for _, chunk := range chunks {
digest, err := source.DigestChunk(chunk)
if err != nil {
return "", fmt.Errorf("digest chunk %q: %w", chunk.ID, err)
}
values = append(values, chunk.ID+"="+digest)
}
sort.Strings(values)
sum := sha256.Sum256([]byte(strings.Join(values, "\n")))
return "sha256:" + hex.EncodeToString(sum[:]), nil
}
func normalizeCheckpointFingerprints(values []CheckpointFingerprint) []CheckpointFingerprint {
if len(values) == 0 {
return nil
}
byName := make(map[string]string, len(values))
for _, value := range values {
name := strings.TrimSpace(value.Name)
fingerprint := strings.TrimSpace(value.Value)
if name == "" || fingerprint == "" {
continue
}
byName[name] = fingerprint
}
if len(byName) == 0 {
return nil
}
names := make([]string, 0, len(byName))
for name := range byName {
names = append(names, name)
}
sort.Strings(names)
out := make([]CheckpointFingerprint, 0, len(names))
for _, name := range names {
out = append(out, CheckpointFingerprint{Name: name, Value: byName[name]})
}
return out
}
func checkpointContentDigest(content []byte) string {
sum := sha256.Sum256(content)
return "sha256:" + hex.EncodeToString(sum[:])
}

View File

@@ -1,29 +0,0 @@
package pipeline
import (
"fmt"
"strings"
)
type ChunkCacheMode string
const (
ChunkCacheAuto ChunkCacheMode = "auto"
ChunkCacheBypass ChunkCacheMode = "bypass"
ChunkCacheRefresh ChunkCacheMode = "refresh"
)
func ParseChunkCacheMode(raw string) (ChunkCacheMode, error) {
mode := ChunkCacheMode(strings.TrimSpace(raw))
switch mode {
case ChunkCacheAuto, ChunkCacheBypass, ChunkCacheRefresh:
return mode, nil
default:
return "", fmt.Errorf("chunk cache mode %q is not supported", strings.TrimSpace(raw))
}
}
func (m ChunkCacheMode) Validate() error {
_, err := ParseChunkCacheMode(string(m))
return err
}

View File

@@ -1,20 +0,0 @@
package pipeline
import "testing"
func TestParseChunkCacheMode(t *testing.T) {
for _, value := range []ChunkCacheMode{ChunkCacheAuto, ChunkCacheBypass, ChunkCacheRefresh} {
got, err := ParseChunkCacheMode(string(value))
if err != nil || got != value {
t.Fatalf("ParseChunkCacheMode(%q) = %q, %v", value, got, err)
}
if err := value.Validate(); err != nil {
t.Fatalf("%q.Validate() error = %v", value, err)
}
}
for _, value := range []string{"", "enabled", "AUTO", "auto,refresh"} {
if _, err := ParseChunkCacheMode(value); err == nil {
t.Fatalf("ParseChunkCacheMode(%q) error = nil, want error", value)
}
}
}

View File

@@ -0,0 +1,425 @@
package pipeline
import (
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
// This producer uses real native preparation/continuation and returns domain
// plans; only the provider is faked by the tests.
type cachedPlanChunker struct{ engine *promptkit.Engine }
func (cachedPlanChunker) Key() string { return "cached-plan" }
func (cachedPlanChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (c cachedPlanChunker) Plan(ctx context.Context, request contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
references := map[string][]string{}
for name, slot := range request.References.Slots {
for _, item := range slot.Items {
references[name] = append(references[name], string(item.Content))
}
}
referenceContent, err := json.Marshal(references)
if err != nil {
return contracts.ChunkPlanResult{}, err
}
initial := nativeProducerInitial()
initial.ProfileID, initial.SessionID = request.LLMProfile, request.SessionID
initial.Inputs["source"] = promptkit.Inline(string(request.SourceInput.Content) + "\n" + string(referenceContent))
plan, outcome, err := RunProducer(ctx, c.engine, request.Operation, initial,
func(_ context.Context, result *promptkit.RunResult) (*source.ChunkPlan, []string, error) {
var plan source.ChunkPlan
err := json.Unmarshal([]byte(result.RawOutput), &plan)
return &plan, nil, err
}, producerCorrection, func(context.Context, *source.ChunkPlan) (source.ChunkPlan, error) {
return source.ChunkPlan{SourceDigest: request.Source.Digest, Ranges: []source.ChunkRange{{StartUnitID: request.Source.Units[0].ID, EndUnitID: request.Source.Units[len(request.Source.Units)-1].ID}}}, nil
})
return contracts.ChunkPlanResult{Plan: plan, Outcome: outcome}, err
}
func planCacheRunInput(t *testing.T, engine *promptkit.Engine, validator chunkAssessmentValidator) RunInput {
t.Helper()
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
if err := catalog.Chunkers.RegisterBuilderWithSpec(ModuleSpec{Key: "cached-plan", Stage: StageChunk, ExecutionClass: contracts.ExecutionClassLLMBacked}, rejectUnconfiguredOptions, func(request BuildRequest) (contracts.Chunker, error) {
return cachedPlanChunker{request.Dependencies.Engine}, nil
}); err != nil {
t.Fatal(err)
}
if err := RegisterChunkValidator(catalog.Validators, ValidatorSpec{Key: "plan-policy", ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.ChunkValidator, error) {
return namedPlanValidator{"plan-policy", validator}, nil
}); err != nil {
t.Fatal(err)
}
profile := typedResolutionProfile()
profile.Chunk = Binding("cached-plan")
profile.Chunk.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{Binding("plan-policy")}}
prepared := prepareRunnerProfile(t, catalog, profile, engine)
return RunInput{Prepared: prepared, pipeline: prepared.resolved}
}
func planCacheSource(doc *source.SourceDocument) contracts.LLMInputMaterial {
var texts []string
for _, unit := range doc.Units {
texts = append(texts, unit.Text)
}
return contracts.NewLLMInputMaterial("source", "text/plain", []byte(strings.Join(texts, "\n")), "", "")
}
func planResponse(t *testing.T, plan source.ChunkPlan) *promptkit.GenerateResponse {
t.Helper()
content, err := json.Marshal(plan)
if err != nil {
t.Fatal(err)
}
return &promptkit.GenerateResponse{Content: string(content)}
}
func TestChunkPlanCacheStoresFinalPlanAndReusesWithoutAssessment(t *testing.T) {
root, store := filesystemProducerCache(t)
doc := typedTestDocumentWithUnits(2)
whole := source.ChunkPlan{SourceDigest: doc.Digest, Ranges: []source.ChunkRange{{StartUnitID: 1, EndUnitID: 2}}}
final := typedTestPlan(doc)
calls, assessments := 0, 0
client := producerClientFunc(func(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
if calls == 1 {
return planResponse(t, whole), nil
}
if calls == 2 {
if len(request.Prompt.Messages) != 4 {
t.Fatalf("correction history=%#v", request.Prompt.Messages)
}
}
return planResponse(t, final), nil
})
engine := nativeProducerEngine(t, client)
input := planCacheRunInput(t, engine, func(_ context.Context, request contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
assessments++
if len(request.Chunks) == 1 {
return contracts.ValidationResult{ReasonCode: "partition", CorrectionGuidance: "Use separate scenes."}, nil
}
return contracts.ValidationResult{Approved: true}, nil
})
input.Cache = store
var keys, decisions []string
input.ObserveCache = func(_ context.Context, origin contracts.DiagnosticOrigin, observation contracts.CacheObservation) error {
if origin.Stage != contracts.DiagnosticOriginStageChunk || origin.ModuleKey != "cached-plan" {
t.Fatalf("cache scope=%#v", origin)
}
if observation.Prepared != nil {
keys = append(keys, observation.MessageContentHash)
}
decisions = append(decisions, observation.Decision)
return nil
}
store.beforeWrite = func(_ string, _ []byte) {
if calls != 2 || assessments != 2 {
t.Fatal("plan stored before correction acceptance")
}
}
first, err := New().runChunkPlan(context.Background(), input, doc, planCacheSource(doc), "first-session")
if err != nil || !first.accepted || len(first.chunks) != 2 || calls != 2 || assessments != 2 || store.writes != 1 {
t.Fatalf("first plan=%#v err=%v calls=%d assessments=%d", first, err, calls, assessments)
}
bytes, err := os.ReadFile(filepath.Join(root, keys[0]+".json"))
var saved source.ChunkPlan
if err != nil || json.Unmarshal(bytes, &saved) != nil || !reflect.DeepEqual(saved, *first.plan) {
t.Fatalf("cache is not the final raw plan: %s %v", bytes, err)
}
var fields map[string]json.RawMessage
if err := json.Unmarshal(bytes, &fields); err != nil {
t.Fatal(err)
}
for _, field := range []string{"producer", "checkpoint", "document", "provenance", "usage"} {
if _, ok := fields[field]; ok {
t.Fatalf("cache contains reporting field %q", field)
}
}
store.beforeWrite = nil
input.pipeline.Chunk.LLMProfile = "chosen"
input.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "another-model"}
second, err := New().runChunkPlan(context.Background(), input, doc, planCacheSource(doc), "changed-session")
if err != nil || !second.outcome.ReuseEligible || len(second.chunks) != 2 || calls != 2 || assessments != 2 || len(keys) != 2 || keys[0] != keys[1] {
t.Fatalf("settings-only reuse=%#v %v calls=%d assessments=%d keys=%v", second, err, calls, assessments, keys)
}
if !reflect.DeepEqual(decisions, []string{"prepared", "miss", "stored", "prepared", "hit"}) {
t.Fatalf("cache decisions=%v", decisions)
}
// Materialization is against the current source, not cached transcript bytes.
changed := *doc
changed.ID = "current-source"
changed.Units = append([]source.SourceUnit(nil), doc.Units...)
for i := range changed.Units {
changed.Units[i].Ref.SourceID = changed.ID
}
changed.Digest, _ = source.DigestDocument(&changed)
// Identity is part of ordinary plan usability if it changes the source digest.
// Supplying the matching digest lets this check focus on rematerialized refs.
saved.SourceDigest = changed.Digest
replacement, _ := json.Marshal(saved)
if err := store.inner.Store(context.Background(), contracts.CacheRefresh, keys[0], replacement); err != nil {
t.Fatal(err)
}
third, err := New().runChunkPlan(context.Background(), input, &changed, planCacheSource(doc), "session")
if err != nil || calls != 2 || assessments != 2 || third.chunks[0].Ref.SourceID != changed.ID || third.chunks[0].Units[0].Ref.SourceID != changed.ID {
t.Fatalf("rematerialization=%#v %v", third, err)
}
}
func TestChunkPlanCacheUsesMessageContentRatherThanSourceIdentity(t *testing.T) {
_, store := filesystemProducerCache(t)
doc := typedTestDocumentWithUnits(2)
calls, assessments := 0, 0
client := producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
plan := typedTestPlan(doc)
if calls > 1 {
plan.Ranges = []source.ChunkRange{{StartUnitID: 1, EndUnitID: 2}}
}
return planResponse(t, plan), nil
})
validator := chunkAssessmentValidator(func(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
assessments++
return contracts.ValidationResult{Approved: true}, nil
})
input := planCacheRunInput(t, nativeProducerEngine(t, client), validator)
input.Cache = store
var keys []string
input.ObserveCache = func(_ context.Context, _ contracts.DiagnosticOrigin, observation contracts.CacheObservation) error {
if observation.Prepared != nil {
keys = append(keys, observation.MessageContentHash)
}
return nil
}
run := func(wantChunks int) {
t.Helper()
result, err := New().runChunkPlan(context.Background(), input, doc, planCacheSource(doc), "session")
if err != nil || len(result.chunks) != wantChunks {
t.Fatalf("chunks=%d err=%v", len(result.chunks), err)
}
}
run(2)
input.pipeline.ChunkReferences.ReferenceSet = contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{"context": {Items: []contracts.ReferenceItem{{Content: []byte("new scene context")}}}}}
run(1)
// Replace only the prompt text, retaining source and reference bytes.
input.Prepared.chunker = cachedPlanChunker{nativeProducerEngine(t, client, promptkit.SettingsInput(configkit.Bytes("changed prompt", []byte(`prompts:
- id: initial
default_profile: base
inputs: [{name: source, required: true}]
messages:
- {role: system, content: 'A changed scene assignment.'}
- {role: user, content: 'Source: {{input "source"}}'}
output: {format: json, validation_mode: json, repair_attempts: 0}
`))))}
run(1)
if calls != 3 || assessments != 3 || len(keys) != 3 || keys[0] == keys[1] || keys[1] == keys[2] {
t.Fatalf("content reuse: calls=%d assessments=%d keys=%v", calls, assessments, keys)
}
}
func TestChunkPlanCacheUnusableEntriesAreMissesNotAssistantCandidates(t *testing.T) {
for _, defect := range []string{"source digest", "source range", "malformed", "legacy envelope"} {
t.Run(defect, func(t *testing.T) {
root, store := filesystemProducerCache(t)
doc := typedTestDocument()
calls, assessments := 0, 0
engine := nativeProducerEngine(t, producerClientFunc(func(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
if len(request.Prompt.Messages) != 2 {
t.Fatal("unusable cache replayed as an assistant candidate")
}
return planResponse(t, typedTestPlan(doc)), nil
}))
input := planCacheRunInput(t, engine, func(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
assessments++
return contracts.ValidationResult{Approved: true}, nil
})
input.Cache = store
var key string
input.ObserveCache = func(_ context.Context, _ contracts.DiagnosticOrigin, observation contracts.CacheObservation) error {
if observation.Prepared != nil {
key = observation.MessageContentHash
}
return nil
}
if _, err := New().runChunkPlan(context.Background(), input, doc, planCacheSource(doc), "session"); err != nil {
t.Fatal(err)
}
bad := typedTestPlan(doc)
switch defect {
case "source digest":
bad.SourceDigest = "not-current"
case "source range":
bad.Ranges[0].EndUnitID = 999
}
content, _ := json.Marshal(bad)
if defect == "malformed" {
content = []byte("{")
}
if defect == "legacy envelope" {
content = []byte(`{"plan":{},"producer":{"key":"legacy"}}`)
}
// Simulate an existing corrupt entry without going through valid writes.
if err := os.WriteFile(filepath.Join(root, key+".json"), content, 0644); err != nil {
t.Fatal(err)
}
result, err := New().runChunkPlan(context.Background(), input, doc, planCacheSource(doc), "session")
if err != nil || !result.accepted || calls != 2 || assessments != 2 || len(result.diagnostics) != 1 {
t.Fatalf("unusable plan=%#v err=%v calls=%d assessments=%d", result, err, calls, assessments)
}
})
}
}
func TestChunkPlanCacheModesAndTerminalExclusions(t *testing.T) {
for _, test := range []struct {
name string
cache contracts.CacheMode
mode contracts.ExecutionMode
reject, ineligible bool
reads, writes int
fallback bool
}{
{"auto", contracts.CacheAuto, contracts.ModeStrict, false, false, 1, 1, false},
{"bypass", contracts.CacheBypass, contracts.ModeStrict, false, false, 0, 0, false},
{"refresh", contracts.CacheRefresh, contracts.ModeStrict, false, false, 0, 1, false},
{"ineligible", contracts.CacheAuto, contracts.ModeStrict, false, true, 0, 0, false},
{"strict rejection", contracts.CacheAuto, contracts.ModeStrict, true, false, 1, 0, false},
{"permissive fallback", contracts.CacheAuto, contracts.ModePermissive, true, false, 1, 0, true},
} {
t.Run(test.name, func(t *testing.T) {
_, store := filesystemProducerCache(t)
doc := typedTestDocumentWithUnits(2)
calls, assessments := 0, 0
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
return planResponse(t, typedTestPlan(doc)), nil
}))
input := planCacheRunInput(t, engine, func(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
assessments++
return contracts.ValidationResult{Approved: !test.reject, ReasonCode: "policy", CorrectionGuidance: "Revise the scene partition."}, nil
})
input.Cache = store
input.reuseIneligible = test.ineligible
input.pipeline.ChunkControls = contracts.ProducerControls{Cache: test.cache, Mode: test.mode, MaxAdditionalTurns: 0}
result, err := New().runChunkPlan(context.Background(), input, doc, planCacheSource(doc), "session")
if (err != nil) != (test.reject && !test.fallback) || calls != 1 || assessments != 1 || store.reads != test.reads || store.writes != test.writes {
t.Fatalf("result=%#v err=%v calls=%d assessments=%d cache=%d/%d", result, err, calls, assessments, store.reads, store.writes)
}
if test.fallback && (result.outcome.Disposition != contracts.ProducerFallback || result.outcome.ReuseEligible || len(result.chunks) != 1 || len(result.chunks[0].Units) != 2) {
t.Fatalf("invalid whole-source fallback: %#v", result)
}
})
}
}
func TestChunkPlanFailedRefreshPreservesAcceptanceAndIgnoresLegacyRecords(t *testing.T) {
root, store := filesystemProducerCache(t)
legacyNames := []string{"checkpoints/run/manifest.json", "chunk-plans/source/plan.json"}
for _, name := range legacyNames {
filename := filepath.Join(root, name)
if err := os.MkdirAll(filepath.Dir(filename), 0755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filename, []byte("unread legacy bytes"), 0644); err != nil {
t.Fatal(err)
}
}
doc := typedTestDocumentWithUnits(2)
reject := false
calls, assessments := 0, 0
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
return planResponse(t, typedTestPlan(doc)), nil
}))
input := planCacheRunInput(t, engine, func(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
assessments++
return contracts.ValidationResult{Approved: !reject, ReasonCode: "partition", CorrectionGuidance: "Reconsider the partition."}, nil
})
input.Cache = store
input.pipeline.ChunkControls.MaxAdditionalTurns = 0
var key string
input.ObserveCache = func(_ context.Context, _ contracts.DiagnosticOrigin, observation contracts.CacheObservation) error {
if observation.Prepared != nil {
key = observation.MessageContentHash
}
return nil
}
if _, err := New().runChunkPlan(context.Background(), input, doc, planCacheSource(doc), "session"); err != nil {
t.Fatal(err)
}
original, err := os.ReadFile(filepath.Join(root, key+".json"))
if err != nil {
t.Fatal(err)
}
reject = true
input.pipeline.ChunkControls.Cache = contracts.CacheRefresh
for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} {
input.pipeline.ChunkControls.Mode = mode
result, err := New().runChunkPlan(context.Background(), input, doc, planCacheSource(doc), "session")
if (err != nil) != (mode == contracts.ModeStrict) {
t.Fatalf("refresh mode=%s result=%#v error=%v", mode, result, err)
}
if mode == contracts.ModePermissive && (result.outcome.ReuseEligible || result.outcome.Disposition != contracts.ProducerFallback) {
t.Fatalf("fallback=%#v", result.outcome)
}
current, err := os.ReadFile(filepath.Join(root, key+".json"))
if err != nil || string(current) != string(original) {
t.Fatal("failed refresh changed old acceptance")
}
}
input.pipeline.ChunkControls.Cache = contracts.CacheAuto
result, err := New().runChunkPlan(context.Background(), input, doc, planCacheSource(doc), "session")
if err != nil || result.outcome.Disposition != contracts.ProducerAccepted || calls != 3 || assessments != 3 || store.reads != 2 || store.writes != 1 {
t.Fatalf("old acceptance not reused: %#v %v calls=%d assessments=%d cache=%d/%d", result, err, calls, assessments, store.reads, store.writes)
}
for _, name := range legacyNames {
content, err := os.ReadFile(filepath.Join(root, name))
if err != nil || string(content) != "unread legacy bytes" {
t.Fatalf("legacy record modified: %s %q %v", name, content, err)
}
}
}
func TestDeterministicChunkingIgnoresCacheAndLegacyRecords(t *testing.T) {
root, store := filesystemProducerCache(t)
legacy := map[string][]byte{"checkpoints/identity/manifest.json": []byte("old checkpoint"), "chunk-plans/source.json": []byte("old plan")}
for name, content := range legacy {
filename := filepath.Join(root, name)
if err := os.MkdirAll(filepath.Dir(filename), 0755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filename, content, 0644); err != nil {
t.Fatal(err)
}
}
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
prepared := prepareRunnerProfile(t, catalog, typedResolutionProfile(), nil)
input := RunInput{Prepared: prepared, RawInput: []byte("input"), Cache: store, ObserveCache: func(context.Context, contracts.DiagnosticOrigin, contracts.CacheObservation) error {
return fmt.Errorf("deterministic chunker prepared a synthetic request")
}}
for range 2 {
if _, err := New().Run(context.Background(), input); err != nil {
t.Fatal(err)
}
}
if store.reads != 0 || store.writes != 0 {
t.Fatalf("deterministic cache I/O=%d/%d", store.reads, store.writes)
}
for name, want := range legacy {
got, err := os.ReadFile(filepath.Join(root, name))
if err != nil || string(got) != string(want) {
t.Fatalf("legacy record modified: %s %q %v", name, got, err)
}
}
}

View File

@@ -1,49 +0,0 @@
package pipeline
import (
"time"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
const ChunkPlanSchemaVersion = "notarius.chunk-plan.v3"
type ChunkPlanProducer struct {
InputModule string `json:"input_module"`
ChunkModule string `json:"chunk_module"`
LLMProfile string `json:"llm_profile,omitempty"`
References []artifacts.ReferenceProvenance `json:"references,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
type ChunkPlanRecord struct {
SchemaVersion string `json:"schema_version"`
SourceDigest string `json:"source_digest"`
PlanDigest string `json:"plan_digest"`
Plan source.ChunkPlan `json:"plan"`
Producer ChunkPlanProducer `json:"producer"`
Diagnostics []contracts.ProducerDiagnostic `json:"diagnostics,omitempty"`
CreatedAt time.Time `json:"created_at"`
}
type ChunkPlanStore interface {
Load(sourceDigest string) (ChunkPlanRecord, ChunkPlanDecision, error)
Save(ChunkPlanRecord) error
}
type ChunkPlanStoreFactory func(root string) (ChunkPlanStore, error)
type ChunkPlanDecision struct {
Status ChunkPlanStatus `json:"status"`
Reason string `json:"reason,omitempty"`
}
type ChunkPlanStatus string
const (
ChunkPlanHit ChunkPlanStatus = "hit"
ChunkPlanMissing ChunkPlanStatus = "missing"
ChunkPlanInvalid ChunkPlanStatus = "invalid"
)

View File

@@ -1,41 +1,11 @@
package pipeline
import (
"fmt"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
frameworkdiagnostics "gitea.maximumdirect.net/eric/notarius/internal/framework/diagnostics"
)
func promoteCheckpointDiagnostics(diagnostics []CheckpointDiagnostic, origin contracts.DiagnosticOrigin, chunk *source.Chunk) ([]contracts.DiagnosticGroup, error) {
groups := make([]contracts.DiagnosticGroup, 0, len(diagnostics))
for _, record := range diagnostics {
validatorOrigin := origin
validatorOrigin.ValidatorKey = record.ValidatorKey
promoted, err := promoteProducerDiagnostics([]contracts.ProducerDiagnostic{record.Diagnostic}, validatorOrigin, chunk)
if err != nil {
return nil, fmt.Errorf("checkpoint diagnostic: %w", err)
}
groups = append(groups, promoted...)
}
return groups, nil
}
func cloneCheckpointDiagnostics(diagnostics []CheckpointDiagnostic) []CheckpointDiagnostic {
if len(diagnostics) == 0 {
return nil
}
cloned := make([]CheckpointDiagnostic, len(diagnostics))
for index, diagnostic := range diagnostics {
cloned[index] = CheckpointDiagnostic{
Diagnostic: contracts.CloneProducerDiagnostics([]contracts.ProducerDiagnostic{diagnostic.Diagnostic})[0],
ValidatorKey: diagnostic.ValidatorKey,
}
}
return cloned
}
func promoteProducerDiagnostics(diagnostics []contracts.ProducerDiagnostic, origin contracts.DiagnosticOrigin, chunk *source.Chunk) ([]contracts.DiagnosticGroup, error) {
if err := contracts.ValidateProducerDiagnostics(diagnostics); err != nil {
return nil, err

View File

@@ -26,7 +26,7 @@ type debugEvidenceContextSummary struct {
// buildOutputEvidenceContext projects the prepared output policy from accepted
// normalized artifacts. It is intentionally separate from lane execution so
// checkpointed normalized outputs use the same reconstruction path.
// cached normalized outputs use the same reconstruction path.
func buildOutputEvidenceContext(prepared *PreparedPipeline, doc *source.SourceDocument, outputs []contracts.SerializedOutput) (*contracts.SerializedArtifact, *debugEvidenceContextSummary, error) {
if prepared == nil || prepared.evidencePlan == nil {
return nil, nil, nil

View File

@@ -1,9 +1,6 @@
package pipeline
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"sort"
"strings"
@@ -301,60 +298,3 @@ func referenceProvenanceForItem(target ResolvedReferenceTarget, item contracts.R
ProducerModule: item.Producer.ModuleKey,
}
}
type generatedReferenceFingerprintIdentity struct {
ProducerPipeline string `json:"producer_pipeline_id"`
ProducerStep string `json:"producer_step_id"`
ProducerLane string `json:"producer_lane_id"`
ProducerModule string `json:"producer_module_key"`
ArtifactKind string `json:"artifact_kind"`
SchemaID string `json:"schema_id"`
SchemaName string `json:"schema_name"`
SchemaVersion string `json:"schema_version"`
SchemaDigest string `json:"schema_digest"`
MediaType string `json:"media_type"`
ContentDigest string `json:"content_digest"`
SizeBytes int64 `json:"size_bytes"`
}
// generatedReferenceDependencies returns the canonical semantic dependency
// identity that receiving operation checkpoints must include.
func generatedReferenceDependencies(set contracts.ReferenceSet) []CheckpointFingerprint {
var fingerprints []CheckpointFingerprint
keys := make([]string, 0, len(set.Slots))
for key := range set.Slots {
keys = append(keys, key)
}
sort.Strings(keys)
for _, slotName := range keys {
for index, item := range set.Slots[slotName].Items {
if item.Origin.Type != "generated" {
continue
}
identity := generatedReferenceFingerprintIdentity{
ProducerPipeline: item.Producer.PipelineID,
ProducerStep: item.Producer.StepID,
ProducerLane: item.Producer.LaneID,
ProducerModule: item.Producer.ModuleKey,
ArtifactKind: string(item.ArtifactKind),
SchemaID: item.ArtifactSchema.ID,
SchemaName: item.ArtifactSchema.Name,
SchemaVersion: item.ArtifactSchema.Version,
SchemaDigest: contracts.DigestArtifactSchema(item.ArtifactSchema),
MediaType: item.MediaType,
ContentDigest: item.Digest,
SizeBytes: item.SizeBytes,
}
encoded, err := json.Marshal(identity)
if err != nil {
continue
}
sum := sha256.Sum256(encoded)
fingerprints = append(fingerprints, CheckpointFingerprint{
Name: fmt.Sprintf("generated-reference:%s:%d", slotName, index),
Value: "sha256:" + hex.EncodeToString(sum[:]),
})
}
}
return normalizeCheckpointFingerprints(fingerprints)
}

View File

@@ -19,15 +19,14 @@ type PreparedPipeline struct {
Steps []PreparedPipelineStep
Output ModuleBinding
resolved ResolvedPipeline
dependencies ModuleDependencies
input contracts.InputAdapter
chunker contracts.Chunker
chunkValidators preparedValidatorChain
output contracts.OutputEncoder
artifactCodecs *ArtifactCodecRegistry
evidencePlan *preparedEvidencePlan
checkpointFingerprints []CheckpointFingerprint
resolved ResolvedPipeline
dependencies ModuleDependencies
input contracts.InputAdapter
chunker contracts.Chunker
chunkValidators preparedValidatorChain
output contracts.OutputEncoder
artifactCodecs *ArtifactCodecRegistry
evidencePlan *preparedEvidencePlan
}
type PreparedPipelineStep struct {
@@ -158,10 +157,6 @@ func Prepare(resolved ResolvedPipeline, registries Registries, deps ModuleDepend
if err != nil {
return nil, err
}
prepared.checkpointFingerprints, err = collectPreparedCheckpointFingerprints(prepared)
if err != nil {
return nil, err
}
return prepared, nil
}

View File

@@ -1,88 +0,0 @@
package pipeline
import (
"fmt"
"sort"
"strings"
)
type checkpointFingerprintComponent struct {
scope string
module any
}
// CheckpointFingerprints returns a defensive copy of the stable semantic
// identities contributed by the pipeline's prepared modules and validators.
func (p *PreparedPipeline) CheckpointFingerprints() []CheckpointFingerprint {
if p == nil {
return nil
}
return append([]CheckpointFingerprint(nil), p.checkpointFingerprints...)
}
func collectPreparedCheckpointFingerprints(prepared *PreparedPipeline) ([]CheckpointFingerprint, error) {
components := []checkpointFingerprintComponent{
{scope: "input:" + prepared.resolved.Input.Module, module: prepared.input},
{scope: "chunk:" + prepared.resolved.Chunk.Module, module: prepared.chunker},
}
components = appendValidatorFingerprintComponents(components, "chunk:"+prepared.resolved.Chunk.Module, prepared.chunkValidators)
for _, step := range prepared.Steps {
for _, lane := range step.lanes {
laneScope := func(stage ModuleStage, moduleKey string) string {
return string(stage) + ":" + lane.resolved.ID + ":" + moduleKey
}
components = append(components, checkpointFingerprintComponent{scope: laneScope(StageExtract, lane.resolved.Extract.Module), module: lane.typed.extractor})
components = appendValidatorFingerprintComponents(components, laneScope(StageExtract, lane.resolved.Extract.Module), lane.extractValidators)
components = append(components, checkpointFingerprintComponent{scope: laneScope(StageMerge, lane.resolved.Merge.Module), module: lane.typed.merger})
components = appendValidatorFingerprintComponents(components, laneScope(StageMerge, lane.resolved.Merge.Module), lane.mergeValidators)
components = append(components, checkpointFingerprintComponent{scope: laneScope(StageNormalize, lane.resolved.Normalize.Module), module: lane.typed.normalizer})
components = appendValidatorFingerprintComponents(components, laneScope(StageNormalize, lane.resolved.Normalize.Module), lane.normalizeValidators)
}
}
components = append(components, checkpointFingerprintComponent{scope: "output:" + prepared.resolved.Output.Module, module: prepared.output})
var values []CheckpointFingerprint
seen := make(map[string]struct{})
for _, item := range components {
provider, ok := item.module.(CheckpointFingerprintProvider)
if !ok {
continue
}
for _, fingerprint := range provider.CheckpointFingerprints() {
name := strings.TrimSpace(fingerprint.Name)
value := strings.TrimSpace(fingerprint.Value)
if name == "" || value == "" {
return nil, fmt.Errorf("pipeline %q checkpoint fingerprint for %s must have a non-empty name and value", prepared.resolved.ID, item.scope)
}
scopedName := item.scope + ":" + name
if _, exists := seen[scopedName]; exists {
return nil, fmt.Errorf("pipeline %q checkpoint fingerprint %q is duplicated", prepared.resolved.ID, scopedName)
}
seen[scopedName] = struct{}{}
values = append(values, CheckpointFingerprint{Name: scopedName, Value: value})
}
}
sort.Slice(values, func(i, j int) bool { return values[i].Name < values[j].Name })
return values, nil
}
func appendValidatorFingerprintComponents(components []checkpointFingerprintComponent, ownerScope string, chain preparedValidatorChain) []checkpointFingerprintComponent {
for index, validator := range chain.validators {
scope := fmt.Sprintf("%s:validator:%d:%s", ownerScope, index, validator.resolved.Binding.Module)
components = append(components, checkpointFingerprintComponent{scope: scope, module: preparedValidatorImplementation(validator)})
}
return components
}
func preparedValidatorImplementation(validator preparedValidator) any {
switch validator.resolved.Target {
case ValidatorTargetTyped:
return validator.typed
case ValidatorTargetChunk:
return validator.chunk
case ValidatorTargetSerialized:
return validator.serialized
default:
return nil
}
}

View File

@@ -44,24 +44,20 @@ type RunInput struct {
Cache contracts.ArtifactCache
// ObserveCache receives scoped transient evidence, including prepared details
// on a hit. Shared observers must synchronize concurrent calls.
ObserveCache func(context.Context, contracts.DiagnosticOrigin, contracts.CacheObservation) error
Execution contracts.ExecutionOptions
Prepared *PreparedPipeline
SourceID string
Path string
RawInput []byte
SessionID string
RunID string
StartedAt time.Time
LLMProfiles []artifacts.LLMProfileManifest
Metadata map[string]any
Diagnostics []contracts.ProducerDiagnostic
ChunkCacheMode ChunkCacheMode
ChunkPlans ChunkPlanStore
Checkpoints CheckpointRecorder
Checkpoint CheckpointLoader
CheckpointPolicy CheckpointExecutionPolicy
Debug DebugRecorder
ObserveCache func(context.Context, contracts.DiagnosticOrigin, contracts.CacheObservation) error
Execution contracts.ExecutionOptions
Prepared *PreparedPipeline
SourceID string
Path string
RawInput []byte
SessionID string
RunID string
StartedAt time.Time
LLMProfiles []artifacts.LLMProfileManifest
Metadata map[string]any
Diagnostics []contracts.ProducerDiagnostic
Debug DebugRecorder
// Workers bounds ready extraction and lane-completion jobs together.
// Zero selects DefaultWorkers; configuration rejects explicit zero.
Workers int
@@ -75,15 +71,15 @@ type RunInput struct {
}
type RunOutput struct {
FallbackCount int `json:"fallback_count,omitempty"`
Manifest artifacts.RunManifest `json:"manifest"`
ChunkPlan *artifacts.ChunkPlanSummary `json:"chunk_plan,omitempty"`
NormalizeOutputs []contracts.SerializedOutput `json:"normalize_outputs,omitempty"`
Rejected []contracts.RejectedOutput `json:"rejected,omitempty"`
Diagnostics contracts.DiagnosticCollection `json:"diagnostics,omitempty"`
OutputFiles []contracts.OutputFile `json:"-"`
CheckpointEvents []CheckpointEvent `json:"checkpoint_events,omitempty"`
ValidationSummaries []artifacts.ValidationSummary `json:"validation_summaries,omitempty"`
FallbackCount int `json:"fallback_count,omitempty"`
Manifest artifacts.RunManifest `json:"manifest"`
NormalizeOutputs []contracts.SerializedOutput `json:"normalize_outputs,omitempty"`
Rejected []contracts.RejectedOutput `json:"rejected,omitempty"`
Diagnostics contracts.DiagnosticCollection `json:"diagnostics,omitempty"`
OutputFiles []contracts.OutputFile `json:"-"`
ValidationSummaries []artifacts.ValidationSummary `json:"validation_summaries,omitempty"`
normalizeReuseEligibility map[generatedOutputKey]bool
diagnosticGroups []contracts.DiagnosticGroup
@@ -111,25 +107,10 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
if err != nil {
return failOutput(output), err
}
output.ChunkPlan = &artifacts.ChunkPlanSummary{
Mode: string(effectiveChunkCacheMode(input.ChunkCacheMode)), RequestedModule: input.pipeline.Chunk.Module,
LookupStatus: "skipped", LookupReason: "chunk plan lookup skipped",
ValidationStatus: "not_run", PublicationStatus: "not_requested",
}
checkpoints := input.Checkpoints
if checkpoints == nil {
checkpoints = NoopCheckpointRecorder()
}
checkpointLoader := input.Checkpoint
if checkpointLoader == nil {
checkpointLoader = NoopCheckpointLoader()
}
debugRecorder := input.Debug
if debugRecorder == nil {
debugRecorder = NoopDebugRecorder()
}
checkpoints = synchronizedCheckpointRecorder(checkpoints)
checkpointLoader = synchronizedCheckpointLoader(checkpointLoader)
debugRecorder = synchronizedDebugRecorder(debugRecorder)
input.Debug = debugRecorder
inputDiagnostics, diagnosticErr := promoteProducerDiagnostics(input.Diagnostics, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageReferences}, nil)
@@ -153,9 +134,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
if err := attachModuleManifestMetadata(&output, "input", adapter); err != nil {
return failOutput(output), err
}
sourceCheckpoint, sourceDecision := checkpointLoader.Source(adapter.Key())
recordCheckpointEvent(&output, checkpointLoader, "source", "", "", adapter.Key(), sourceDecision)
doc := sourceCheckpoint.Document
var doc *source.SourceDocument
sourceStarted := time.Now().UTC()
if err := writeDebugTimed(debugRecorder, "source/input.json", debugTimedEnvelope{
Stage: "source",
@@ -171,48 +150,32 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
}); err != nil {
return failOutput(output), fmt.Errorf("write source debug artifact: %w", err)
}
if !sourceDecision.Reused {
if err := checkpoints.SourceRunning(adapter.Key()); err != nil {
return failOutput(output), fmt.Errorf("write source checkpoint: %w", err)
}
requestMetadata, metadataErr := cloneMetadata(input.Metadata)
if metadataErr != nil {
return failOutput(output), fmt.Errorf("clone input adapter metadata: %w", metadataErr)
}
if err := ctx.Err(); err != nil {
_ = checkpoints.SourceFailed(adapter.Key(), err)
return failOutput(output), err
}
doc, err = adapter.Parse(ctx, contracts.ParseRequest{
SourceID: input.SourceID,
Path: input.Path,
Raw: input.RawInput,
LLMProfile: input.pipeline.Input.LLMProfile,
Metadata: requestMetadata,
})
if ctxErr := ctx.Err(); ctxErr != nil {
_ = checkpoints.SourceFailed(adapter.Key(), ctxErr)
return failOutput(output), ctxErr
}
if err != nil {
_ = checkpoints.SourceFailed(adapter.Key(), err)
return failOutput(output), fmt.Errorf("parse input with adapter %q: %w", adapter.Key(), err)
}
if err := source.ValidateDocument(doc); err != nil {
_ = checkpoints.SourceFailed(adapter.Key(), err)
return failOutput(output), fmt.Errorf("validate source document: %w", err)
}
if err := checkpoints.SourceSucceeded(adapter.Key(), doc); err != nil {
return failOutput(output), fmt.Errorf("write source checkpoint: %w", err)
}
requestMetadata, err := cloneMetadata(input.Metadata)
if err != nil {
return failOutput(output), fmt.Errorf("clone input adapter metadata: %w", err)
}
if err := ctx.Err(); err != nil {
return failOutput(output), err
}
doc, err = adapter.Parse(ctx, contracts.ParseRequest{
SourceID: input.SourceID, Path: input.Path, Raw: input.RawInput,
LLMProfile: input.pipeline.Input.LLMProfile, Metadata: requestMetadata,
})
if ctxErr := ctx.Err(); ctxErr != nil {
return failOutput(output), ctxErr
}
if err != nil {
return failOutput(output), fmt.Errorf("parse input with adapter %q: %w", adapter.Key(), err)
}
if err := source.ValidateDocument(doc); err != nil {
return failOutput(output), fmt.Errorf("validate source document: %w", err)
}
if err := writeDebugTimed(debugRecorder, "source/output.json", debugTimedEnvelope{
Stage: "source",
ModuleKey: adapter.Key(),
StartedAt: sourceStarted,
Payload: map[string]any{
"reused": sourceDecision.Reused,
"decision": sourceDecision,
"document": debugSourceDocumentEnvelope(doc),
},
}); err != nil {
@@ -226,13 +189,12 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
return failOutput(output), err
}
chunkStarted := time.Now().UTC()
chunkMode := effectiveChunkCacheMode(input.ChunkCacheMode)
if err := writeDebugTimed(debugRecorder, "chunk/input.json", debugTimedEnvelope{
Stage: string(StageChunk),
ModuleKey: chunker.Key(),
StartedAt: chunkStarted,
Payload: map[string]any{
"cache_mode": chunkMode,
"cache_mode": input.pipeline.ChunkControls.Cache,
"source": debugSourceDocumentEnvelope(doc),
"source_input": debugContentEnvelope(sourceInput.Content, sourceInput.MediaType, nil, nil),
"options": redactSensitiveMap(input.pipeline.Chunk.Options),
@@ -242,31 +204,18 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
return failOutput(output), fmt.Errorf("write chunk debug artifact: %w", err)
}
chunkResult, err := r.runChunkPlan(ctx, input, doc, sourceInput, sessionID)
if applyErr := applyChunkPlanExecution(&output, chunkResult); applyErr != nil {
return failOutput(output), applyErr
}
if err != nil {
return failOutput(output), err
}
if chunkResult.rejection != nil {
output.Rejected = append(output.Rejected, *chunkResult.rejection)
}
if chunkResult.validation != nil {
output.ValidationSummaries = append(output.ValidationSummaries, artifacts.CloneValidationSummary(*chunkResult.validation))
}
appendDiagnosticGroups(&output, chunkResult.diagnostics)
chunkDebugPayload := map[string]any{
"cache_mode": chunkMode,
"lookup": chunkResult.lookup,
"cache_mode": input.pipeline.ChunkControls.Cache,
"accepted": chunkResult.accepted,
"materialized_chunks": debugSourceChunkEnvelopes(chunkResult.chunks),
}
if chunkResult.plan != nil {
chunkDebugPayload["plan"] = debugChunkPlanEnvelope(*chunkResult.plan)
}
if chunkResult.rejection != nil {
chunkDebugPayload["rejection"] = debugRejectedOutputEnvelope(*chunkResult.rejection)
}
if err := writeDebugTimed(debugRecorder, "chunk/output.json", debugTimedEnvelope{
Stage: string(StageChunk),
ModuleKey: chunker.Key(),
@@ -278,8 +227,12 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
var acceptedChunkMap *contracts.SerializedArtifact
if chunkResult.accepted {
if chunkResult.plan == nil || chunkResult.record == nil {
return failOutput(output), fmt.Errorf("accepted chunk plan is missing plan or producer record")
if chunkResult.plan == nil {
return failOutput(output), fmt.Errorf("accepted chunk plan is missing")
}
chunkProfile := ""
if input.pipeline.ChunkExecutionClass == contracts.ExecutionClassLLMBacked {
chunkProfile = input.pipeline.Chunk.LLMProfile
}
artifact, buildErr := chunkmap.Serialize(chunkmap.BuildRequest{
Source: doc,
@@ -287,9 +240,9 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
Chunks: chunkResult.chunks,
RequestedChunker: input.pipeline.Chunk.Module,
Producer: chunkmap.Producer{
InputModule: chunkResult.record.Producer.InputModule,
ChunkModule: chunkResult.record.Producer.ChunkModule,
LLMProfile: chunkResult.record.Producer.LLMProfile,
InputModule: adapter.Key(),
ChunkModule: chunker.Key(),
LLMProfile: chunkProfile,
},
})
if buildErr != nil {
@@ -303,7 +256,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
if chunkResult.outcome.Disposition == contracts.ProducerFallback {
output.FallbackCount++
}
if err := r.runPreparedSteps(ctx, input, checkpoints, checkpointLoader, doc, sourceInput, sessionID, chunkResult.chunks, &output); err != nil {
if err := r.runPreparedSteps(ctx, input, doc, sourceInput, sessionID, chunkResult.chunks, &output); err != nil {
return failOutput(output), err
}
}
@@ -427,7 +380,7 @@ func hasIncompleteValidation(summaries []artifacts.ValidationSummary) bool {
return false
}
func (r *Runner) runPreparedSteps(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, loader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk, output *RunOutput) error {
func (r *Runner) runPreparedSteps(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk, output *RunOutput) error {
for _, step := range input.Prepared.Steps {
stepInput := input
stepInput.stepID = step.ID
@@ -438,7 +391,7 @@ func (r *Runner) runPreparedSteps(ctx context.Context, input RunInput, checkpoin
stepInput.references = stepReferences
stepInput.referenceReuseEligibility = buildStepReferenceReuseEligibility(step, output.normalizeReuseEligibility)
output.Manifest.References = append(output.Manifest.References, referenceProvenance...)
laneOutput, laneErr := r.runLanes(ctx, stepInput, step, checkpoints, loader, doc, sourceInput, sessionID, chunks)
laneOutput, laneErr := r.runLanes(ctx, stepInput, step, doc, sourceInput, sessionID, chunks)
if err := mergeLaneOutput(output, laneOutput); err != nil {
return err
}
@@ -481,13 +434,6 @@ func validateRunInput(input RunInput) error {
if input.Prepared == nil {
return fmt.Errorf("prepared pipeline must not be nil")
}
mode := effectiveChunkCacheMode(input.ChunkCacheMode)
if err := mode.Validate(); err != nil {
return err
}
if mode != ChunkCacheBypass && input.ChunkPlans == nil {
return fmt.Errorf("chunk plan store is required for %q mode", mode)
}
if err := validateResolvedPipeline(input.Prepared.resolved); err != nil {
return err
}
@@ -557,7 +503,7 @@ func manifestFromPipeline(input RunInput) artifacts.RunManifest {
InputModule: pipeline.Input.Module,
Chunker: pipeline.Chunk.Module,
ChunkPlan: &artifacts.ChunkPlanManifest{
Mode: string(effectiveChunkCacheMode(input.ChunkCacheMode)),
Mode: string(pipeline.ChunkControls.Cache),
RequestedModule: pipeline.Chunk.Module,
},
OutputEncoder: pipeline.Output.Module,
@@ -617,34 +563,6 @@ func failOutput(output RunOutput) RunOutput {
return output
}
func recordCheckpointEvent(output *RunOutput, loader CheckpointLoader, stage string, stepID string, laneID string, moduleKey string, decision CheckpointDecision) {
if output == nil || loader == nil || !loader.Enabled() {
return
}
action := checkpointDecisionCategory(decision)
output.CheckpointEvents = append(output.CheckpointEvents, CheckpointEvent{
Stage: stage,
StepID: stepID,
LaneID: laneID,
ModuleKey: moduleKey,
Action: action,
Category: checkpointDecisionCategory(decision),
ReasonCode: decision.ReasonCode,
Detail: decision.Detail,
Reason: decision.Reason,
})
}
func checkpointDecisionCategory(decision CheckpointDecision) CheckpointDecisionCategory {
if decision.Category != "" {
return decision.Category
}
if decision.Reused {
return CheckpointDecisionReused
}
return CheckpointDecisionExecuted
}
func populateOutputManifest(output *RunOutput) {
if output == nil {
return
@@ -654,13 +572,6 @@ func populateOutputManifest(output *RunOutput) {
if len(output.ValidationSummaries) > 0 {
output.Manifest.ValidationSummaries = cloneValidationSummaries(output.ValidationSummaries)
}
if len(output.CheckpointEvents) > 0 {
decisions := make([]artifacts.CheckpointDecisionManifest, 0, len(output.CheckpointEvents))
for _, event := range output.CheckpointEvents {
decisions = append(decisions, artifacts.CheckpointDecisionManifest{Stage: event.Stage, StepID: event.StepID, LaneID: event.LaneID, ModuleKey: event.ModuleKey, Category: string(event.Category), ReasonCode: string(event.ReasonCode), Detail: event.Detail})
}
output.Manifest.CheckpointDecisions = decisions
}
}
func normalizedOutputManifests(outputs []contracts.SerializedOutput) []artifacts.NormalizedOutputManifest {

View File

@@ -1,396 +0,0 @@
package pipeline
import (
"context"
"encoding/json"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
type acceptedCheckpointLoader struct {
CheckpointLoader
accepted map[string]NormalizeCheckpoint
acceptedDecision map[string]CheckpointDecision
extractDeps map[string][]CheckpointFingerprint
}
func newAcceptedCheckpointLoader() *acceptedCheckpointLoader {
return &acceptedCheckpointLoader{
CheckpointLoader: NoopCheckpointLoader(),
accepted: make(map[string]NormalizeCheckpoint),
acceptedDecision: make(map[string]CheckpointDecision),
extractDeps: make(map[string][]CheckpointFingerprint),
}
}
func (l *acceptedCheckpointLoader) Enabled() bool { return true }
func (l *acceptedCheckpointLoader) AcceptedNormalize(stepID, laneID, _ string) (NormalizeCheckpoint, CheckpointDecision) {
key := CheckpointLaneKey(stepID, laneID)
return l.accepted[key], l.acceptedDecision[key]
}
func (l *acceptedCheckpointLoader) Extract(laneID string, _ string, dependencies []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
l.extractDeps[laneID] = append([]CheckpointFingerprint(nil), dependencies...)
return ExtractCheckpoint{}, NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonMissing)
}
func (l *acceptedCheckpointLoader) Merge(_ string, _ string, _ []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision) {
return MergeCheckpoint{}, NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonMissing)
}
func (l *acceptedCheckpointLoader) Normalize(_ string, _ string, _ []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision) {
return NormalizeCheckpoint{}, NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonMissing)
}
func TestRunnerHydratesRequiredNormalizedArtifact(t *testing.T) {
value := codecNotes{Items: []string{"canonical producer value"}}
input, _, _ := handoffFixture(t, value)
prepared := input.Prepared
producer := &prepared.Steps[0].lanes[0]
consumer := &prepared.Steps[1].lanes[0]
doc := prepared.input.(*typedTestInput).doc
stored, err := checkpointArtifact(producer.typed.codec, producer.resolved.ID, producer.resolved.Normalize.Module, doc.ID, value)
if err != nil {
t.Fatal(err)
}
operationCalls := 0
producer.typed.extract = func(context.Context, any, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
operationCalls++
return erasedTypedResult{Value: value}, nil
}
producer.typed.merge = func(context.Context, any, contracts.TypedMergeRequest[any]) (erasedTypedResult, error) {
operationCalls++
return erasedTypedResult{Value: value}, nil
}
producer.typed.normalize = func(context.Context, any, contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) {
operationCalls++
return erasedTypedResult{Value: value}, nil
}
validatorCalls := 0
validator := preparedValidator{typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
validatorCalls++
return contracts.ValidationResult{Approved: true}, nil
}}
producer.extractValidators.validators = []preparedValidator{validator}
producer.mergeValidators.validators = []preparedValidator{validator}
producer.normalizeValidators.validators = []preparedValidator{validator}
var received contracts.ReferenceSet
consumer.typed.extract = func(_ context.Context, _ any, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
received = CloneReferenceSet(request.References)
return erasedTypedResult{Value: codecScore{Value: 3}}, nil
}
loader := newAcceptedCheckpointLoader()
producerKey := CheckpointLaneKey(producer.resolved.StepID, producer.resolved.ID)
consumerKey := CheckpointLaneKey(consumer.resolved.StepID, consumer.resolved.ID)
stored.Diagnostics = []CheckpointDiagnostic{{Diagnostic: contracts.ProducerDiagnostic{Disposition: contracts.DiagnosticDispositionWarning, Category: contracts.DiagnosticCategoryFallback, ReasonCode: "stored-warning", OccurrenceCount: 1, Samples: []contracts.DiagnosticSample{{Scope: "normalize", Message: "stored normalize warning"}}}}}
loader.accepted[producerKey] = NormalizeCheckpoint{Output: stored}
loader.acceptedDecision[producerKey] = NewCheckpointDecision(CheckpointDecisionReused, CheckpointReasonAcceptedArtifactReused)
policy := CheckpointExecutionPolicy{
RequireReusableLanes: map[string]struct{}{producerKey: {}},
ForcedLanes: map[string]struct{}{consumerKey: {}},
}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoint: loader, CheckpointPolicy: policy})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if operationCalls != 0 || validatorCalls != 0 {
t.Fatalf("hydrated producer calls = operations %d validators %d, want zero", operationCalls, validatorCalls)
}
item := received.Slots["producer-output"].Items[0]
if string(item.Content) != string(stored.Artifact.Content) || item.Producer.StepID != producer.resolved.StepID || item.Producer.LaneID != producer.resolved.ID {
t.Fatalf("consumer generated reference = %#v, want exact hydrated producer bytes and identity", item)
}
if len(output.Diagnostics.Groups) != 1 || output.Diagnostics.Groups[0].ReasonCode != "stored-warning" {
t.Fatalf("hydrated diagnostics = %#v, want normalize checkpoint diagnostics only", output.Diagnostics)
}
assertAcceptedNormalizeEvent(t, output.CheckpointEvents, producer.resolved.StepID, producer.resolved.ID, CheckpointDecisionReused, CheckpointReasonAcceptedArtifactReused)
for _, event := range output.CheckpointEvents {
if event.StepID == producer.resolved.StepID && event.LaneID == producer.resolved.ID && event.Stage != string(StageNormalize) {
t.Fatalf("hydrated producer synthesized checkpoint event: %#v", event)
}
}
freshInput, _, _ := handoffFixture(t, value)
freshPrepared := freshInput.Prepared
freshPrepared.Steps[0].lanes[0].typed.extract = func(context.Context, any, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
return erasedTypedResult{Value: value}, nil
}
freshPrepared.Steps[0].lanes[0].typed.normalize = func(context.Context, any, contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) {
return erasedTypedResult{Value: value}, nil
}
freshPrepared.Steps[1].lanes[0].typed.extract = func(context.Context, any, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
return erasedTypedResult{Value: codecScore{Value: 3}}, nil
}
freshLoader := &handoffDependencyLoader{CheckpointLoader: NoopCheckpointLoader()}
freshOutput, err := New().Run(context.Background(), RunInput{Prepared: freshPrepared, RawInput: []byte("input"), Checkpoint: freshLoader})
if err != nil {
t.Fatalf("fresh Run() error = %v", err)
}
hydratedDependencies := loader.extractDeps[consumer.resolved.ID]
if generatedFingerprintCount(hydratedDependencies) != 1 {
t.Fatalf("hydrated consumer dependencies = %#v, want generated producer fingerprint", hydratedDependencies)
}
var matchedFreshDependencies bool
for _, dependencies := range freshLoader.extract {
if reflect.DeepEqual(dependencies, hydratedDependencies) {
matchedFreshDependencies = true
break
}
}
if !matchedFreshDependencies {
t.Fatalf("consumer dependencies differ: fresh %#v hydrated %#v", freshLoader.extract, hydratedDependencies)
}
if !reflect.DeepEqual(freshOutput.Manifest.References, output.Manifest.References) {
t.Fatalf("generated provenance differs: fresh %#v hydrated %#v", freshOutput.Manifest.References, output.Manifest.References)
}
}
func TestRunnerRejectsInvalidRequiredNormalizedArtifactBeforeConsumer(t *testing.T) {
const contentSentinel = "sensitive-campaign-payload-74291"
tests := []struct {
name string
decision CheckpointDecision
mutate func(*CheckpointArtifact)
wantCode CheckpointReasonCode
}{
{"missing", NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonMissing), nil, CheckpointReasonMissing},
{"rejected status", NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonStatusNotReusable), nil, CheckpointReasonStatusNotReusable},
{"corrupt payload", NewCheckpointDecision(CheckpointDecisionReused, CheckpointReasonAcceptedArtifactReused), func(v *CheckpointArtifact) { v.Artifact.Content = []byte(`{"items":[`) }, CheckpointReasonArtifactPayloadInvalid},
{"non canonical", NewCheckpointDecision(CheckpointDecisionReused, CheckpointReasonAcceptedArtifactReused), func(v *CheckpointArtifact) { v.Artifact.Content = []byte(`{"items": ["stored"]}`) }, CheckpointReasonArtifactNotCanonical},
{"wrong codec identity", NewCheckpointDecision(CheckpointDecisionReused, CheckpointReasonAcceptedArtifactReused), func(v *CheckpointArtifact) { v.Artifact.Kind = "test/score" }, CheckpointReasonArtifactCodecIncompatible},
{"wrong content digest", NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonArtifactDigestMismatch), nil, CheckpointReasonArtifactDigestMismatch},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
input, _, _ := handoffFixture(t, codecNotes{Items: []string{contentSentinel}})
prepared := input.Prepared
producer := &prepared.Steps[0].lanes[0]
consumer := &prepared.Steps[1].lanes[0]
doc := prepared.input.(*typedTestInput).doc
stored, err := checkpointArtifact(producer.typed.codec, producer.resolved.ID, producer.resolved.Normalize.Module, doc.ID, codecNotes{Items: []string{contentSentinel}})
if err != nil {
t.Fatal(err)
}
if test.mutate != nil {
test.mutate(&stored)
}
consumerCalls := 0
consumer.typed.extract = func(context.Context, any, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
consumerCalls++
return erasedTypedResult{Value: codecScore{Value: 1}}, nil
}
loader := newAcceptedCheckpointLoader()
producerKey := CheckpointLaneKey(producer.resolved.StepID, producer.resolved.ID)
loader.accepted[producerKey] = NormalizeCheckpoint{Output: stored}
loader.acceptedDecision[producerKey] = test.decision
policy := CheckpointExecutionPolicy{RequireReusableLanes: map[string]struct{}{producerKey: {}}}
output, runErr := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoint: loader, CheckpointPolicy: policy})
if runErr == nil || !strings.Contains(runErr.Error(), string(test.wantCode)) || consumerCalls != 0 {
t.Fatalf("Run() error = %v consumer calls = %d, want %q before consumer", runErr, consumerCalls, test.wantCode)
}
assertAcceptedNormalizeEvent(t, output.CheckpointEvents, producer.resolved.StepID, producer.resolved.ID, CheckpointDecisionExecuted, test.wantCode)
encoded, err := json.Marshal(struct {
Manifest any
Events any
Error string
}{output.Manifest, output.CheckpointEvents, runErr.Error()})
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(encoded), contentSentinel) || strings.Contains(string(encoded), string(stored.Artifact.Content)) {
t.Fatalf("failed hydration diagnostics leaked artifact content: %s", encoded)
}
})
}
}
func TestRunnerRetainsEarlierHydratedProducerWhenLaterRequiredProducerFails(t *testing.T) {
prepared := preparedPipelineWithSharedProducerStep(t)
first := &prepared.Steps[0].lanes[0]
second := &prepared.Steps[0].lanes[1]
consumer := &prepared.Steps[1].lanes[0]
doc := prepared.input.(*typedTestInput).doc
stored, err := checkpointArtifact(first.typed.codec, first.resolved.ID, first.resolved.Normalize.Module, doc.ID, codecNotes{Items: []string{"retained producer"}})
if err != nil {
t.Fatal(err)
}
consumerCalls := 0
consumer.typed.extract = func(context.Context, any, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
consumerCalls++
return erasedTypedResult{Value: codecScore{Value: 1}}, nil
}
loader := newAcceptedCheckpointLoader()
firstKey := CheckpointLaneKey(first.resolved.StepID, first.resolved.ID)
secondKey := CheckpointLaneKey(second.resolved.StepID, second.resolved.ID)
stored.Diagnostics = []CheckpointDiagnostic{{Diagnostic: contracts.ProducerDiagnostic{Disposition: contracts.DiagnosticDispositionWarning, Category: contracts.DiagnosticCategoryFallback, ReasonCode: "retained-warning", OccurrenceCount: 1, Samples: []contracts.DiagnosticSample{{Scope: "normalize", Message: "retained warning"}}}}}
loader.accepted[firstKey] = NormalizeCheckpoint{Output: stored}
loader.acceptedDecision[firstKey] = NewCheckpointDecision(CheckpointDecisionReused, CheckpointReasonAcceptedArtifactReused)
loader.acceptedDecision[secondKey] = NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonMissing)
policy := CheckpointExecutionPolicy{RequireReusableLanes: map[string]struct{}{firstKey: {}, secondKey: {}}}
output, runErr := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoint: loader, CheckpointPolicy: policy})
if runErr == nil || !strings.Contains(runErr.Error(), string(CheckpointReasonMissing)) {
t.Fatalf("Run() error = %v, want missing required producer", runErr)
}
if consumerCalls != 0 {
t.Fatalf("consumer calls = %d, want zero", consumerCalls)
}
if len(output.NormalizeOutputs) != 1 || output.NormalizeOutputs[0].LaneID != first.resolved.ID || string(output.NormalizeOutputs[0].Artifact.Content) != string(stored.Artifact.Content) {
t.Fatalf("retained normalize outputs = %#v, want first producer", output.NormalizeOutputs)
}
if len(output.Diagnostics.Groups) != 1 || output.Diagnostics.Groups[0].ReasonCode != "retained-warning" {
t.Fatalf("retained diagnostics = %#v", output.Diagnostics)
}
type decisionExpectation struct {
step, lane string
category CheckpointDecisionCategory
reason CheckpointReasonCode
}
want := []decisionExpectation{
{first.resolved.StepID, first.resolved.ID, CheckpointDecisionReused, CheckpointReasonAcceptedArtifactReused},
{second.resolved.StepID, second.resolved.ID, CheckpointDecisionExecuted, CheckpointReasonMissing},
}
var got []decisionExpectation
for _, event := range output.CheckpointEvents {
if event.Stage == string(StageNormalize) {
got = append(got, decisionExpectation{event.StepID, event.LaneID, event.Category, event.ReasonCode})
}
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("normalize decisions = %#v, want %#v", got, want)
}
var manifestGot []decisionExpectation
for _, decision := range output.Manifest.CheckpointDecisions {
if decision.Stage == string(StageNormalize) {
manifestGot = append(manifestGot, decisionExpectation{
decision.StepID, decision.LaneID,
CheckpointDecisionCategory(decision.Category), CheckpointReasonCode(decision.ReasonCode),
})
}
}
if !reflect.DeepEqual(manifestGot, want) {
t.Fatalf("manifest normalize decisions = %#v, want %#v", manifestGot, want)
}
}
func preparedPipelineWithSharedProducerStep(t *testing.T) *PreparedPipeline {
t.Helper()
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
base := typedResolutionProfile()
profile := base
profile.Artifacts = nil
profile.Steps = []PipelineStepProfile{
{ID: "producers", Artifacts: map[string]ArtifactLaneProfile{"first": base.Artifacts["notes"], "second": base.Artifacts["notes"]}},
{ID: "consumer", Artifacts: map[string]ArtifactLaneProfile{"score": base.Artifacts["score"]}},
}
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v", err)
}
prepared, err := Prepare(resolved, registriesFromModuleCatalog(catalog), ModuleDependencies{})
if err != nil {
t.Fatalf("Prepare() error = %v", err)
}
doc := typedTestDocumentWithUnits(1)
prepared.input.(*typedTestInput).doc = doc
prepared.chunker = &typedTestChunker{key: "typed/chunk", plan: typedTestPlan(doc)}
return prepared
}
func TestForcedRequiredLaneExecutesInsteadOfHydrating(t *testing.T) {
prepared := preparedOrderedPipeline(t, 1,
orderedLaneSpec{id: "unrelated", profile: "score"},
orderedLaneSpec{id: "producer", profile: "notes"},
orderedLaneSpec{id: "consumer", profile: "score"},
)
unrelated := &prepared.Steps[0].lanes[0]
producer := &prepared.Steps[1].lanes[0]
consumer := &prepared.Steps[2].lanes[0]
installGeneratedReferenceTarget(&consumer.resolved.ExtractReferences, StageExtract, consumer, "step-2", "producer")
doc := prepared.input.(*typedTestInput).doc
unrelatedArtifact, err := checkpointArtifact(unrelated.typed.codec, unrelated.resolved.ID, unrelated.resolved.Normalize.Module, doc.ID, codecScore{Value: 9})
if err != nil {
t.Fatal(err)
}
producerArtifact, err := checkpointArtifact(producer.typed.codec, producer.resolved.ID, producer.resolved.Normalize.Module, doc.ID, codecNotes{Items: []string{"stale"}})
if err != nil {
t.Fatal(err)
}
unrelatedCalls, producerCalls := 0, 0
unrelated.typed.extract = func(context.Context, any, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
unrelatedCalls++
return erasedTypedResult{Value: codecScore{Value: 9}}, nil
}
producer.typed.extract = func(context.Context, any, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
producerCalls++
return erasedTypedResult{Value: codecNotes{Items: []string{"fresh"}}}, nil
}
producer.typed.normalize = func(context.Context, any, contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) {
return erasedTypedResult{Value: codecNotes{Items: []string{"fresh"}}}, nil
}
var consumerReferences contracts.ReferenceSet
consumer.typed.extract = func(_ context.Context, _ any, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
consumerReferences = CloneReferenceSet(request.References)
return erasedTypedResult{Value: codecScore{Value: 1}}, nil
}
loader := newAcceptedCheckpointLoader()
unrelatedKey := CheckpointLaneKey(unrelated.resolved.StepID, unrelated.resolved.ID)
producerKey := CheckpointLaneKey(producer.resolved.StepID, producer.resolved.ID)
consumerKey := CheckpointLaneKey(consumer.resolved.StepID, consumer.resolved.ID)
loader.accepted[unrelatedKey] = NormalizeCheckpoint{Output: unrelatedArtifact}
loader.acceptedDecision[unrelatedKey] = NewCheckpointDecision(CheckpointDecisionReused, CheckpointReasonAcceptedArtifactReused)
loader.accepted[producerKey] = NormalizeCheckpoint{Output: producerArtifact}
loader.acceptedDecision[producerKey] = NewCheckpointDecision(CheckpointDecisionReused, CheckpointReasonAcceptedArtifactReused)
policy := CheckpointExecutionPolicy{
ForcedLanes: map[string]struct{}{producerKey: {}, consumerKey: {}},
RequireReusableLanes: map[string]struct{}{unrelatedKey: {}, producerKey: {}},
}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoint: loader, CheckpointPolicy: policy})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if producerCalls == 0 || unrelatedCalls != 0 {
t.Fatalf("calls producer=%d unrelated=%d, want forced producer execution and unrelated hydration", producerCalls, unrelatedCalls)
}
if got := string(consumerReferences.Slots["producer-output"].Items[0].Content); !strings.Contains(got, "fresh") || strings.Contains(got, "stale") {
t.Fatalf("forced producer reference = %q, want freshly executed output", got)
}
assertAcceptedNormalizeEvent(t, output.CheckpointEvents, unrelated.resolved.StepID, unrelated.resolved.ID, CheckpointDecisionReused, CheckpointReasonAcceptedArtifactReused)
assertAcceptedNormalizeEvent(t, output.CheckpointEvents, producer.resolved.StepID, producer.resolved.ID, CheckpointDecisionForcedRecompute, CheckpointReasonRecomputeStep)
}
func installGeneratedReferenceTarget(target *ResolvedReferenceTarget, stage ModuleStage, consumer *preparedLaneExecutor, producerStep, producerLane string) {
*target = ResolvedReferenceTarget{
Stage: stage,
StepID: consumer.resolved.StepID,
LaneID: consumer.resolved.ID,
Module: consumer.resolved.Extract.Module,
Bindings: []ReferenceBinding{{
Stage: stage,
LaneID: consumer.resolved.ID,
SlotName: "producer-output",
Artifact: &ArtifactReference{Step: producerStep, Lane: producerLane},
}},
ReferenceSet: contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
"producer-output": {Slot: contracts.ReferenceSlot{Name: "producer-output", AcceptedArtifactKinds: []contracts.ArtifactKind{"test/notes"}, AcceptedMediaTypes: []string{"application/json"}}},
}},
}
}
func assertAcceptedNormalizeEvent(t *testing.T, events []CheckpointEvent, stepID, laneID string, category CheckpointDecisionCategory, code CheckpointReasonCode) {
t.Helper()
for _, event := range events {
if event.Stage == string(StageNormalize) && event.StepID == stepID && event.LaneID == laneID {
if event.Category != category || event.ReasonCode != code {
t.Fatalf("accepted normalize event = %#v, want %q/%q", event, category, code)
}
return
}
}
t.Fatalf("accepted normalize event missing from %#v", events)
}

View File

@@ -36,29 +36,33 @@ func artifactProducerCache(input RunInput, typed *preparedTypedLane, class contr
}
return value, nil
},
Observe: func(ctx context.Context, observation contracts.CacheObservation) error {
if chunk != nil {
chunkIndex := chunk.Index
observation.ChunkID, observation.ChunkIndex = chunk.ID, &chunkIndex
}
if observation.Err != nil {
message := strings.ToValidUTF8(fmt.Sprintf("Artifact cache %s: %v", observation.Decision, observation.Err), "<22>")
if len(message) > contracts.MaxDiagnosticMessageBytes {
message = message[:contracts.MaxDiagnosticMessageBytes]
for !utf8.ValidString(message) {
message = message[:len(message)-1]
}
}
*warnings = append(*warnings, contracts.ProducerDiagnostic{
Disposition: contracts.DiagnosticDispositionWarning, Category: contracts.DiagnosticCategoryDegradation,
ReasonCode: "cache_" + observation.Decision, OccurrenceCount: 1,
Samples: []contracts.DiagnosticSample{{Scope: origin.ModuleKey, Message: message}},
})
}
if input.ObserveCache != nil {
return input.ObserveCache(ctx, origin, observation)
}
return nil
},
Observe: producerCacheObserver(input, origin, chunk, warnings),
}
}
func producerCacheObserver(input RunInput, origin contracts.DiagnosticOrigin, chunk *source.Chunk, warnings *[]contracts.ProducerDiagnostic) func(context.Context, contracts.CacheObservation) error {
return func(ctx context.Context, observation contracts.CacheObservation) error {
if chunk != nil {
chunkIndex := chunk.Index
observation.ChunkID, observation.ChunkIndex = chunk.ID, &chunkIndex
}
if observation.Err != nil {
message := strings.ToValidUTF8(fmt.Sprintf("Artifact cache %s: %v", observation.Decision, observation.Err), "<22>")
if len(message) > contracts.MaxDiagnosticMessageBytes {
message = message[:contracts.MaxDiagnosticMessageBytes]
for !utf8.ValidString(message) {
message = message[:len(message)-1]
}
}
*warnings = append(*warnings, contracts.ProducerDiagnostic{
Disposition: contracts.DiagnosticDispositionWarning, Category: contracts.DiagnosticCategoryDegradation,
ReasonCode: "cache_" + observation.Decision, OccurrenceCount: 1,
Samples: []contracts.DiagnosticSample{{Scope: origin.ModuleKey, Message: message}},
})
}
if input.ObserveCache != nil {
return input.ObserveCache(ctx, origin, observation)
}
return nil
}
}

View File

@@ -1,11 +1,12 @@
package pipeline
import (
"bytes"
"context"
"encoding/json"
"fmt"
"time"
"io"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
@@ -16,32 +17,18 @@ type chunkPlanExecution struct {
chunks []source.Chunk
plan *source.ChunkPlan
diagnostics []contracts.DiagnosticGroup
rejection *contracts.RejectedOutput
lookup ChunkPlanDecision
record *ChunkPlanRecord
action string
summary artifacts.ChunkPlanSummary
validation *artifacts.ValidationSummary
}
func effectiveChunkCacheMode(mode ChunkCacheMode) ChunkCacheMode {
if mode == "" {
return ChunkCacheBypass
}
parsed, _ := ParseChunkCacheMode(string(mode))
return parsed
}
// Chunk-plan lookup and publication happen serially before lane workers start.
// The runner therefore does not add synchronization around the store.
// Chunk planning remains serial before lane workers. Only a native chunk
// producer uses the accepted-artifact cache; deterministic plans run directly.
func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string) (chunkPlanExecution, error) {
mode := effectiveChunkCacheMode(input.ChunkCacheMode)
chunker := input.Prepared.chunker
result := chunkPlanExecution{lookup: ChunkPlanDecision{Reason: "chunk plan lookup skipped"}, summary: artifacts.ChunkPlanSummary{Mode: string(mode), SourceDigest: doc.Digest, RequestedModule: chunker.Key(), LookupStatus: "skipped", LookupReason: "chunk plan lookup skipped", ValidationStatus: "not_run", PublicationStatus: "not_requested"}}
result := chunkPlanExecution{}
origin := contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageChunk, ModuleKey: chunker.Key()}
target := typedValidationTarget{stage: StageChunk, moduleKey: chunker.Key(), source: doc, sourceID: doc.ID, sourceInput: sourceInput.Clone(), sessionID: sessionID, references: input.pipeline.ChunkReferences.ReferenceSet, metadata: input.Metadata, execution: input.Execution}
assessChunks := chunkCandidateAssessor(target, input.Prepared.chunkValidators)
var assessment contracts.CandidateAssessment
operation := contracts.ProducerOperation[source.ChunkPlan]{Controls: input.pipeline.ChunkControls, Execution: input.Execution, ReuseEligible: true,
operation := contracts.ProducerOperation[source.ChunkPlan]{Controls: input.pipeline.ChunkControls, Execution: input.Execution, ReuseEligible: !input.reuseIneligible,
Assess: func(ctx context.Context, plan source.ChunkPlan) (contracts.CandidateAssessment, error) {
_, chunks, err := validateAndMaterializeChunkPlan(doc, plan)
if err != nil {
@@ -55,35 +42,32 @@ func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.S
return assessment, err
},
}
if mode == ChunkCacheAuto {
record, decision, err := input.ChunkPlans.Load(doc.Digest)
if err != nil {
return result, fmt.Errorf("load chunk plan: %w", err)
}
result.lookup = decision
result.summary.LookupStatus, result.summary.LookupReason = chunkPlanLookupStatus(decision.Status), chunkPlanLookupReason(decision.Status)
if decision.Status == ChunkPlanHit {
plan, chunks, err := validateAndMaterializeChunkPlan(doc, record.Plan)
if err == nil {
cachedAssessment, err := assessChunks(ctx, chunks)
var cacheDiagnostics []contracts.ProducerDiagnostic
if input.Cache != nil && input.pipeline.ChunkExecutionClass == contracts.ExecutionClassLLMBacked {
operation.Cache = &contracts.ProducerCache[source.ChunkPlan]{
Store: input.Cache,
Encode: func(plan source.ChunkPlan) ([]byte, error) {
canonical, _, err := validateAndMaterializeChunkPlan(doc, plan)
if err != nil {
return result, err
return nil, err
}
if cachedAssessment.Accepted() {
if err := result.setCandidate(record, "reused"); err != nil {
return result, err
}
result.plan, result.chunks, result.accepted = &plan, chunks, true
result.outcome = contracts.ProducerOutcome{Disposition: contracts.ProducerAccepted, ReuseEligible: true}
result.diagnostics, err = operationDiagnosticGroups(record.Diagnostics, nil, cachedAssessment, result.outcome, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageChunk, ModuleKey: chunker.Key()}, nil)
result.setValidation(nil, err)
return result, err
return json.Marshal(canonical)
},
Decode: func(content []byte) (source.ChunkPlan, error) {
var plan source.ChunkPlan
decoder := json.NewDecoder(bytes.NewReader(content))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&plan); err != nil {
return source.ChunkPlan{}, err
}
}
result.lookup = ChunkPlanDecision{Status: ChunkPlanInvalid, Reason: chunkPlanLookupReason(ChunkPlanInvalid)}
result.summary.LookupStatus, result.summary.LookupReason = "invalid", chunkPlanLookupReason(ChunkPlanInvalid)
} else if decision.Status != ChunkPlanMissing && decision.Status != ChunkPlanInvalid {
return result, fmt.Errorf("load chunk plan returned unsupported decision status")
var trailing any
if err := decoder.Decode(&trailing); err != io.EOF {
return source.ChunkPlan{}, fmt.Errorf("cached chunk plan must contain one JSON value")
}
canonical, _, err := validateAndMaterializeChunkPlan(doc, plan)
return canonical, err
},
Observe: producerCacheObserver(input, origin, nil, &cacheDiagnostics),
}
}
metadata, err := cloneMetadata(input.Metadata)
@@ -105,142 +89,7 @@ func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.S
if err != nil {
return result, fmt.Errorf("materialize final chunk plan: %w", err)
}
digest, err := source.DigestChunkPlan(plan)
if err != nil {
return result, err
}
producerMetadata, _, err := moduleManifestMetadata(chunker)
if err != nil {
return result, err
}
profile := ""
if input.pipeline.ChunkExecutionClass == contracts.ExecutionClassLLMBacked {
profile = input.pipeline.Chunk.LLMProfile
}
record := ChunkPlanRecord{SchemaVersion: ChunkPlanSchemaVersion, SourceDigest: doc.Digest, PlanDigest: digest, Plan: source.CloneChunkPlan(plan), CreatedAt: time.Now().UTC(),
Producer: ChunkPlanProducer{InputModule: input.Prepared.input.Key(), ChunkModule: chunker.Key(), LLMProfile: profile, References: append([]artifacts.ReferenceProvenance(nil), referenceTargetProvenance(input.pipeline.ChunkReferences)...), Metadata: producerMetadata},
}
if err := result.setCandidate(record, "generated"); err != nil {
return result, err
}
if mode == ChunkCacheRefresh {
result.action, result.summary.Action = "refreshed", "refreshed"
}
if mode == ChunkCacheBypass {
result.action, result.summary.Action = "bypassed", "bypassed"
}
result.plan, result.chunks, result.accepted, result.outcome = &plan, chunks, true, outcome
result.setValidation(nil, nil)
if outcome.Disposition == contracts.ProducerFallback {
result.summary.ValidationStatus = "incomplete"
}
result.diagnostics, err = operationDiagnosticGroups(produced.Diagnostics, nil, assessment, outcome, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageChunk, ModuleKey: chunker.Key()}, nil)
if err != nil {
return result, err
}
if mode == ChunkCacheAuto || mode == ChunkCacheRefresh {
result.summary.PublicationStatus = "not_published"
if outcome.ReuseEligible {
record.Diagnostics = contracts.CloneProducerDiagnostics(produced.Diagnostics)
if err := input.ChunkPlans.Save(record); err != nil {
return result, fmt.Errorf("save chunk plan: %w", err)
}
result.summary.PublicationStatus = "published"
}
}
return result, nil
}
func chunkPlanLookupStatus(status ChunkPlanStatus) string {
switch status {
case ChunkPlanHit:
return "hit"
case ChunkPlanMissing:
return "missing"
case ChunkPlanInvalid:
return "invalid"
default:
return "skipped"
}
}
func chunkPlanLookupReason(status ChunkPlanStatus) string {
switch status {
case ChunkPlanHit:
return "stored chunk plan is valid"
case ChunkPlanMissing:
return "chunk plan not found"
case ChunkPlanInvalid:
return "stored chunk plan is invalid"
default:
return "chunk plan lookup skipped"
}
}
func (result *chunkPlanExecution) setCandidate(record ChunkPlanRecord, action string) error {
cloned, err := cloneChunkPlanRecord(record)
if err != nil {
return err
}
result.record = &cloned
result.action = action
result.summary.Action = action
result.summary.SourceDigest = record.SourceDigest
result.summary.CandidateDigest = record.PlanDigest
return nil
}
func (result *chunkPlanExecution) setValidation(rejection *contracts.RejectedOutput, err error) {
switch {
case err != nil:
result.summary.ValidationStatus = "error"
case rejection != nil:
result.summary.ValidationStatus = "rejected"
default:
result.summary.ValidationStatus = "approved"
}
}
func cloneChunkPlanRecord(record ChunkPlanRecord) (ChunkPlanRecord, error) {
record.Plan = source.CloneChunkPlan(record.Plan)
record.Producer.References = append([]artifacts.ReferenceProvenance(nil), record.Producer.References...)
metadata, err := cloneMetadata(record.Producer.Metadata)
if err != nil {
return ChunkPlanRecord{}, fmt.Errorf("clone chunk plan producer metadata: %w", err)
}
record.Producer.Metadata = metadata
record.Diagnostics = contracts.CloneProducerDiagnostics(record.Diagnostics)
return record, nil
}
func applyChunkPlanExecution(output *RunOutput, result chunkPlanExecution) error {
if output == nil {
return nil
}
summary := result.summary
output.ChunkPlan = &summary
if output.Manifest.ChunkPlan == nil {
return nil
}
manifest := output.Manifest.ChunkPlan
manifest.Action = result.action
if result.record == nil {
return nil
}
record := result.record
manifest.SourceDigest = record.SourceDigest
manifest.PlanDigest = record.PlanDigest
manifest.PlanSchemaVersion = record.SchemaVersion
manifest.ProducerInputModule = record.Producer.InputModule
manifest.ProducerModule = record.Producer.ChunkModule
manifest.ProducerLLMProfile = record.Producer.LLMProfile
manifest.ProducerReferences = append([]artifacts.ReferenceProvenance(nil), record.Producer.References...)
metadata, err := cloneMetadata(record.Producer.Metadata)
if err != nil {
return fmt.Errorf("clone chunk plan manifest producer metadata: %w", err)
}
manifest.ProducerMetadata = metadata
createdAt := record.CreatedAt
manifest.CreatedAt = &createdAt
return nil
result.diagnostics, err = operationDiagnosticGroups(produced.Diagnostics, cacheDiagnostics, assessment, outcome, origin, nil)
return result, err
}

View File

@@ -2,837 +2,91 @@ package pipeline
import (
"context"
"encoding/json"
"errors"
"reflect"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkmap"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"go.maximumdirect.net/promptkit"
)
type recordingChunkPlanStore struct {
record ChunkPlanRecord
decision ChunkPlanDecision
loadErr error
saveErr error
loads int
saves int
saved ChunkPlanRecord
}
func (s *recordingChunkPlanStore) Load(string) (ChunkPlanRecord, ChunkPlanDecision, error) {
s.loads++
return s.record, s.decision, s.loadErr
}
func (s *recordingChunkPlanStore) Save(record ChunkPlanRecord) error {
s.saves++
s.saved = record
return s.saveErr
}
type countingChunkValidator struct {
calls int
result contracts.ValidationResult
err error
}
type capturingChunkMapOutput struct {
requests []contracts.OutputRequest
}
type capturingChunkMapOutput struct{ requests []contracts.OutputRequest }
func (*capturingChunkMapOutput) Key() string { return "capture/chunk-map" }
func (output *capturingChunkMapOutput) Encode(_ context.Context, request contracts.OutputRequest) (contracts.OutputResult, error) {
output.requests = append(output.requests, request)
return contracts.OutputResult{}, nil
}
func (*countingChunkValidator) Name() string { return "test/counting-chunks" }
func (*countingChunkValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (v *countingChunkValidator) Validate(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
v.calls++
return v.result, v.err
}
type manifestChunker struct {
terminalChunker
metadata map[string]any
}
func (c manifestChunker) ManifestMetadata() map[string]any { return c.metadata }
type llmCountingChunker struct {
terminalChunker
llmCalls *int
}
func (c llmCountingChunker) Plan(ctx context.Context, request contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
(*c.llmCalls)++
return c.terminalChunker.Plan(ctx, request)
}
type retryingChunker struct {
key string
plan source.ChunkPlan
calls int
}
type correctionAwareChunker struct {
key string
plan source.ChunkPlan
calls int
correction *contracts.SemanticCorrection
}
func (c *correctionAwareChunker) Key() string { return c.key }
func (*correctionAwareChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (c *correctionAwareChunker) Plan(_ context.Context, request contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
c.calls++
if request.Correction != nil {
correction, err := contracts.CloneSemanticCorrection(request.Correction)
if err != nil {
return contracts.ChunkPlanResult{}, err
}
c.correction = correction
}
response := "initial chunk response"
if c.calls > 1 {
response = "corrected chunk response"
}
candidate, err := contracts.NewModelCandidate([]byte(response), contracts.CorrectionProtocolSingleResponseV1)
if err != nil {
return contracts.ChunkPlanResult{}, err
}
return contracts.ChunkPlanResult{Plan: source.CloneChunkPlan(c.plan), ModelCandidate: candidate}, nil
}
func (c *retryingChunker) Key() string { return c.key }
func (*retryingChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (c *retryingChunker) Plan(context.Context, contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
c.calls++
if c.calls == 1 {
return contracts.ChunkPlanResult{Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("discarded", "discarded diagnostic")}}, errors.New("retry generation")
}
return contracts.ChunkPlanResult{Plan: source.CloneChunkPlan(c.plan), Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("accepted", "accepted diagnostic")}}, nil
}
type dependencyLoader struct {
CheckpointLoader
extractDependencies []CheckpointFingerprint
}
func (l *dependencyLoader) Extract(_ string, _ string, dependencies []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
l.extractDependencies = append([]CheckpointFingerprint(nil), dependencies...)
return ExtractCheckpoint{}, CheckpointDecision{Reason: "not found"}
}
func TestRunnerChunkPlanModeMatrix(t *testing.T) {
tests := []struct {
name string
mode ChunkCacheMode
decision ChunkPlanDecision
wantLoads int
wantSaves int
wantCalls int
wantAction string
}{
{name: "auto hit", mode: ChunkCacheAuto, decision: ChunkPlanDecision{Status: ChunkPlanHit}, wantLoads: 1, wantAction: "reused"},
{name: "auto missing", mode: ChunkCacheAuto, decision: ChunkPlanDecision{Status: ChunkPlanMissing}, wantLoads: 1, wantSaves: 1, wantCalls: 1, wantAction: "generated"},
{name: "auto invalid", mode: ChunkCacheAuto, decision: ChunkPlanDecision{Status: ChunkPlanInvalid}, wantLoads: 1, wantSaves: 1, wantCalls: 1, wantAction: "generated"},
{name: "bypass", mode: ChunkCacheBypass, wantCalls: 1, wantAction: "bypassed"},
{name: "empty bypass", wantCalls: 1, wantAction: "bypassed"},
{name: "refresh", mode: ChunkCacheRefresh, wantSaves: 1, wantCalls: 1, wantAction: "refreshed"},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
calls := 0
llmCalls := 0
prepared.chunker = llmCountingChunker{terminalChunker: terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &calls}, llmCalls: &llmCalls}
store := &recordingChunkPlanStore{record: chunkPlanRecord(t, prepared, plan), decision: tc.decision}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: tc.mode, ChunkPlans: store})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if store.loads != tc.wantLoads || store.saves != tc.wantSaves || calls != tc.wantCalls {
t.Fatalf("calls = load %d save %d module %d, want %d %d %d", store.loads, store.saves, calls, tc.wantLoads, tc.wantSaves, tc.wantCalls)
}
if output.Manifest.ChunkPlan == nil || output.Manifest.ChunkPlan.Action != tc.wantAction || output.ChunkPlan == nil || output.ChunkPlan.Action != tc.wantAction {
t.Fatalf("chunk plan manifest = %#v summary = %#v, want action %q", output.Manifest.ChunkPlan, output.ChunkPlan, tc.wantAction)
}
})
}
}
func TestRunnerUsesStatusDerivedChunkPlanSummaryReasons(t *testing.T) {
for _, tc := range []struct {
name string
status ChunkPlanStatus
wantStatus string
wantReason string
}{
{name: "hit", status: ChunkPlanHit, wantStatus: "hit", wantReason: "stored chunk plan is valid"},
{name: "missing", status: ChunkPlanMissing, wantStatus: "missing", wantReason: "chunk plan not found"},
{name: "invalid", status: ChunkPlanInvalid, wantStatus: "invalid", wantReason: "stored chunk plan is invalid"},
} {
t.Run(tc.name, func(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
store := &recordingChunkPlanStore{
record: chunkPlanRecord(t, prepared, plan),
decision: ChunkPlanDecision{Status: tc.status, Reason: "SENTINEL_STORE_REASON"},
}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto, ChunkPlans: store})
if err != nil {
t.Fatal(err)
}
if output.ChunkPlan.LookupStatus != tc.wantStatus || output.ChunkPlan.LookupReason != tc.wantReason || strings.Contains(output.ChunkPlan.LookupReason, "SENTINEL_STORE_REASON") {
t.Fatalf("summary = %#v", output.ChunkPlan)
}
})
}
prepared, _ := preparedTerminalDebugPipeline(t)
output, err := New().Run(context.Background(), RunInput{
Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto,
ChunkPlans: &recordingChunkPlanStore{loadErr: errors.New("read failed")},
})
if err == nil || output.ChunkPlan == nil || output.ChunkPlan.LookupStatus != "skipped" || output.ChunkPlan.LookupReason != "chunk plan lookup skipped" {
t.Fatalf("operational lookup output=%#v error=%v", output.ChunkPlan, err)
}
}
func TestRunnerChunkPlanHitUsesStoredProducerProvenance(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
prepared.resolved.Chunk.Module = "chunk/requested"
prepared.resolved.Chunk.LLMProfile = "current-profile"
record := chunkPlanRecord(t, prepared, plan)
record.Producer.InputModule = "input/original"
record.Producer.ChunkModule = "chunk/original"
record.Producer.LLMProfile = "original-profile"
record.Producer.Metadata = map[string]any{"prompt": map[string]any{"id": "original"}}
store := &recordingChunkPlanStore{record: record, decision: ChunkPlanDecision{Status: ChunkPlanHit}}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto, ChunkPlans: store})
if err != nil {
t.Fatal(err)
}
manifest := output.Manifest.ChunkPlan
if manifest.RequestedModule != "chunk/requested" || manifest.ProducerModule != "chunk/original" || manifest.ProducerLLMProfile != "original-profile" {
t.Fatalf("chunk plan manifest = %#v", manifest)
}
storedNested := record.Producer.Metadata["prompt"].(map[string]any)
storedNested["id"] = "mutated"
if got := manifest.ProducerMetadata["prompt"].(map[string]any)["id"]; got != "original" {
t.Fatalf("producer metadata changed through stored alias: %v", got)
}
}
func TestRunnerReusesChunkPlanDiagnosticsWithCurrentValidatorDiagnostics(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
record := chunkPlanRecord(t, prepared, plan)
record.Diagnostics = []contracts.ProducerDiagnostic{{
Disposition: contracts.DiagnosticDispositionWarning,
Category: contracts.DiagnosticCategoryConfiguration,
ReasonCode: "stored_chunk_diagnostic",
OccurrenceCount: 1,
Samples: []contracts.DiagnosticSample{{Scope: "reference", Message: "Stored configuration signal."}},
}}
validator := &countingChunkValidator{result: contracts.ValidationResult{Approved: true, Diagnostics: []contracts.ProducerDiagnostic{{
Disposition: contracts.DiagnosticDispositionAdvisory,
Category: contracts.DiagnosticCategoryDataQuality,
ReasonCode: "current_validator_diagnostic",
OccurrenceCount: 1,
Samples: []contracts.DiagnosticSample{{Scope: "chunk", Message: "Current validator finding."}},
}}}}
prepared.chunkValidators.validators = []preparedValidator{{resolved: ResolvedValidator{Binding: Binding(validator.Name()), Target: ValidatorTargetChunk}, chunk: validator}}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto, ChunkPlans: &recordingChunkPlanStore{record: record, decision: ChunkPlanDecision{Status: ChunkPlanHit}}})
if err != nil {
t.Fatal(err)
}
if validator.calls != 1 || len(output.Diagnostics.Groups) != 2 {
t.Fatalf("validator calls = %d diagnostics = %#v", validator.calls, output.Diagnostics)
}
if output.Diagnostics.Groups[0].ReasonCode != "stored_chunk_diagnostic" || output.Diagnostics.Groups[1].ReasonCode != "current_validator_diagnostic" || output.Diagnostics.Groups[1].Origin.ValidatorKey != validator.Name() {
t.Fatalf("diagnostic groups = %#v", output.Diagnostics.Groups)
}
}
func TestRunnerProvidesAcceptedChunkMapToOutput(t *testing.T) {
for _, test := range []struct {
name string
configure func(*PreparedPipeline, source.ChunkPlan) (RunInput, chunkmap.Producer)
}{
{
name: "bypassed plan",
configure: func(prepared *PreparedPipeline, _ source.ChunkPlan) (RunInput, chunkmap.Producer) {
return RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheBypass}, chunkmap.Producer{
InputModule: prepared.input.Key(), ChunkModule: prepared.chunker.Key(),
}
},
},
{
name: "cached plan",
configure: func(prepared *PreparedPipeline, plan source.ChunkPlan) (RunInput, chunkmap.Producer) {
prepared.resolved.Chunk.Module = "chunk/requested"
record := chunkPlanRecord(t, prepared, plan)
record.Producer.InputModule = "input/original"
record.Producer.ChunkModule = "chunk/original"
record.Producer.LLMProfile = "original-profile"
return RunInput{
Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto,
ChunkPlans: &recordingChunkPlanStore{record: record, decision: ChunkPlanDecision{Status: ChunkPlanHit}},
}, chunkmap.Producer{InputModule: "input/original", ChunkModule: "chunk/original", LLMProfile: "original-profile"}
},
},
} {
t.Run(test.name, func(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
encoder := &capturingChunkMapOutput{}
prepared.output = encoder
input, wantProducer := test.configure(prepared, plan)
if _, err := New().Run(context.Background(), input); err != nil {
t.Fatalf("Run() error = %v", err)
}
if len(encoder.requests) != 1 || encoder.requests[0].ChunkMap == nil {
t.Fatalf("output requests = %#v, want one accepted chunk map", encoder.requests)
}
artifact := encoder.requests[0].ChunkMap
if artifact.Kind != chunkmap.ArtifactKind || artifact.Schema.ID != chunkmap.SchemaID || artifact.MediaType != chunkmap.MediaType {
t.Fatalf("chunk map artifact = %#v, want fixed serialized identity", artifact)
}
value, err := chunkmap.New().Decode(artifact.Content)
if err != nil {
t.Fatalf("decode output chunk map: %v", err)
}
if value.RequestedChunker != prepared.resolved.Chunk.Module || value.Producer != wantProducer {
t.Fatalf("chunk map = %#v, want current request and producer %#v", value, wantProducer)
}
})
}
}
func TestRunnerOmitsChunkMapForRejectedPlan(t *testing.T) {
prepared, _ := preparedTerminalDebugPipeline(t)
calls, assessments := 0, 0
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
return planResponse(t, typedTestPlan(typedTestDocument())), nil
}))
input := planCacheRunInput(t, engine, func(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
assessments++
return contracts.ValidationResult{Approved: true}, nil
})
_, store := filesystemProducerCache(t)
encoder := &capturingChunkMapOutput{}
prepared.output = encoder
prepared.chunkValidators.validators = []preparedValidator{{
resolved: ResolvedValidator{Binding: Binding("reject/chunk"), Target: ValidatorTargetChunk},
chunk: &countingChunkValidator{result: contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "not accepted", CorrectionGuidance: "return an acceptable chunk plan"}},
}}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if len(output.Rejected) != 1 || len(encoder.requests) != 1 || encoder.requests[0].ChunkMap != nil {
t.Fatalf("output = %#v requests = %#v, want rejected plan without chunk map", output.Rejected, encoder.requests)
}
}
func TestRunnerCorrectsRejectedGeneratedChunkPlan(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
prepared.resolved.ChunkExecutionClass = contracts.ExecutionClassLLMBacked
prepared.resolved.Chunk.Retries = 1
chunker := &correctionAwareChunker{key: prepared.resolved.Chunk.Module, plan: plan}
prepared.chunker = chunker
validatorCalls := 0
prepared.chunkValidators.validators = []preparedValidator{{
resolved: ResolvedValidator{Binding: Binding("reject-then-approve"), Target: ValidatorTargetChunk},
chunk: chunkValidationFunc{name: "reject-then-approve", validate: func(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
validatorCalls++
if validatorCalls == 1 {
return contracts.ValidationResult{ReasonCode: "chunk_scope", Message: "scope needs correction", CorrectionGuidance: "correct the chunk scope"}, nil
}
return contracts.ValidationResult{Approved: true}, nil
}},
}}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if len(output.Rejected) != 0 || chunker.calls != 2 || validatorCalls != 2 {
t.Fatalf("output = %#v chunker calls = %d validator calls = %d", output.Rejected, chunker.calls, validatorCalls)
}
if got := string(chunker.correction.AssistantResponse); got != "initial chunk response" {
t.Fatalf("correction response = %q", got)
}
}
func TestRunnerDoesNotPublishValidationIncompleteChunkPlan(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
prepared.resolved.ChunkValidationPolicy.ValidatorFailure = ValidatorFailureWarnContinue
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan}
prepared.chunkValidators.validators = []preparedValidator{{
resolved: ResolvedValidator{Binding: Binding("unavailable"), Target: ValidatorTargetChunk},
chunk: &countingChunkValidator{err: errors.New("validator unavailable")},
}}
store := &recordingChunkPlanStore{decision: ChunkPlanDecision{Status: ChunkPlanMissing}}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto, ChunkPlans: store})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if output.ChunkPlan == nil || output.ChunkPlan.ValidationStatus != "incomplete" || output.ChunkPlan.PublicationStatus != "not_published" || store.saves != 0 {
t.Fatalf("chunk summary = %#v saves = %d", output.ChunkPlan, store.saves)
}
}
func TestRunnerRegeneratesValidationIncompleteChunkPlanHit(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
prepared.resolved.ChunkValidationPolicy.ValidatorFailure = ValidatorFailureWarnContinue
calls := 0
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &calls}
validatorCalls := 0
prepared.chunkValidators.validators = []preparedValidator{{
resolved: ResolvedValidator{Binding: Binding("cache-then-generated"), Target: ValidatorTargetChunk},
chunk: chunkValidationFunc{name: "cache-then-generated", validate: func(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
validatorCalls++
if validatorCalls == 1 {
return contracts.ValidationResult{}, errors.New("cached candidate could not be validated")
}
return contracts.ValidationResult{Approved: true}, nil
}},
}}
record := chunkPlanRecord(t, prepared, plan)
record.Diagnostics = []contracts.ProducerDiagnostic{producerDiagnostic("stored", "discarded stored diagnostic")}
store := &recordingChunkPlanStore{record: record, decision: ChunkPlanDecision{Status: ChunkPlanHit}}
debug := newCapturedDebugRecorder()
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto, ChunkPlans: store, Debug: debug})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if calls != 1 || validatorCalls != 2 || store.saves != 1 {
t.Fatalf("calls = producer %d validator %d saves %d, want 1 2 1", calls, validatorCalls, store.saves)
}
if output.ChunkPlan == nil || output.ChunkPlan.Action != "generated" || output.ChunkPlan.LookupStatus != "invalid" || output.ChunkPlan.ValidationStatus != "approved" {
t.Fatalf("chunk plan summary = %#v", output.ChunkPlan)
}
if len(output.Diagnostics.Groups) != 0 {
t.Fatalf("diagnostics = %#v, want discarded cache diagnostics omitted", output.Diagnostics)
}
assertAttemptEnvelopeSequence(t, debug, "chunk", 1)
}
func TestRunnerKeepsStoredPlanWhenCacheAndGeneratedValidationAreIncomplete(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
prepared.resolved.ChunkValidationPolicy.ValidatorFailure = ValidatorFailureWarnContinue
calls := 0
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &calls}
validator := &countingChunkValidator{err: errors.New("validator unavailable")}
prepared.chunkValidators.validators = []preparedValidator{{resolved: ResolvedValidator{Binding: Binding(validator.Name()), Target: ValidatorTargetChunk}, chunk: validator}}
record := chunkPlanRecord(t, prepared, plan)
record.Diagnostics = []contracts.ProducerDiagnostic{producerDiagnostic("stored", "discarded stored diagnostic")}
store := &recordingChunkPlanStore{record: record, decision: ChunkPlanDecision{Status: ChunkPlanHit}}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto, ChunkPlans: store})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if calls != 1 || validator.calls != 2 || store.saves != 0 {
t.Fatalf("calls = producer %d validator %d saves %d, want 1 2 0", calls, validator.calls, store.saves)
}
if output.ChunkPlan == nil || output.ChunkPlan.Action != "generated" || output.ChunkPlan.LookupStatus != "invalid" || output.ChunkPlan.ValidationStatus != "incomplete" {
t.Fatalf("chunk plan summary = %#v", output.ChunkPlan)
}
if len(output.Diagnostics.Groups) != 1 || output.Diagnostics.Groups[0].ReasonCode != "validator_execution_incomplete" {
t.Fatalf("diagnostics = %#v, want only generated incomplete diagnostic", output.Diagnostics)
}
if !reflect.DeepEqual(store.record, record) {
t.Fatal("discarded incomplete candidates mutated the stored record")
}
}
func TestRunnerFailsOnValidationIncompleteChunkPlanHitUnderFailRun(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
calls := 0
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &calls}
validator := &countingChunkValidator{err: errors.New("validator unavailable")}
prepared.chunkValidators.validators = []preparedValidator{{resolved: ResolvedValidator{Binding: Binding(validator.Name()), Target: ValidatorTargetChunk}, chunk: validator}}
store := &recordingChunkPlanStore{record: chunkPlanRecord(t, prepared, plan), decision: ChunkPlanDecision{Status: ChunkPlanHit}}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto, ChunkPlans: store})
if err == nil || !strings.Contains(err.Error(), "validator unavailable") {
t.Fatalf("Run() error = %v, want cached validation failure", err)
}
if calls != 0 || validator.calls != 1 || store.saves != 0 {
t.Fatalf("calls = producer %d validator %d saves %d, want 0 1 0", calls, validator.calls, store.saves)
}
if output.ChunkPlan == nil || output.ChunkPlan.ValidationStatus != "error" {
t.Fatalf("chunk plan summary = %#v", output.ChunkPlan)
}
}
func TestRunnerRetainsChunkMapAfterLaneRejection(t *testing.T) {
prepared, _ := preparedTerminalDebugPipeline(t)
encoder := &capturingChunkMapOutput{}
prepared.output = encoder
prepared.Steps[0].lanes[0].mergeValidators.validators = []preparedValidator{{
resolved: ResolvedValidator{Binding: Binding("reject/lane"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
return contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "not accepted", CorrectionGuidance: "return an acceptable candidate"}, nil
},
}}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if len(output.Rejected) != 1 || len(encoder.requests) != 1 || encoder.requests[0].ChunkMap == nil {
t.Fatalf("output = %#v requests = %#v, want lane rejection with accepted chunk map", output.Rejected, encoder.requests)
}
}
func TestRunnerChunkMapRequestDoesNotAliasStoredPlan(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
encoder := &capturingChunkMapOutput{}
prepared.output = encoder
record := chunkPlanRecord(t, prepared, plan)
record.Plan.Annotations = source.ChunkAnnotations{"dnd/scenes": json.RawMessage(`{"title":"Opening"}`)}
store := &recordingChunkPlanStore{record: record, decision: ChunkPlanDecision{Status: ChunkPlanHit}}
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto, ChunkPlans: store}); err != nil {
t.Fatalf("Run() error = %v", err)
}
store.record.Plan.Annotations["dnd/scenes"][0] = '['
value, err := chunkmap.New().Decode(encoder.requests[0].ChunkMap.Content)
if err != nil {
t.Fatalf("decode captured chunk map after store mutation: %v", err)
}
if string(value.PlanAnnotations["dnd/scenes"]) != `{"title":"Opening"}` {
t.Fatalf("captured chunk map aliases stored plan: %s", value.PlanAnnotations["dnd/scenes"])
}
}
func TestRunnerChunkPlanManifestRetainsCandidateOnRejection(t *testing.T) {
prepared, _ := preparedTerminalDebugPipeline(t)
validator := &countingChunkValidator{result: contracts.ValidationResult{Approved: false, ReasonCode: "policy", Message: "no", CorrectionGuidance: "return a policy-compliant chunk plan"}}
prepared.chunkValidators.validators = []preparedValidator{{resolved: ResolvedValidator{Binding: Binding(validator.Name()), Target: ValidatorTargetChunk}, chunk: validator}}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheRefresh, ChunkPlans: &recordingChunkPlanStore{}})
if err != nil {
t.Fatal(err)
}
if output.Manifest.ChunkPlan.Action != "" || output.Manifest.ChunkPlan.PlanDigest != "" || output.ChunkPlan.ValidationStatus != "rejected" || output.ChunkPlan.PublicationStatus != "not_published" {
t.Fatalf("manifest = %#v summary = %#v; rejected output must not retain a usable chunk candidate", output.Manifest.ChunkPlan, output.ChunkPlan)
}
}
func TestRunnerValidatesChunkPlanPolicyInputs(t *testing.T) {
for _, tc := range []struct {
name string
mode ChunkCacheMode
want string
}{
{name: "auto requires store", mode: ChunkCacheAuto, want: "store is required"},
{name: "refresh requires store", mode: ChunkCacheRefresh, want: "store is required"},
{name: "invalid mode", mode: "sometimes", want: "not supported"},
} {
t.Run(tc.name, func(t *testing.T) {
prepared, _ := preparedTerminalDebugPipeline(t)
_, err := New().Run(context.Background(), RunInput{Prepared: prepared, ChunkCacheMode: tc.mode})
if err == nil || !strings.Contains(err.Error(), tc.want) {
t.Fatalf("Run() error = %v, want %q", err, tc.want)
}
})
}
}
func TestRunnerChunkPlanStoreErrorsAreFrameworkErrors(t *testing.T) {
for _, tc := range []struct {
name string
mode ChunkCacheMode
decision ChunkPlanDecision
loadErr error
saveErr error
wantError string
wantCalls int
}{
{name: "load", mode: ChunkCacheAuto, loadErr: errors.New("read failed"), wantError: "load chunk plan", wantCalls: 0},
{name: "save", mode: ChunkCacheRefresh, saveErr: errors.New("write failed"), wantError: "save chunk plan", wantCalls: 1},
} {
t.Run(tc.name, func(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
calls := 0
llmCalls := 0
prepared.chunker = llmCountingChunker{terminalChunker: terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &calls}, llmCalls: &llmCalls}
store := &recordingChunkPlanStore{decision: tc.decision, loadErr: tc.loadErr, saveErr: tc.saveErr}
_, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: tc.mode, ChunkPlans: store})
if err == nil || !strings.Contains(err.Error(), tc.wantError) || calls != tc.wantCalls {
t.Fatalf("Run() error = %v, module calls = %d, want %q and %d", err, calls, tc.wantError, tc.wantCalls)
}
})
}
}
func TestRunnerRegeneratesStructurallyInvalidHit(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
calls := 0
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &calls}
invalid := chunkPlanRecord(t, prepared, plan)
invalid.Plan.Ranges[0].StartUnitID = 999
store := &recordingChunkPlanStore{record: invalid, decision: ChunkPlanDecision{Status: ChunkPlanHit}}
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto, ChunkPlans: store}); err != nil {
t.Fatal(err)
}
if calls != 1 || store.loads != 1 || store.saves != 1 {
t.Fatalf("calls = module %d load %d save %d, want 1 1 1", calls, store.loads, store.saves)
}
}
func TestRunnerReusesAnnotationsFromDifferentChunkerAsOptionalData(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
plan.Annotations = source.ChunkAnnotations{"test/chunker": json.RawMessage(`{"label":"fixture"}`)}
plan.Ranges[0].Annotations = source.ChunkAnnotations{"test/chunker": json.RawMessage(`{"category":"sample"}`)}
record := chunkPlanRecord(t, prepared, plan)
record.Producer.ChunkModule = "chunk/producer"
store := &recordingChunkPlanStore{record: record, decision: ChunkPlanDecision{Status: ChunkPlanHit}}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto, ChunkPlans: store})
if err != nil {
t.Fatal(err)
}
if output.Manifest.ChunkPlan.ProducerModule != "chunk/producer" || output.Manifest.ChunkPlan.Action != "reused" {
t.Fatalf("different-chunker annotation plan was not reused: %#v", output.Manifest.ChunkPlan)
}
}
func TestRunnerRetriesBeforePublishingAcceptedPlan(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
prepared.resolved.Chunk.Retries = 1
chunker := &retryingChunker{key: prepared.resolved.Chunk.Module, plan: plan}
prepared.chunker = chunker
store := &recordingChunkPlanStore{decision: ChunkPlanDecision{Status: ChunkPlanMissing}}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto, ChunkPlans: store})
if err != nil {
t.Fatal(err)
}
if chunker.calls != 2 || store.saves != 1 {
t.Fatalf("module calls = %d saves = %d, want 2 and 1", chunker.calls, store.saves)
}
if len(output.Diagnostics.Groups) != 1 || output.Diagnostics.Groups[0].Samples[0].Scope != "accepted" || len(store.saved.Diagnostics) != 1 || store.saved.Diagnostics[0].Samples[0].Scope != "accepted" {
t.Fatalf("output diagnostics = %#v stored diagnostics = %#v", output.Diagnostics, store.saved.Diagnostics)
}
}
func TestRunnerAutoHitValidatesOnceWithoutRegenerationOrMutation(t *testing.T) {
tests := []struct {
name string
result contracts.ValidationResult
validatorErr error
wantError string
wantReject bool
}{
{name: "diagnostic", result: contracts.ValidationResult{Approved: true, Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("current", "current diagnostic")}}},
{name: "rejection", result: contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "rejected hit", CorrectionGuidance: "return an acceptable chunk plan"}, wantReject: true},
{name: "error", validatorErr: errors.New("validator failed"), wantError: "validator failed"},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
calls := 0
llmCalls := 0
prepared.chunker = llmCountingChunker{terminalChunker: terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &calls}, llmCalls: &llmCalls}
validator := &countingChunkValidator{result: tc.result, err: tc.validatorErr}
prepared.chunkValidators.validators = []preparedValidator{{resolved: ResolvedValidator{Binding: Binding(validator.Name()), Target: ValidatorTargetChunk}, chunk: validator}}
record := chunkPlanRecord(t, prepared, plan)
record.Diagnostics = []contracts.ProducerDiagnostic{producerDiagnostic("stored", "stored diagnostic")}
store := &recordingChunkPlanStore{record: record, decision: ChunkPlanDecision{Status: ChunkPlanHit}}
debug := newCapturedDebugRecorder()
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto, ChunkPlans: store, Debug: debug})
if tc.wantError != "" {
if err == nil || !strings.Contains(err.Error(), tc.wantError) {
t.Fatalf("Run() error = %v, want %q", err, tc.wantError)
}
} else if err != nil {
t.Fatalf("Run() error = %v", err)
}
wantProducerCalls := 0
if tc.wantReject {
wantProducerCalls = 1
}
if validator.calls != 1+wantProducerCalls || calls != wantProducerCalls || llmCalls != wantProducerCalls || store.loads != 1 || store.saves != 0 {
t.Fatalf("calls = validator %d module %d llm %d load %d save %d", validator.calls, calls, llmCalls, store.loads, store.saves)
}
if tc.wantReject {
assertAttemptEnvelopeSequence(t, debug, "chunk", 1)
} else {
assertAttemptEnvelopeSequence(t, debug, "chunk")
}
if tc.wantError == "" {
encoded := string(debug.json["chunk/output.json"])
if tc.wantReject {
if !strings.Contains(encoded, `"status":"invalid"`) || strings.Contains(encoded, `"plan":`) {
t.Fatalf("rejected cache debug = %s", encoded)
}
} else if !strings.Contains(encoded, `"status":"hit"`) || !strings.Contains(encoded, `"plan":`) || !strings.Contains(encoded, `"materialized_chunks":`) {
t.Fatalf("chunk hit debug = %s", encoded)
}
}
if tc.wantReject && (len(output.Rejected) != 1 || output.Rejected[0].ReasonCode != "rejected") {
t.Fatalf("rejected = %#v", output.Rejected)
}
if tc.wantError == "" && !tc.wantReject && len(output.Diagnostics.Groups) != 1+len(tc.result.Diagnostics) {
t.Fatalf("diagnostics = %#v, want stored diagnostic once plus current diagnostics", output.Diagnostics)
}
})
}
}
func TestRunnerPublishesOnlyAcceptedGeneratedPlans(t *testing.T) {
for _, tc := range []struct {
name string
moduleErr error
validator contracts.ValidationResult
cancel bool
wantCalls int
wantReject bool
}{
{name: "module error", moduleErr: errors.New("generation failed"), wantCalls: 2},
{name: "validator rejection", validator: contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "rejected generated plan", CorrectionGuidance: "return an acceptable chunk plan"}, wantCalls: 1, wantReject: true},
{name: "cancellation", cancel: true},
} {
t.Run(tc.name, func(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
prepared.resolved.Chunk.Retries = 1
calls := 0
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &calls, err: tc.moduleErr}
if tc.wantReject {
validator := &countingChunkValidator{result: tc.validator}
prepared.chunkValidators.validators = []preparedValidator{{resolved: ResolvedValidator{Binding: Binding(validator.Name()), Target: ValidatorTargetChunk}, chunk: validator}}
}
ctx := context.Background()
if tc.cancel {
cancelled, cancel := context.WithCancel(ctx)
cancel()
ctx = cancelled
}
store := &recordingChunkPlanStore{decision: ChunkPlanDecision{Status: ChunkPlanInvalid}}
output, err := New().Run(ctx, RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto, ChunkPlans: store})
if tc.wantReject {
if err != nil || len(output.Rejected) != 1 {
t.Fatalf("Run() error = %v rejected = %#v", err, output.Rejected)
}
} else if err == nil {
t.Fatal("Run() error = nil")
}
if calls != tc.wantCalls || store.saves != 0 {
t.Fatalf("module calls = %d, saves = %d; want %d, 0", calls, store.saves, tc.wantCalls)
}
})
}
}
func TestRunnerStoresProducerProvenanceAndDiagnostics(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
prepared.resolved.Chunk.LLMProfile = "chunk-profile"
prepared.resolved.ChunkExecutionClass = contracts.ExecutionClassLLMBacked
prepared.resolved.ChunkReferences = ResolvedReferenceTarget{
Stage: StageChunk,
ReferenceSet: contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
"guide": {Items: []contracts.ReferenceItem{{SlotName: "guide", Digest: "sha256:guide", Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///guide.txt"}, Content: []byte("sensitive")}}},
}},
}
prepared.chunker = manifestChunker{terminalChunker: terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("producer", "producer diagnostic")}}, metadata: map[string]any{"prompt_id": "chunk/prompt"}}
validator := &countingChunkValidator{result: contracts.ValidationResult{Approved: true, Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("validator", "validator diagnostic")}}}
prepared.chunkValidators.validators = []preparedValidator{{resolved: ResolvedValidator{Binding: Binding(validator.Name()), Target: ValidatorTargetChunk}, chunk: validator}}
store := &recordingChunkPlanStore{}
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheRefresh, ChunkPlans: store}); err != nil {
t.Fatal(err)
}
producer := store.saved.Producer
if producer.InputModule != prepared.input.Key() || producer.ChunkModule != prepared.chunker.Key() || producer.LLMProfile != "chunk-profile" || producer.Metadata["prompt_id"] != "chunk/prompt" {
t.Fatalf("producer = %#v", producer)
}
if len(producer.References) != 1 || producer.References[0].Digest != "sha256:guide" {
t.Fatalf("producer references = %#v", producer.References)
}
if len(store.saved.Diagnostics) != 1 || store.saved.Diagnostics[0].Samples[0].Scope != "producer" {
t.Fatalf("stored diagnostics = %#v, want only producer diagnostic", store.saved.Diagnostics)
}
}
func TestRunnerRejectsUncloneableModuleManifestMetadata(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
cyclic := make(map[string]any)
cyclic["self"] = cyclic
prepared.chunker = manifestChunker{
terminalChunker: terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan},
metadata: map[string]any{"cyclic": cyclic},
}
_, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")})
if err == nil || !strings.Contains(err.Error(), `clone manifest metadata for module "chunker": metadata.cyclic.self contains a cycle`) {
t.Fatalf("Run() error = %v, want contextual module metadata clone failure", err)
}
}
func TestRunnerOmitsProducerProfileForDeterministicChunker(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
prepared.resolved.Chunk.LLMProfile = "configured-but-unused"
prepared.resolved.ChunkExecutionClass = contracts.ExecutionClassDeterministic
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan}
store := &recordingChunkPlanStore{}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheRefresh, ChunkPlans: store})
if err != nil {
t.Fatal(err)
}
if store.saved.Producer.LLMProfile != "" || output.Manifest.ChunkPlan.ProducerLLMProfile != "" {
t.Fatalf("deterministic producer profile = stored %q manifest %q", store.saved.Producer.LLMProfile, output.Manifest.ChunkPlan.ProducerLLMProfile)
}
}
func TestRunnerDoesNotInventProducerProfileForLLMChunker(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
prepared.resolved.ChunkExecutionClass = contracts.ExecutionClassLLMBacked
prepared.resolved.Chunk.LLMProfile = ""
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan}
store := &recordingChunkPlanStore{}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheRefresh, ChunkPlans: store})
if err != nil {
t.Fatal(err)
}
if store.saved.Producer.LLMProfile != "" || output.Manifest.ChunkPlan.ProducerLLMProfile != "" {
t.Fatalf("LLM producer profile = stored %q manifest %q, want empty", store.saved.Producer.LLMProfile, output.Manifest.ChunkPlan.ProducerLLMProfile)
}
}
func TestRunnerRefreshChangesDownstreamChunkFingerprint(t *testing.T) {
doc := typedTestDocumentWithUnits(2)
prepared := preparedConcurrentPipeline(t, 1)
prepared.Steps[0].lanes = prepared.Steps[0].lanes[:1]
prepared.resolved.Steps[0].ArtifactLanes = prepared.resolved.Steps[0].ArtifactLanes[:1]
prepared.Steps[0].ArtifactLanes = prepared.Steps[0].ArtifactLanes[:1]
prepared.input = &typedTestInput{key: prepared.resolved.Input.Module, doc: doc}
run := func(plan source.ChunkPlan) []CheckpointFingerprint {
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan}
loader := &dependencyLoader{CheckpointLoader: NoopCheckpointLoader()}
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, ChunkCacheMode: ChunkCacheRefresh, ChunkPlans: &recordingChunkPlanStore{}, Checkpoint: loader}); err != nil {
input.Prepared.output = encoder
input.RawInput = []byte("input")
input.Cache = store
for range 2 {
if _, err := New().Run(context.Background(), input); err != nil {
t.Fatal(err)
}
return loader.extractDependencies
}
separate := typedTestPlan(doc)
combined := source.ChunkPlan{SourceDigest: doc.Digest, Ranges: []source.ChunkRange{{StartUnitID: 1, EndUnitID: 2}}}
if first, second := run(separate), run(combined); reflect.DeepEqual(first, second) {
t.Fatalf("downstream fingerprints did not change: %#v", first)
if calls != 1 || assessments != 1 || len(encoder.requests) != 2 {
t.Fatalf("calls=%d assessments=%d output requests=%d", calls, assessments, len(encoder.requests))
}
for _, request := range encoder.requests {
artifact := request.ChunkMap
if artifact == nil || artifact.Kind != chunkmap.ArtifactKind || artifact.Schema.ID != chunkmap.SchemaID || artifact.MediaType != chunkmap.MediaType {
t.Fatalf("chunk map artifact=%#v", artifact)
}
value, err := chunkmap.New().Decode(artifact.Content)
if err != nil {
t.Fatal(err)
}
wantProducer := chunkmap.Producer{InputModule: input.Prepared.input.Key(), ChunkModule: "cached-plan"}
if value.RequestedChunker != "cached-plan" || value.Producer != wantProducer || len(value.Chunks) != 1 {
t.Fatalf("chunk map=%#v", value)
}
}
}
func chunkPlanRecord(t *testing.T, prepared *PreparedPipeline, plan source.ChunkPlan) ChunkPlanRecord {
t.Helper()
digest, err := source.DigestChunkPlan(plan)
if err != nil {
t.Fatal(err)
}
return ChunkPlanRecord{
SchemaVersion: ChunkPlanSchemaVersion,
SourceDigest: plan.SourceDigest,
PlanDigest: digest,
Plan: source.CloneChunkPlan(plan),
Producer: ChunkPlanProducer{InputModule: prepared.input.Key(), ChunkModule: prepared.chunker.Key()},
CreatedAt: time.Now().UTC(),
func TestRunnerDoesNotPublishRejectedChunkPlan(t *testing.T) {
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
return planResponse(t, typedTestPlan(typedTestDocument())), nil
}))
input := planCacheRunInput(t, engine, func(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
return contracts.ValidationResult{ReasonCode: "partition", CorrectionGuidance: "Choose a valid partition."}, nil
})
input.Prepared.resolved.ChunkControls.MaxAdditionalTurns = 0
encoder := &capturingChunkMapOutput{}
input.Prepared.output = encoder
input.RawInput = []byte("input")
_, err := New().Run(context.Background(), input)
if err == nil || len(encoder.requests) != 0 {
t.Fatalf("rejected plan error=%v output requests=%d", err, len(encoder.requests))
}
}
func TestDeterministicChunkMapDoesNotInventModelProvenance(t *testing.T) {
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
prepared := prepareRunnerProfile(t, catalog, typedResolutionProfile(), nil)
prepared.resolved.Chunk.LLMProfile = "configured-but-not-used"
encoder := &capturingChunkMapOutput{}
prepared.output = encoder
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")}); err != nil {
t.Fatal(err)
}
if len(encoder.requests) != 1 || encoder.requests[0].ChunkMap == nil {
t.Fatal("missing chunk map")
}
value, err := chunkmap.New().Decode(encoder.requests[0].ChunkMap.Content)
if err != nil || value.Producer.LLMProfile != "" {
t.Fatalf("model provenance=%#v %v", value.Producer, err)
}
}

View File

@@ -19,40 +19,20 @@ type laneExtractState struct {
fallbackCount int
index int
prepared preparedLaneExecutor
deps []CheckpointFingerprint
decision CheckpointDecision
reuseEligible bool
values []erasedExtractArtifact
serialized []CheckpointArtifact
diagnostics []contracts.DiagnosticGroup
rejected []contracts.RejectedOutput
results map[int]extractJobResult
remaining int
failed bool
terminal bool
output RunOutput
}
type finalizedExtractResults struct {
accepted []erasedExtractArtifact
serialized []CheckpointArtifact
diagnostics []contracts.DiagnosticGroup
rejected []contracts.RejectedOutput
decision CheckpointDecision
reuseEligible bool
}
func loadExtract(loader CheckpointLoader, stepID, laneID, moduleKey string, deps []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
if stepAware, ok := loader.(StepCheckpointLoader); ok {
return stepAware.ExtractForStep(stepID, laneID, moduleKey, deps)
}
return loader.Extract(laneID, moduleKey, deps)
}
func recordExtract(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, outputs []CheckpointArtifact, rejected []contracts.RejectedOutput) error {
return checkpointExtractSucceeded(recorder, stepID, laneID, moduleKey, deps, outputs, rejected)
}
type extractJob struct {
lane *laneExtractState
chunk source.Chunk
@@ -62,9 +42,7 @@ type extractJobResult struct {
laneIndex int
chunkIndex int
value erasedExtractArtifact
serialized CheckpointArtifact
diagnostics []contracts.DiagnosticGroup
rejected *contracts.RejectedOutput
outcome contracts.ProducerOutcome
err error
}
@@ -82,17 +60,14 @@ type orderedRunError struct {
err error
}
func (r *Runner) runLanes(parent context.Context, input RunInput, step PreparedPipelineStep, checkpoints CheckpointRecorder, loader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk) (RunOutput, error) {
func (r *Runner) runLanes(parent context.Context, input RunInput, step PreparedPipelineStep, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk) (RunOutput, error) {
output := RunOutput{Manifest: manifestFromPipeline(input)}
states, err := initializeLaneStates(input, step, checkpoints, loader, doc, chunks, &output)
states, err := initializeLaneStates(input, step, chunks, &output)
if err != nil {
if mergeErr := mergeTerminalLaneStates(&output, states); mergeErr != nil {
return output, errors.Join(err, mergeErr)
}
return output, err
}
completedOutputs, runErrors := r.runLaneEngine(parent, laneEngineConfig{
input: input, checkpoints: checkpoints, loader: loader, doc: doc,
input: input, doc: doc,
sourceInput: sourceInput, sessionID: sessionID, chunks: chunks, states: states,
})
if err := mergeCompletedLanes(&output, completedOutputs); err != nil {
@@ -104,46 +79,22 @@ func (r *Runner) runLanes(parent context.Context, input RunInput, step PreparedP
return output, nil
}
func initializeLaneStates(input RunInput, step PreparedPipelineStep, checkpoints CheckpointRecorder, loader CheckpointLoader, doc *source.SourceDocument, chunks []source.Chunk, output *RunOutput) ([]*laneExtractState, error) {
func initializeLaneStates(input RunInput, step PreparedPipelineStep, chunks []source.Chunk, output *RunOutput) ([]*laneExtractState, error) {
states := make([]*laneExtractState, len(step.lanes))
for i, prepared := range step.lanes {
if prepared.typed == nil {
return states, fmt.Errorf("typed lane %q executor is not prepared", prepared.resolved.ID)
return nil, fmt.Errorf("typed lane %q executor is not prepared", prepared.resolved.ID)
}
if err := setTypedLaneManifestMetadata(output, prepared.resolved.ID, prepared.typed.extractor, prepared.typed.merger, prepared.typed.normalizer); err != nil {
return states, err
return nil, err
}
if input.CheckpointPolicy.requiresReusable(input.stepID, prepared.resolved.ID) && !input.CheckpointPolicy.forced(input.stepID, prepared.resolved.ID) {
if input.reuseIneligible || !laneReferencesReuseEligible(input, prepared.resolved) {
return states, fmt.Errorf("required reusable checkpoint unavailable for step %q lane %q: generated input lineage is validation-incomplete", input.stepID, prepared.resolved.ID)
}
state, err := hydrateRequiredLane(input, loader, doc, i, prepared)
states[i] = state
if err != nil {
return states, err
}
continue
}
state, err := prepareLaneExtract(input, loader, doc, chunks, i, prepared, output)
if err != nil {
return states, err
}
if !state.decision.Reused && state.reuseEligible {
if err := checkpointExtractRunning(checkpoints, input.stepID, prepared.resolved.ID, prepared.resolved.Extract.Module, state.deps); err != nil {
return states, fmt.Errorf("write extract checkpoint for lane %q: %w", prepared.resolved.ID, err)
}
} else if err := finalizeLaneExtract(checkpoints, input.stepID, state); err != nil {
return states, err
}
states[i] = state
states[i] = &laneExtractState{index: i, prepared: prepared, reuseEligible: !input.reuseIneligible && referenceTargetReuseEligible(input, prepared.resolved.ExtractReferences), remaining: len(chunks), results: make(map[int]extractJobResult, len(chunks))}
}
return states, nil
}
type laneEngineConfig struct {
input RunInput
checkpoints CheckpointRecorder
loader CheckpointLoader
doc *source.SourceDocument
sourceInput contracts.LLMInputMaterial
sessionID string
@@ -200,13 +151,6 @@ func newLaneEngine(r *Runner, parent context.Context, config laneEngineConfig) *
func (e *laneEngine) run() ([]RunOutput, []orderedRunError) {
defer e.cancel()
for _, state := range e.states {
if state.terminal {
e.completedOutputs[state.index] = state.output
} else if state.decision.Reused {
e.pending = append(e.pending, state)
}
}
e.startWorkers()
e.collect()
close(e.jobs)
@@ -229,7 +173,7 @@ func (e *laneEngine) startWorkers() {
if err := e.ctx.Err(); err != nil {
completion.err = err
} else {
completion.output, completion.err = e.runner.continueLane(e.ctx, e.input, e.checkpoints, e.loader, e.doc, e.sourceInput, e.sessionID, e.chunks, job.lane)
completion.output, completion.err = e.runner.continueLane(e.ctx, e.input, e.doc, e.sourceInput, e.sessionID, e.chunks, job.lane)
}
e.results <- laneJobResult{completion: &completion}
}
@@ -244,12 +188,9 @@ func (e *laneEngine) nextJob() (laneJob, bool) {
if len(e.pending) > 0 {
return laneJob{lane: e.pending[0]}, true
}
for e.nextExtract < len(e.chunks)*len(e.states) {
if e.nextExtract < len(e.chunks)*len(e.states) {
state := e.states[e.nextExtract%len(e.states)]
if !state.terminal && !state.decision.Reused {
return laneJob{lane: state, chunk: &e.chunks[e.nextExtract/len(e.states)]}, true
}
e.nextExtract++
return laneJob{lane: state, chunk: &e.chunks[e.nextExtract/len(e.states)]}, true
}
return laneJob{}, false
}
@@ -303,9 +244,6 @@ func (e *laneEngine) handleExtractResult(result extractJobResult) {
if result.err != nil {
state.failed = true
e.runErrors = append(e.runErrors, orderedRunError{stage: 0, lane: state.index, chunk: result.chunkIndex, err: result.err})
if state.reuseEligible {
_ = checkpointExtractFailed(e.checkpoints, e.input.stepID, state.prepared.resolved.ID, state.prepared.resolved.Extract.Module, state.deps, result.err)
}
e.cancel()
} else {
state.results[result.chunkIndex] = result
@@ -313,12 +251,7 @@ func (e *laneEngine) handleExtractResult(result extractJobResult) {
if state.remaining != 0 || state.failed || e.ctx.Err() != nil {
return
}
if err := finalizeLaneExtract(e.checkpoints, e.input.stepID, state); err != nil {
state.failed = true
e.runErrors = append(e.runErrors, orderedRunError{stage: 0, lane: state.index, chunk: len(e.chunks), err: err})
e.cancel()
return
}
finalizeLaneExtract(state)
e.pending = append(e.pending, state)
}
@@ -330,52 +263,6 @@ func (e *laneEngine) handleCompletion(completion laneCompletion) {
}
}
func hydrateRequiredLane(input RunInput, loader CheckpointLoader, doc *source.SourceDocument, index int, prepared preparedLaneExecutor) (*laneExtractState, error) {
lane, typed := prepared.resolved, prepared.typed
local := RunOutput{Manifest: manifestFromPipeline(input)}
checkpoint, decision := loader.AcceptedNormalize(input.stepID, lane.ID, lane.Normalize.Module)
if decision.Reused {
decision = checkpointDecision(CheckpointDecisionReused, CheckpointReasonAcceptedArtifactReused)
if checkpoint.Output.LaneID != lane.ID || checkpoint.Output.ModuleKey != lane.Normalize.Module || checkpoint.Output.SourceID != doc.ID {
decision = checkpointDecision(CheckpointDecisionExecuted, CheckpointReasonArtifactPayloadInvalid)
}
}
resolution, err := resolveCheckpointDecision(&local, loader, input.CheckpointPolicy, StageNormalize, input.stepID, lane.ID, lane.Normalize.Module, decision, typed.codec, []CheckpointArtifact{checkpoint.Output})
decision = resolution.decision
state := &laneExtractState{index: index, prepared: prepared, decision: decision, reuseEligible: true, terminal: true, output: local}
if err != nil {
return state, err
}
hydrated := resolution.artifacts[0]
diagnostics, diagnosticErr := promoteCheckpointDiagnostics(hydrated.Diagnostics, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageNormalize, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Normalize.Module}, nil)
if diagnosticErr != nil {
return state, fmt.Errorf("promote reused accepted diagnostics: %w", diagnosticErr)
}
appendDiagnosticGroups(&local, diagnostics)
local.NormalizeOutputs = append(local.NormalizeOutputs, contracts.SerializedOutput{
StepID: input.stepID,
LaneID: lane.ID,
NormalizerKey: lane.Normalize.Module,
SourceID: doc.ID,
Artifact: contracts.CloneSerializedArtifact(hydrated.Artifact),
})
local.normalizeReuseEligibility = map[generatedOutputKey]bool{generatedOutputKeyFor(input.stepID, lane.ID): true}
state.output = local
return state, nil
}
func mergeTerminalLaneStates(output *RunOutput, states []*laneExtractState) error {
for _, state := range states {
if state == nil || !state.terminal {
continue
}
if err := mergeLaneOutput(output, state.output); err != nil {
return err
}
}
return nil
}
func mergeCompletedLanes(output *RunOutput, completedOutputs []RunOutput) error {
for i := range completedOutputs {
if err := mergeLaneOutput(output, completedOutputs[i]); err != nil {
@@ -385,49 +272,6 @@ func mergeCompletedLanes(output *RunOutput, completedOutputs []RunOutput) error
return nil
}
func prepareLaneExtract(input RunInput, loader CheckpointLoader, doc *source.SourceDocument, chunks []source.Chunk, index int, prepared preparedLaneExecutor, output *RunOutput) (*laneExtractState, error) {
lane, typed := prepared.resolved, prepared.typed
digest, err := joinedChunkDigest(chunks)
if err != nil {
return nil, fmt.Errorf("digest chunks for lane %q: %w", lane.ID, err)
}
extractReferences := operationReferenceSet(input, lane.ExtractReferences)
deps := append(digestFingerprints("chunks", digest), generatedReferenceDependencies(extractReferences)...)
state := &laneExtractState{index: index, prepared: prepared, deps: normalizeCheckpointFingerprints(deps), reuseEligible: !input.reuseIneligible && referenceTargetReuseEligible(input, lane.ExtractReferences), remaining: len(chunks), results: make(map[int]extractJobResult, len(chunks))}
if !state.reuseEligible {
state.decision = checkpointDecision(CheckpointDecisionExecuted, CheckpointReasonValidationIncompleteLineage)
recordCheckpointEvent(output, loader, string(StageExtract), input.stepID, lane.ID, lane.Extract.Module, state.decision)
return state, nil
}
cp, decision := loadExtract(loader, input.stepID, lane.ID, lane.Extract.Module, state.deps)
resolution, err := resolveCheckpointDecision(output, loader, input.CheckpointPolicy, StageExtract, input.stepID, lane.ID, lane.Extract.Module, decision, typed.codec, cp.Outputs)
if err != nil {
return nil, err
}
decision = resolution.decision
state.decision = decision
if decision.Reused {
state.remaining = 0
for i, stored := range resolution.artifacts {
value := resolution.values[i]
artifact := erasedExtractArtifact{LaneID: lane.ID, ExtractorKey: lane.Extract.Module, SourceID: doc.ID, ChunkID: stored.ChunkID, ChunkIndex: stored.ChunkIndex, ChunkRef: stored.ChunkRef, Value: value}
if stored.ChunkIndex >= 0 && stored.ChunkIndex < len(chunks) && artifact.ChunkRef == (source.SourceRef{}) {
artifact.ChunkRef = chunks[stored.ChunkIndex].Ref
}
state.values = append(state.values, artifact)
state.serialized = append(state.serialized, cloneCheckpointArtifact(stored))
chunk := source.Chunk{ID: stored.ChunkID, Index: stored.ChunkIndex}
diagnostics, diagnosticErr := promoteCheckpointDiagnostics(stored.Diagnostics, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module}, &chunk)
if diagnosticErr != nil {
return nil, fmt.Errorf("promote reused extract diagnostics: %w", diagnosticErr)
}
state.diagnostics = append(state.diagnostics, diagnostics...)
}
state.rejected = cloneRejectedOutputs(cp.Rejected)
}
return state, nil
}
func (r *Runner) runExtractJob(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, job extractJob) extractJobResult {
state := job.lane
lane, typed := state.prepared.resolved, state.prepared.typed
@@ -469,24 +313,22 @@ func (r *Runner) runExtractJob(ctx context.Context, input RunInput, doc *source.
result.err = fmt.Errorf("accept extract lane %q chunk %q: %w", lane.ID, chunk.ID, err)
return result
}
stored, err := checkpointArtifact(typed.codec, lane.ID, lane.Extract.Module, doc.ID, value)
stored, err := serializeArtifact(typed.codec, value, false)
if err != nil {
result.err = fmt.Errorf("encode extract lane %q chunk %q: %w", lane.ID, chunk.ID, err)
return result
}
stored.ChunkID, stored.ChunkIndex, stored.ChunkRef = chunk.ID, chunk.Index, chunk.Ref
result.diagnostics, err = operationDiagnosticGroups(extracted.Diagnostics, cacheDiagnostics, assessment, outcome, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module}, &chunk)
if err != nil {
result.err = err
return result
}
result.value = erasedExtractArtifact{LaneID: lane.ID, ExtractorKey: lane.Extract.Module, SourceID: doc.ID, ChunkID: chunk.ID, ChunkIndex: chunk.Index, ChunkRef: chunk.Ref, Value: value}
result.serialized, result.outcome = stored, outcome
result.value = erasedExtractArtifact{LaneID: lane.ID, ExtractorKey: lane.Extract.Module, SourceID: doc.ID, ChunkID: chunk.ID, ChunkIndex: chunk.Index, ChunkRef: chunk.Ref, Value: value, artifact: stored}
result.outcome = outcome
return result
}
func finalizeLaneExtract(checkpoints CheckpointRecorder, stepID string, state *laneExtractState) error {
lane := state.prepared.resolved
func finalizeLaneExtract(state *laneExtractState) {
indexes := make([]int, 0, len(state.results))
for index := range state.results {
indexes = append(indexes, index)
@@ -494,13 +336,7 @@ func finalizeLaneExtract(checkpoints CheckpointRecorder, stepID string, state *l
sort.Ints(indexes)
for _, index := range indexes {
result := state.results[index]
if result.rejected != nil {
state.rejected = append(state.rejected, *result.rejected)
state.diagnostics = append(state.diagnostics, result.diagnostics...)
continue
}
state.values = append(state.values, result.value)
state.serialized = append(state.serialized, result.serialized)
state.diagnostics = append(state.diagnostics, result.diagnostics...)
if !result.outcome.ReuseEligible {
state.reuseEligible = false
@@ -509,43 +345,38 @@ func finalizeLaneExtract(checkpoints CheckpointRecorder, stepID string, state *l
state.fallbackCount++
}
}
sort.SliceStable(state.values, func(i, j int) bool { return state.values[i].ChunkIndex < state.values[j].ChunkIndex })
sort.SliceStable(state.serialized, func(i, j int) bool { return state.serialized[i].ChunkIndex < state.serialized[j].ChunkIndex })
sort.SliceStable(state.rejected, func(i, j int) bool { return state.rejected[i].ChunkIndex < state.rejected[j].ChunkIndex })
if !state.decision.Reused && state.reuseEligible {
if err := recordExtract(checkpoints, stepID, lane.ID, lane.Extract.Module, state.deps, state.serialized, state.rejected); err != nil {
return fmt.Errorf("write extract checkpoint for lane %q: %w", lane.ID, err)
}
}
return nil
}
func (r *Runner) continueLane(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, loader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk, state *laneExtractState) (RunOutput, error) {
func (r *Runner) continueLane(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk, state *laneExtractState) (RunOutput, error) {
lane := state.prepared.resolved
local := RunOutput{Manifest: manifestFromPipeline(input), FallbackCount: state.fallbackCount}
results := finalizedExtractResults{
accepted: state.values,
serialized: state.serialized,
diagnostics: state.diagnostics,
rejected: state.rejected,
decision: state.decision,
reuseEligible: state.reuseEligible,
}
local.diagnosticGroups = append(local.diagnosticGroups, contracts.CloneDiagnosticCollection(contracts.DiagnosticCollection{Groups: results.diagnostics}).Groups...)
local.Rejected = append(local.Rejected, cloneRejectedOutputs(results.rejected)...)
if err := writeDebugTimed(input.Debug, path.Join("extract", fileio.EncodePathComponent(lane.ID), "input.json"), debugTimedEnvelope{Stage: string(StageExtract), StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": results.decision.Reused, "decision": results.decision, "source": debugSourceDocumentEnvelope(doc), "chunks": debugSourceChunkEnvelopes(chunks), "options": redactSensitiveMap(lane.Extract.Options), "metadata": redactSensitiveMap(input.Metadata)}}); err != nil {
if err := writeDebugTimed(input.Debug, path.Join("extract", fileio.EncodePathComponent(lane.ID), "input.json"), debugTimedEnvelope{Stage: string(StageExtract), StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"source": debugSourceDocumentEnvelope(doc), "chunks": debugSourceChunkEnvelopes(chunks), "options": redactSensitiveMap(lane.Extract.Options), "metadata": redactSensitiveMap(input.Metadata)}}); err != nil {
return local, &laneRunError{stage: StageExtract, err: err}
}
if err := writeDebugTimed(input.Debug, path.Join("extract", fileio.EncodePathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageExtract), StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": results.decision.Reused, "outputs": debugCheckpointArtifacts(results.serialized), "rejected": debugRejectedOutputEnvelopes(results.rejected)}}); err != nil {
if err := writeDebugTimed(input.Debug, path.Join("extract", fileio.EncodePathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageExtract), StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"outputs": debugExtractArtifacts(results.accepted)}}); err != nil {
return local, &laneRunError{stage: StageExtract, err: err}
}
if len(results.accepted) == 0 {
return local, nil
}
err := r.continueTypedLane(ctx, input, checkpoints, loader, doc, sourceInput, sessionID, state.prepared, results, &local)
err := r.continueTypedLane(ctx, input, doc, sourceInput, sessionID, state.prepared, results, &local)
return local, err
}
func debugExtractArtifacts(values []erasedExtractArtifact) []any {
outputs := make([]any, 0, len(values))
for _, value := range values {
outputs = append(outputs, map[string]any{"chunk_id": value.ChunkID, "chunk_index": value.ChunkIndex, "chunk_ref": value.ChunkRef, "output": debugSerializedOutputEnvelope(contracts.SerializedOutput{LaneID: value.LaneID, SourceID: value.SourceID, Artifact: value.artifact})})
}
return outputs
}
func classifyLaneError(lane, sentinel int, err error) orderedRunError {
stage := 1
var laneErr *laneRunError
@@ -603,7 +434,6 @@ func mergeLaneOutput(dst *RunOutput, src RunOutput) error {
dst.NormalizeOutputs = append(dst.NormalizeOutputs, cloneSerializedOutputs(src.NormalizeOutputs)...)
dst.Rejected = append(dst.Rejected, cloneRejectedOutputs(src.Rejected)...)
appendDiagnosticGroups(dst, src.diagnosticGroups)
dst.CheckpointEvents = append(dst.CheckpointEvents, src.CheckpointEvents...)
dst.ValidationSummaries = append(dst.ValidationSummaries, cloneValidationSummaries(src.ValidationSummaries)...)
if len(src.normalizeReuseEligibility) > 0 {
if dst.normalizeReuseEligibility == nil {

View File

@@ -1,11 +1,7 @@
package pipeline
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"path"
"time"
@@ -15,74 +11,6 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func loadMerge(loader CheckpointLoader, stepID, laneID, moduleKey string, deps []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision) {
if stepAware, ok := loader.(StepCheckpointLoader); ok {
return stepAware.MergeForStep(stepID, laneID, moduleKey, deps)
}
return loader.Merge(laneID, moduleKey, deps)
}
func loadNormalize(loader CheckpointLoader, stepID, laneID, moduleKey string, deps []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision) {
if stepAware, ok := loader.(StepCheckpointLoader); ok {
return stepAware.NormalizeForStep(stepID, laneID, moduleKey, deps)
}
return loader.Normalize(laneID, moduleKey, deps)
}
func recordMerge(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, output CheckpointArtifact) error {
return checkpointMergeSucceeded(recorder, stepID, laneID, moduleKey, deps, output)
}
func recordNormalize(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, output CheckpointArtifact) error {
return checkpointNormalizeSucceeded(recorder, stepID, laneID, moduleKey, deps, output)
}
func cloneCheckpointArtifact(output CheckpointArtifact) CheckpointArtifact {
output.Artifact = contracts.CloneSerializedArtifact(output.Artifact)
output.Diagnostics = cloneCheckpointDiagnostics(output.Diagnostics)
return output
}
func hydrateCheckpointArtifact(codec artifactCodecEntry, output CheckpointArtifact, value any) (CheckpointArtifact, error) {
output.Artifact.Schema = contracts.CloneArtifactSchema(codec.spec.Schema)
if codec.metadata != nil {
metadata, err := codec.metadata(value)
if err != nil {
return CheckpointArtifact{}, fmt.Errorf("clone artifact metadata: %w", err)
}
output.Artifact.Metadata = metadata
} else {
output.Artifact.Metadata = nil
}
return output, nil
}
func artifactCheckpointDigests(outputs []CheckpointArtifact) []CheckpointFingerprint {
values := make([]CheckpointFingerprint, 0, len(outputs))
for i, output := range outputs {
sum := sha256.Sum256(output.Artifact.Content)
values = append(values, CheckpointFingerprint{Name: fmt.Sprintf("artifact[%d]", i), Value: "sha256:" + hex.EncodeToString(sum[:])})
}
return normalizeCheckpointFingerprints(values)
}
func debugCheckpointArtifact(output CheckpointArtifact) map[string]any {
artifact := output.Artifact
schema := contracts.CloneArtifactSchema(artifact.Schema)
digest := output.SchemaDigest
if digest == "" {
digest = contracts.DigestArtifactSchema(schema)
}
schema.JSONSchema = nil
content := debugContentEnvelope(artifact.Content, artifact.MediaType, artifact.Metadata, nil)
content.ContentDigest = debugContentDigest(artifact.Content)
return map[string]any{"lane_id": output.LaneID, "module_key": output.ModuleKey, "source_id": output.SourceID, "chunk_id": output.ChunkID, "chunk_index": output.ChunkIndex, "chunk_ref": output.ChunkRef, "artifact_kind": artifact.Kind, "schema": schema, "schema_digest": digest, "content": content}
}
func debugCheckpointArtifacts(outputs []CheckpointArtifact) []map[string]any {
if len(outputs) == 0 {
return nil
}
out := make([]map[string]any, 0, len(outputs))
for _, output := range outputs {
out = append(out, debugCheckpointArtifact(output))
}
return out
}
func serializeArtifact(codec artifactCodecEntry, value any, candidate bool) (contracts.SerializedArtifact, error) {
encode := codec.encode
if candidate {
@@ -103,82 +31,6 @@ func serializeArtifact(codec artifactCodecEntry, value any, candidate bool) (con
return contracts.SerializedArtifact{Kind: codec.spec.Kind, Schema: contracts.CloneArtifactSchema(schema), MediaType: codec.spec.MediaType, Content: append([]byte(nil), content...), Metadata: metadata}, nil
}
type checkpointArtifactValidationError struct {
code CheckpointReasonCode
err error
}
func (e *checkpointArtifactValidationError) Error() string { return e.err.Error() }
func (e *checkpointArtifactValidationError) Unwrap() error { return e.err }
func checkpointArtifactValidationFailure(code CheckpointReasonCode, format string, args ...any) error {
return &checkpointArtifactValidationError{code: code, err: fmt.Errorf(format, args...)}
}
func checkpointArtifactReasonCode(err error) CheckpointReasonCode {
var validationErr *checkpointArtifactValidationError
if errors.As(err, &validationErr) {
return validationErr.code
}
return CheckpointReasonArtifactPayloadInvalid
}
func decodeCheckpointArtifact(codec artifactCodecEntry, artifact CheckpointArtifact) (any, error) {
expectedDigest := contracts.DigestArtifactSchema(codec.spec.Schema)
if artifact.Artifact.Kind != codec.spec.Kind {
return nil, checkpointArtifactValidationFailure(CheckpointReasonArtifactCodecIncompatible, "artifact kind %q does not match codec %q", artifact.Artifact.Kind, codec.spec.Kind)
}
if artifact.Artifact.Schema.ID != codec.spec.Schema.ID || artifact.Artifact.Schema.Name != codec.spec.Schema.Name || artifact.Artifact.Schema.Version != codec.spec.Schema.Version {
return nil, checkpointArtifactValidationFailure(CheckpointReasonArtifactCodecIncompatible, "artifact schema %q version %q does not match codec schema %q version %q", artifact.Artifact.Schema.ID, artifact.Artifact.Schema.Version, codec.spec.Schema.ID, codec.spec.Schema.Version)
}
if artifact.SchemaDigest != expectedDigest {
return nil, checkpointArtifactValidationFailure(CheckpointReasonArtifactCodecIncompatible, "artifact schema digest %q does not match codec schema digest %q", artifact.SchemaDigest, expectedDigest)
}
if artifact.Artifact.MediaType != codec.spec.MediaType {
return nil, checkpointArtifactValidationFailure(CheckpointReasonArtifactCodecIncompatible, "artifact media type %q does not match codec media type %q", artifact.Artifact.MediaType, codec.spec.MediaType)
}
value, err := codec.decode(append([]byte(nil), artifact.Artifact.Content...))
if err != nil {
return nil, checkpointArtifactValidationFailure(CheckpointReasonArtifactPayloadInvalid, "decode artifact payload: %w", err)
}
return value, nil
}
func decodeCanonicalCheckpointArtifact(codec artifactCodecEntry, artifact CheckpointArtifact) (any, CheckpointArtifact, error) {
value, err := decodeCheckpointArtifact(codec, artifact)
if err != nil {
return nil, CheckpointArtifact{}, err
}
canonical, err := serializeArtifact(codec, value, false)
if err != nil {
return nil, CheckpointArtifact{}, checkpointArtifactValidationFailure(CheckpointReasonArtifactPayloadInvalid, "encode canonical artifact: %w", err)
}
if canonical.Kind != artifact.Artifact.Kind || canonical.Schema.ID != artifact.Artifact.Schema.ID || canonical.Schema.Name != artifact.Artifact.Schema.Name || canonical.Schema.Version != artifact.Artifact.Schema.Version || canonical.MediaType != artifact.Artifact.MediaType || !bytes.Equal(canonical.Content, artifact.Artifact.Content) || checkpointContentDigest(canonical.Content) != checkpointContentDigest(artifact.Artifact.Content) {
return nil, CheckpointArtifact{}, checkpointArtifactValidationFailure(CheckpointReasonArtifactNotCanonical, "stored artifact is not canonical")
}
hydrated, err := hydrateCheckpointArtifact(codec, cloneCheckpointArtifact(artifact), value)
if err != nil {
return nil, CheckpointArtifact{}, checkpointArtifactValidationFailure(CheckpointReasonArtifactPayloadInvalid, "hydrate checkpoint artifact: %w", err)
}
return value, hydrated, nil
}
func checkpointArtifact(codec artifactCodecEntry, laneID, moduleKey, sourceID string, value any) (CheckpointArtifact, error) {
serialized, err := serializeArtifact(codec, value, false)
if err != nil {
return CheckpointArtifact{}, err
}
return CheckpointArtifact{LaneID: laneID, ModuleKey: moduleKey, SourceID: sourceID, Artifact: serialized, SchemaDigest: contracts.DigestArtifactSchema(serialized.Schema)}, nil
}
func serializeCandidateArtifact(codec artifactCodecEntry, laneID, moduleKey, sourceID string, value any) (CheckpointArtifact, error) {
serialized, err := serializeArtifact(codec, value, true)
if err != nil {
return CheckpointArtifact{}, err
}
return CheckpointArtifact{LaneID: laneID, ModuleKey: moduleKey, SourceID: sourceID, Artifact: serialized, SchemaDigest: contracts.DigestArtifactSchema(serialized.Schema)}, nil
}
type laneRunError struct {
stage ModuleStage
err error
@@ -189,17 +41,17 @@ func (e *laneRunError) Unwrap() error { return e.err }
type mergeStageResult struct {
artifact erasedMergeArtifact
serialized CheckpointArtifact
serialized contracts.SerializedArtifact
reuseEligible bool
}
type normalizeStageResult struct {
serialized CheckpointArtifact
serialized contracts.SerializedArtifact
accepted bool
reuseEligible bool
}
func (r *Runner) continueTypedLane(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, loader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, prepared preparedLaneExecutor, extracts finalizedExtractResults, output *RunOutput) error {
func (r *Runner) continueTypedLane(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, prepared preparedLaneExecutor, extracts finalizedExtractResults, output *RunOutput) error {
lane := prepared.resolved
if prepared.typed == nil {
return fmt.Errorf("typed lane %q executor is not prepared", lane.ID)
@@ -207,16 +59,16 @@ func (r *Runner) continueTypedLane(ctx context.Context, input RunInput, checkpoi
if err := setTypedLaneManifestMetadata(output, lane.ID, prepared.typed.extractor, prepared.typed.merger, prepared.typed.normalizer); err != nil {
return err
}
merged, err := r.runMergeStage(ctx, input, checkpoints, loader, doc, sourceInput, sessionID, prepared, extracts, output)
merged, err := r.runMergeStage(ctx, input, doc, sourceInput, sessionID, prepared, extracts, output)
if err != nil {
return &laneRunError{stage: StageMerge, err: err}
}
normalized, err := r.runNormalizeStage(ctx, input, checkpoints, loader, doc, sourceInput, sessionID, prepared, merged.artifact, merged.serialized, merged.reuseEligible, output)
normalized, err := r.runNormalizeStage(ctx, input, doc, sourceInput, sessionID, prepared, merged.artifact, merged.reuseEligible, output)
if err != nil {
return &laneRunError{stage: StageNormalize, err: err}
}
if normalized.accepted {
output.NormalizeOutputs = append(output.NormalizeOutputs, contracts.SerializedOutput{StepID: input.stepID, LaneID: lane.ID, NormalizerKey: lane.Normalize.Module, SourceID: doc.ID, Artifact: contracts.CloneSerializedArtifact(normalized.serialized.Artifact)})
output.NormalizeOutputs = append(output.NormalizeOutputs, contracts.SerializedOutput{StepID: input.stepID, LaneID: lane.ID, NormalizerKey: lane.Normalize.Module, SourceID: doc.ID, Artifact: contracts.CloneSerializedArtifact(normalized.serialized)})
if output.normalizeReuseEligibility == nil {
output.normalizeReuseEligibility = make(map[generatedOutputKey]bool)
}
@@ -225,45 +77,18 @@ func (r *Runner) continueTypedLane(ctx context.Context, input RunInput, checkpoi
return nil
}
func (r *Runner) runMergeStage(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, loader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, prepared preparedLaneExecutor, extracts finalizedExtractResults, output *RunOutput) (mergeStageResult, error) {
func (r *Runner) runMergeStage(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, prepared preparedLaneExecutor, extracts finalizedExtractResults, output *RunOutput) (mergeStageResult, error) {
state := mergeStageResult{}
lane, typed := prepared.resolved, prepared.typed
references := operationReferenceSet(input, lane.MergeReferences)
state.reuseEligible = extracts.reuseEligible && !input.reuseIneligible && referenceTargetReuseEligible(input, lane.MergeReferences)
deps := normalizeCheckpointFingerprints(append(artifactCheckpointDigests(extracts.serialized), generatedReferenceDependencies(references)...))
var checkpoint MergeCheckpoint
decision := checkpointDecision(CheckpointDecisionExecuted, CheckpointReasonValidationIncompleteLineage)
if state.reuseEligible {
checkpoint, decision = loadMerge(loader, input.stepID, lane.ID, lane.Merge.Module, deps)
}
resolution, err := resolveCheckpointDecision(output, loader, input.CheckpointPolicy, StageMerge, input.stepID, lane.ID, lane.Merge.Module, decision, typed.codec, []CheckpointArtifact{checkpoint.Output})
if err != nil {
return state, err
}
if resolution.decision.Reused {
state.serialized = resolution.artifacts[0]
state.artifact = erasedMergeArtifact{LaneID: lane.ID, MergerKey: lane.Merge.Module, SourceID: doc.ID, Value: resolution.values[0]}
diagnostics, err := promoteCheckpointDiagnostics(state.serialized.Diagnostics, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageMerge, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Merge.Module}, nil)
if err != nil {
return state, err
}
appendDiagnosticGroups(output, diagnostics)
return state, nil
}
if err := ctx.Err(); err != nil {
return state, err
}
if state.reuseEligible {
if err := checkpointMergeRunning(checkpoints, input.stepID, lane.ID, lane.Merge.Module, deps); err != nil {
return state, err
}
}
inputs := make([]contracts.ExtractArtifact[any], len(extracts.accepted))
for i, value := range extracts.accepted {
inputs[i] = contracts.ExtractArtifact[any]{LaneID: value.LaneID, ExtractorKey: value.ExtractorKey, SourceID: value.SourceID, ChunkID: value.ChunkID, ChunkIndex: value.ChunkIndex, ChunkRef: value.ChunkRef, Value: value.Value}
}
if err := ctx.Err(); err != nil {
return state, err
}
metadata, err := cloneMetadata(input.Metadata)
if err != nil {
return state, err
@@ -282,16 +107,13 @@ func (r *Runner) runMergeStage(ctx context.Context, input RunInput, checkpoints
operation.Cache = artifactProducerCache(input, typed, lane.MergeExecutionClass, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageMerge, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Merge.Module}, doc, nil, &cacheDiagnostics)
produced, err := typed.merge(ctx, typed.merger, contracts.TypedMergeRequest[any]{Operation: operation, Source: doc, LaneID: lane.ID, ExtractOutputs: inputs, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: lane.Merge.LLMProfile, Metadata: metadata})
if err != nil {
if state.reuseEligible {
_ = checkpointMergeFailed(checkpoints, input.stepID, lane.ID, lane.Merge.Module, deps, err)
}
return state, fmt.Errorf("merge lane %q: %w", lane.ID, err)
}
value, outcome, err := completeProducerOperation(ctx, lane.MergeExecutionClass, operation, produced.Value, produced.Outcome, produced.Fallback)
if err != nil {
return state, fmt.Errorf("accept merge lane %q: %w", lane.ID, err)
}
state.serialized, err = checkpointArtifact(typed.codec, lane.ID, lane.Merge.Module, doc.ID, value)
state.serialized, err = serializeArtifact(typed.codec, value, false)
if err != nil {
return state, fmt.Errorf("encode merge lane %q: %w", lane.ID, err)
}
@@ -305,51 +127,20 @@ func (r *Runner) runMergeStage(ctx context.Context, input RunInput, checkpoints
}
state.reuseEligible = outcome.ReuseEligible
state.artifact = erasedMergeArtifact{LaneID: lane.ID, MergerKey: lane.Merge.Module, SourceID: doc.ID, Value: value}
if state.reuseEligible {
if err := recordMerge(checkpoints, input.stepID, lane.ID, lane.Merge.Module, deps, state.serialized); err != nil {
return state, err
}
}
if err := writeDebugTimed(input.Debug, path.Join("merge", fileio.EncodePathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageMerge), StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Merge.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"outcome": outcome, "output": debugCheckpointArtifact(state.serialized)}}); err != nil {
if err := writeDebugTimed(input.Debug, path.Join("merge", fileio.EncodePathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageMerge), StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Merge.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"outcome": outcome, "output": debugSerializedOutputEnvelope(contracts.SerializedOutput{StepID: input.stepID, LaneID: lane.ID, SourceID: doc.ID, Artifact: state.serialized})}}); err != nil {
return state, err
}
return state, nil
}
func (r *Runner) runNormalizeStage(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, loader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, prepared preparedLaneExecutor, merged erasedMergeArtifact, serializedMerge CheckpointArtifact, upstreamReuseEligible bool, output *RunOutput) (normalizeStageResult, error) {
func (r *Runner) runNormalizeStage(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, prepared preparedLaneExecutor, merged erasedMergeArtifact, upstreamReuseEligible bool, output *RunOutput) (normalizeStageResult, error) {
state := normalizeStageResult{}
lane, typed := prepared.resolved, prepared.typed
references := operationReferenceSet(input, lane.NormalizeReferences)
state.reuseEligible = upstreamReuseEligible && !input.reuseIneligible && referenceTargetReuseEligible(input, lane.NormalizeReferences)
deps := normalizeCheckpointFingerprints(append(artifactCheckpointDigests([]CheckpointArtifact{serializedMerge}), generatedReferenceDependencies(references)...))
var checkpoint NormalizeCheckpoint
decision := checkpointDecision(CheckpointDecisionExecuted, CheckpointReasonValidationIncompleteLineage)
if state.reuseEligible {
checkpoint, decision = loadNormalize(loader, input.stepID, lane.ID, lane.Normalize.Module, deps)
}
resolution, err := resolveCheckpointDecision(output, loader, input.CheckpointPolicy, StageNormalize, input.stepID, lane.ID, lane.Normalize.Module, decision, typed.codec, []CheckpointArtifact{checkpoint.Output})
if err != nil {
return state, err
}
if resolution.decision.Reused {
state.serialized = resolution.artifacts[0]
state.accepted = true
diagnostics, err := promoteCheckpointDiagnostics(state.serialized.Diagnostics, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageNormalize, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Normalize.Module}, nil)
if err != nil {
return state, err
}
appendDiagnosticGroups(output, diagnostics)
return state, nil
}
if err := ctx.Err(); err != nil {
return state, err
}
if state.reuseEligible {
if err := checkpointNormalizeRunning(checkpoints, input.stepID, lane.ID, lane.Normalize.Module, deps); err != nil {
return state, err
}
}
metadata, err := cloneMetadata(input.Metadata)
if err != nil {
return state, err
@@ -368,16 +159,13 @@ func (r *Runner) runNormalizeStage(ctx context.Context, input RunInput, checkpoi
operation.Cache = artifactProducerCache(input, typed, lane.NormalizeExecutionClass, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageNormalize, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Normalize.Module}, doc, nil, &cacheDiagnostics)
produced, err := typed.normalize(ctx, typed.normalizer, contracts.TypedNormalizeRequest[any]{Operation: operation, Source: doc, LaneID: lane.ID, MergeOutput: contracts.MergeArtifact[any]{LaneID: merged.LaneID, MergerKey: merged.MergerKey, SourceID: merged.SourceID, Value: merged.Value}, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: lane.Normalize.LLMProfile, Metadata: metadata})
if err != nil {
if state.reuseEligible {
_ = checkpointNormalizeFailed(checkpoints, input.stepID, lane.ID, lane.Normalize.Module, deps, err)
}
return state, fmt.Errorf("normalize lane %q: %w", lane.ID, err)
}
value, outcome, err := completeProducerOperation(ctx, lane.NormalizeExecutionClass, operation, produced.Value, produced.Outcome, produced.Fallback)
if err != nil {
return state, fmt.Errorf("accept normalize lane %q: %w", lane.ID, err)
}
state.serialized, err = checkpointArtifact(typed.codec, lane.ID, lane.Normalize.Module, doc.ID, value)
state.serialized, err = serializeArtifact(typed.codec, value, false)
if err != nil {
return state, fmt.Errorf("encode normalize lane %q: %w", lane.ID, err)
}
@@ -391,12 +179,7 @@ func (r *Runner) runNormalizeStage(ctx context.Context, input RunInput, checkpoi
}
state.reuseEligible = outcome.ReuseEligible
state.accepted = true
if state.reuseEligible {
if err := recordNormalize(checkpoints, input.stepID, lane.ID, lane.Normalize.Module, deps, state.serialized); err != nil {
return state, err
}
}
if err := writeDebugTimed(input.Debug, path.Join("normalize", fileio.EncodePathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageNormalize), StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Normalize.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"outcome": outcome, "output": debugCheckpointArtifact(state.serialized)}}); err != nil {
if err := writeDebugTimed(input.Debug, path.Join("normalize", fileio.EncodePathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageNormalize), StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Normalize.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"outcome": outcome, "output": debugSerializedOutputEnvelope(contracts.SerializedOutput{StepID: input.stepID, LaneID: lane.ID, SourceID: doc.ID, Artifact: state.serialized})}}); err != nil {
return state, err
}
return state, nil

View File

@@ -1,165 +0,0 @@
package pipeline
import (
"context"
"encoding/json"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
type requiredCheckpointLoader struct {
CheckpointLoader
checkpoint ExtractCheckpoint
decision CheckpointDecision
}
func (l requiredCheckpointLoader) Enabled() bool { return true }
func (l requiredCheckpointLoader) Extract(string, string, []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
return l.checkpoint, l.decision
}
func (l requiredCheckpointLoader) AcceptedNormalize(string, string, string) (NormalizeCheckpoint, CheckpointDecision) {
output := CheckpointArtifact{}
if len(l.checkpoint.Outputs) > 0 {
output = l.checkpoint.Outputs[0]
}
return NormalizeCheckpoint{Output: output}, l.decision
}
func TestRequiredCheckpointFailureRetainsDecision(t *testing.T) {
tests := []struct {
name string
decision CheckpointDecision
corrupt bool
wantCode CheckpointReasonCode
wantAction CheckpointDecisionCategory
}{
{"missing", NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonMissing), false, CheckpointReasonMissing, CheckpointDecisionExecuted},
{"corrupt", NewCheckpointDecision(CheckpointDecisionReused, CheckpointReasonReused), true, CheckpointReasonArtifactNotCanonical, CheckpointDecisionExecuted},
{"incompatible", NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonArtifactCodecIncompatible), false, CheckpointReasonArtifactCodecIncompatible, CheckpointDecisionExecuted},
{"dependency invalidated", NewCheckpointDecision(CheckpointDecisionDependencyInvalidated, CheckpointReasonDependencyMismatch), false, CheckpointReasonDependencyMismatch, CheckpointDecisionDependencyInvalidated},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
prepared := preparedConcurrentPipeline(t, 1)
step := prepared.Steps[0]
lane := step.lanes[0]
doc := prepared.input.(*typedTestInput).doc
checkpoint := ExtractCheckpoint{}
if test.corrupt {
stored, err := checkpointArtifact(lane.typed.codec, lane.resolved.ID, lane.resolved.Normalize.Module, doc.ID, codecNotes{Items: []string{"stored"}})
if err != nil {
t.Fatal(err)
}
stored.Artifact.Content = []byte(`{"items": ["stored"]}`)
checkpoint.Outputs = []CheckpointArtifact{stored}
}
loader := requiredCheckpointLoader{CheckpointLoader: NoopCheckpointLoader(), checkpoint: checkpoint, decision: test.decision}
policy := CheckpointExecutionPolicy{RequireReusableLanes: map[string]struct{}{CheckpointLaneKey(step.ID, lane.resolved.ID): {}}}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoint: loader, CheckpointPolicy: policy})
if err == nil || !strings.Contains(err.Error(), step.ID) || !strings.Contains(err.Error(), lane.resolved.ID) || !strings.Contains(err.Error(), string(test.wantCode)) {
t.Fatalf("Run() error = %v, want step, lane, and reason code %q", err, test.wantCode)
}
var found bool
for _, event := range output.CheckpointEvents {
if event.Stage == string(StageNormalize) && event.StepID == step.ID && event.LaneID == lane.resolved.ID {
found = true
if event.Action != test.wantAction || event.ReasonCode != test.wantCode {
t.Fatalf("checkpoint event = %#v, want action %q and reason %q", event, test.wantAction, test.wantCode)
}
if event.Detail == "" || event.Detail != event.Reason {
t.Fatalf("checkpoint event detail is not code-owned compatibility text: %#v", event)
}
}
}
if !found {
t.Fatalf("required checkpoint decision missing from %#v", output.CheckpointEvents)
}
var manifestFound bool
for _, decision := range output.Manifest.CheckpointDecisions {
if decision.Stage == string(StageNormalize) && decision.StepID == step.ID && decision.LaneID == lane.resolved.ID {
manifestFound = decision.Category == string(test.wantAction) && decision.ReasonCode == string(test.wantCode)
}
}
if !manifestFound {
t.Fatalf("manifest checkpoint decision missing category %q and code %q: %#v", test.wantAction, test.wantCode, output.Manifest.CheckpointDecisions)
}
encoded, marshalErr := json.Marshal(output.CheckpointEvents)
if marshalErr != nil || !strings.Contains(string(encoded), `"category":"`+string(test.wantAction)+`"`) || !strings.Contains(string(encoded), `"reason_code":"`+string(test.wantCode)+`"`) {
t.Fatalf("checkpoint event JSON = %s, error = %v", encoded, marshalErr)
}
})
}
}
func TestCheckpointDecisionDetailDoesNotCopyUnknownReasonCode(t *testing.T) {
const sentinel = "opaque-sensitive-value-78421"
decision := NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonCode(sentinel))
if decision.Detail == "" || decision.Detail != decision.Reason {
t.Fatalf("decision detail is not bounded compatibility text: %#v", decision)
}
if strings.Contains(decision.Detail, sentinel) || strings.Contains(decision.Reason, sentinel) {
t.Fatalf("decision detail copied caller-controlled reason code: %#v", decision)
}
}
func TestDecodeCheckpointArtifactRejectsIncompatibleCodecIdentityAndBytes(t *testing.T) {
registry := NewArtifactCodecRegistry()
if err := RegisterArtifactCodec(registry, notesCodec()); err != nil {
t.Fatalf("RegisterArtifactCodec: %v", err)
}
codec, _, err := registry.entry("test/notes")
if err != nil {
t.Fatalf("entry: %v", err)
}
artifact, err := serializeArtifact(codec, codecNotes{Items: []string{"one"}}, false)
if err != nil {
t.Fatalf("serializeArtifact: %v", err)
}
base := CheckpointArtifact{Artifact: artifact, SchemaDigest: contracts.DigestArtifactSchema(artifact.Schema)}
tests := []struct {
name string
mutate func(*CheckpointArtifact)
want string
}{
{name: "missing kind", mutate: func(v *CheckpointArtifact) { v.Artifact.Kind = "" }, want: "artifact kind"},
{name: "schema version", mutate: func(v *CheckpointArtifact) { v.Artifact.Schema.Version = "v999" }, want: "does not match codec schema"},
{name: "schema digest", mutate: func(v *CheckpointArtifact) { v.SchemaDigest = "sha256:different" }, want: "schema digest"},
{name: "media type", mutate: func(v *CheckpointArtifact) { v.Artifact.MediaType = "text/plain" }, want: "media type"},
{name: "decode failure", mutate: func(v *CheckpointArtifact) { v.Artifact.Content = []byte(`{"items":[`) }, want: "unexpected EOF"},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
stored := cloneCheckpointArtifact(base)
test.mutate(&stored)
if _, err := decodeCheckpointArtifact(codec, stored); err == nil || !strings.Contains(err.Error(), test.want) {
t.Fatalf("decode error = %v, want %q", err, test.want)
}
})
}
}
func TestDecodeCanonicalCheckpointArtifactRejectsNonCanonicalBytes(t *testing.T) {
registry := NewArtifactCodecRegistry()
if err := RegisterArtifactCodec(registry, notesCodec()); err != nil {
t.Fatalf("RegisterArtifactCodec: %v", err)
}
codec, _, err := registry.entry("test/notes")
if err != nil {
t.Fatalf("entry: %v", err)
}
artifact, err := serializeArtifact(codec, codecNotes{Items: []string{"one"}}, false)
if err != nil {
t.Fatalf("serializeArtifact: %v", err)
}
stored := CheckpointArtifact{Artifact: artifact, SchemaDigest: contracts.DigestArtifactSchema(artifact.Schema)}
if _, _, err := decodeCanonicalCheckpointArtifact(codec, stored); err != nil {
t.Fatalf("canonical artifact rejected: %v", err)
}
stored.Artifact.Content = []byte(`{"items": ["one"]}`)
if _, _, err := decodeCanonicalCheckpointArtifact(codec, stored); err == nil {
t.Fatal("non-canonical artifact was accepted")
}
}

View File

@@ -1,11 +1,6 @@
package pipeline
import (
"sync"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
import "sync"
type lockedDebugRecorder struct {
inner DebugRecorder
@@ -39,217 +34,3 @@ func (r *lockedDebugRecorder) WriteBytes(name string, data []byte) error {
defer r.mu.Unlock()
return r.inner.WriteBytes(name, data)
}
type lockedCheckpointLoader struct {
inner CheckpointLoader
mu sync.Mutex
}
func synchronizedCheckpointLoader(inner CheckpointLoader) CheckpointLoader {
if _, ok := inner.(*lockedCheckpointLoader); ok {
return inner
}
return &lockedCheckpointLoader{inner: inner}
}
func (l *lockedCheckpointLoader) Enabled() bool {
l.mu.Lock()
defer l.mu.Unlock()
return l.inner.Enabled()
}
func (l *lockedCheckpointLoader) Source(key string) (SourceCheckpoint, CheckpointDecision) {
l.mu.Lock()
defer l.mu.Unlock()
return l.inner.Source(key)
}
func (l *lockedCheckpointLoader) Extract(lane, key string, deps []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
l.mu.Lock()
defer l.mu.Unlock()
return l.inner.Extract(lane, key, deps)
}
func (l *lockedCheckpointLoader) Merge(lane, key string, deps []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision) {
l.mu.Lock()
defer l.mu.Unlock()
return l.inner.Merge(lane, key, deps)
}
func (l *lockedCheckpointLoader) Normalize(lane, key string, deps []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision) {
l.mu.Lock()
defer l.mu.Unlock()
return l.inner.Normalize(lane, key, deps)
}
func (l *lockedCheckpointLoader) AcceptedNormalize(step, lane, key string) (NormalizeCheckpoint, CheckpointDecision) {
l.mu.Lock()
defer l.mu.Unlock()
return l.inner.AcceptedNormalize(step, lane, key)
}
func (l *lockedCheckpointLoader) ExtractForStep(step, lane, key string, deps []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
l.mu.Lock()
defer l.mu.Unlock()
if stepAware, ok := l.inner.(StepCheckpointLoader); ok {
return stepAware.ExtractForStep(step, lane, key, deps)
}
return l.inner.Extract(lane, key, deps)
}
func (l *lockedCheckpointLoader) MergeForStep(step, lane, key string, deps []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision) {
l.mu.Lock()
defer l.mu.Unlock()
if stepAware, ok := l.inner.(StepCheckpointLoader); ok {
return stepAware.MergeForStep(step, lane, key, deps)
}
return l.inner.Merge(lane, key, deps)
}
func (l *lockedCheckpointLoader) NormalizeForStep(step, lane, key string, deps []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision) {
l.mu.Lock()
defer l.mu.Unlock()
if stepAware, ok := l.inner.(StepCheckpointLoader); ok {
return stepAware.NormalizeForStep(step, lane, key, deps)
}
return l.inner.Normalize(lane, key, deps)
}
type lockedCheckpointRecorder struct {
inner CheckpointRecorder
mu sync.Mutex
}
func synchronizedCheckpointRecorder(inner CheckpointRecorder) CheckpointRecorder {
if _, ok := inner.(*lockedCheckpointRecorder); ok {
return inner
}
return &lockedCheckpointRecorder{inner: inner}
}
func (r *lockedCheckpointRecorder) call(fn func() error) error {
r.mu.Lock()
defer r.mu.Unlock()
return fn()
}
func (r *lockedCheckpointRecorder) SourceRunning(key string) error {
return r.call(func() error { return r.inner.SourceRunning(key) })
}
func (r *lockedCheckpointRecorder) SourceSucceeded(key string, doc *source.SourceDocument) error {
return r.call(func() error { return r.inner.SourceSucceeded(key, doc) })
}
func (r *lockedCheckpointRecorder) SourceFailed(key string, err error) error {
return r.call(func() error { return r.inner.SourceFailed(key, err) })
}
func (r *lockedCheckpointRecorder) ExtractRunning(lane, key string, deps []CheckpointFingerprint) error {
return r.call(func() error { return r.inner.ExtractRunning(lane, key, deps) })
}
func (r *lockedCheckpointRecorder) ExtractSucceeded(lane, key string, deps []CheckpointFingerprint, outputs []CheckpointArtifact, rejected []contracts.RejectedOutput) error {
return r.call(func() error { return r.inner.ExtractSucceeded(lane, key, deps, outputs, rejected) })
}
func (r *lockedCheckpointRecorder) ExtractFailed(lane, key string, deps []CheckpointFingerprint, err error) error {
return r.call(func() error { return r.inner.ExtractFailed(lane, key, deps, err) })
}
func (r *lockedCheckpointRecorder) MergeRunning(lane, key string, deps []CheckpointFingerprint) error {
return r.call(func() error { return r.inner.MergeRunning(lane, key, deps) })
}
func (r *lockedCheckpointRecorder) MergeSucceeded(lane, key string, deps []CheckpointFingerprint, output CheckpointArtifact) error {
return r.call(func() error { return r.inner.MergeSucceeded(lane, key, deps, output) })
}
func (r *lockedCheckpointRecorder) MergeRejected(lane, key string, deps []CheckpointFingerprint, rejected contracts.RejectedOutput) error {
return r.call(func() error { return r.inner.MergeRejected(lane, key, deps, rejected) })
}
func (r *lockedCheckpointRecorder) MergeFailed(lane, key string, deps []CheckpointFingerprint, err error) error {
return r.call(func() error { return r.inner.MergeFailed(lane, key, deps, err) })
}
func (r *lockedCheckpointRecorder) NormalizeRunning(lane, key string, deps []CheckpointFingerprint) error {
return r.call(func() error { return r.inner.NormalizeRunning(lane, key, deps) })
}
func (r *lockedCheckpointRecorder) NormalizeSucceeded(lane, key string, deps []CheckpointFingerprint, output CheckpointArtifact) error {
return r.call(func() error { return r.inner.NormalizeSucceeded(lane, key, deps, output) })
}
func (r *lockedCheckpointRecorder) NormalizeRejected(lane, key string, deps []CheckpointFingerprint, rejected contracts.RejectedOutput) error {
return r.call(func() error { return r.inner.NormalizeRejected(lane, key, deps, rejected) })
}
func (r *lockedCheckpointRecorder) NormalizeFailed(lane, key string, deps []CheckpointFingerprint, err error) error {
return r.call(func() error { return r.inner.NormalizeFailed(lane, key, deps, err) })
}
func (r *lockedCheckpointRecorder) ExtractRunningForStep(step, lane, key string, deps []CheckpointFingerprint) error {
return r.call(func() error {
if v, ok := r.inner.(StepCheckpointRecorder); ok {
return v.ExtractRunningForStep(step, lane, key, deps)
}
return r.inner.ExtractRunning(lane, key, deps)
})
}
func (r *lockedCheckpointRecorder) ExtractSucceededForStep(step, lane, key string, deps []CheckpointFingerprint, outputs []CheckpointArtifact, rejected []contracts.RejectedOutput) error {
return r.call(func() error {
if v, ok := r.inner.(StepCheckpointRecorder); ok {
return v.ExtractSucceededForStep(step, lane, key, deps, outputs, rejected)
}
return r.inner.ExtractSucceeded(lane, key, deps, outputs, rejected)
})
}
func (r *lockedCheckpointRecorder) ExtractFailedForStep(step, lane, key string, deps []CheckpointFingerprint, err error) error {
return r.call(func() error {
if v, ok := r.inner.(StepCheckpointRecorder); ok {
return v.ExtractFailedForStep(step, lane, key, deps, err)
}
return r.inner.ExtractFailed(lane, key, deps, err)
})
}
func (r *lockedCheckpointRecorder) MergeRunningForStep(step, lane, key string, deps []CheckpointFingerprint) error {
return r.call(func() error {
if v, ok := r.inner.(StepCheckpointRecorder); ok {
return v.MergeRunningForStep(step, lane, key, deps)
}
return r.inner.MergeRunning(lane, key, deps)
})
}
func (r *lockedCheckpointRecorder) MergeSucceededForStep(step, lane, key string, deps []CheckpointFingerprint, output CheckpointArtifact) error {
return r.call(func() error {
if v, ok := r.inner.(StepCheckpointRecorder); ok {
return v.MergeSucceededForStep(step, lane, key, deps, output)
}
return r.inner.MergeSucceeded(lane, key, deps, output)
})
}
func (r *lockedCheckpointRecorder) MergeRejectedForStep(step, lane, key string, deps []CheckpointFingerprint, rejected contracts.RejectedOutput) error {
return r.call(func() error {
if v, ok := r.inner.(StepCheckpointRecorder); ok {
return v.MergeRejectedForStep(step, lane, key, deps, rejected)
}
return r.inner.MergeRejected(lane, key, deps, rejected)
})
}
func (r *lockedCheckpointRecorder) MergeFailedForStep(step, lane, key string, deps []CheckpointFingerprint, err error) error {
return r.call(func() error {
if v, ok := r.inner.(StepCheckpointRecorder); ok {
return v.MergeFailedForStep(step, lane, key, deps, err)
}
return r.inner.MergeFailed(lane, key, deps, err)
})
}
func (r *lockedCheckpointRecorder) NormalizeRunningForStep(step, lane, key string, deps []CheckpointFingerprint) error {
return r.call(func() error {
if v, ok := r.inner.(StepCheckpointRecorder); ok {
return v.NormalizeRunningForStep(step, lane, key, deps)
}
return r.inner.NormalizeRunning(lane, key, deps)
})
}
func (r *lockedCheckpointRecorder) NormalizeSucceededForStep(step, lane, key string, deps []CheckpointFingerprint, output CheckpointArtifact) error {
return r.call(func() error {
if v, ok := r.inner.(StepCheckpointRecorder); ok {
return v.NormalizeSucceededForStep(step, lane, key, deps, output)
}
return r.inner.NormalizeSucceeded(lane, key, deps, output)
})
}
func (r *lockedCheckpointRecorder) NormalizeRejectedForStep(step, lane, key string, deps []CheckpointFingerprint, rejected contracts.RejectedOutput) error {
return r.call(func() error {
if v, ok := r.inner.(StepCheckpointRecorder); ok {
return v.NormalizeRejectedForStep(step, lane, key, deps, rejected)
}
return r.inner.NormalizeRejected(lane, key, deps, rejected)
})
}
func (r *lockedCheckpointRecorder) NormalizeFailedForStep(step, lane, key string, deps []CheckpointFingerprint, err error) error {
return r.call(func() error {
if v, ok := r.inner.(StepCheckpointRecorder); ok {
return v.NormalizeFailedForStep(step, lane, key, deps, err)
}
return r.inner.NormalizeFailed(lane, key, deps, err)
})
}

View File

@@ -14,6 +14,7 @@ type erasedExtractArtifact struct {
ChunkIndex int
ChunkRef source.SourceRef
Value any
artifact contracts.SerializedArtifact
}
type erasedMergeArtifact struct {

View File

@@ -181,7 +181,7 @@ func TestPrepareConstructsHeterogeneousTypedLanes(t *testing.T) {
}
}
func TestRunExecutesHeterogeneousTypedLanesWithCheckpointsDisabled(t *testing.T) {
func TestRunExecutesHeterogeneousTypedLanes(t *testing.T) {
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
resolved, err := ResolvePipeline(typedResolutionProfile(), ResolveOptions{}, catalog)
if err != nil {
@@ -198,9 +198,6 @@ func TestRunExecutesHeterogeneousTypedLanesWithCheckpointsDisabled(t *testing.T)
if len(output.NormalizeOutputs) != 2 {
t.Fatalf("normalize outputs = %#v, want two typed lane results", output.NormalizeOutputs)
}
if len(output.CheckpointEvents) != 0 {
t.Fatalf("checkpoint events = %#v, want none with checkpoint loading disabled", output.CheckpointEvents)
}
}
func TestResolveTypedLaneRejectsIncompatibleComposition(t *testing.T) {