Unify chunk-plan caching and remove checkpoint execution paths

This commit is contained in:
2026-11-22 02:24:45 +00:00
parent 36906cb3d1
commit 3b1b4c9bcf
26 changed files with 717 additions and 4258 deletions

View File

@@ -20,8 +20,7 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle"
"gitea.maximumdirect.net/eric/notarius/internal/core/fileio"
"gitea.maximumdirect.net/eric/notarius/internal/framework/checkpoint"
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkplan"
"gitea.maximumdirect.net/eric/notarius/internal/framework/artifactcache"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
frameworkdebug "gitea.maximumdirect.net/eric/notarius/internal/framework/debug"
frameworkllm "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
@@ -32,19 +31,18 @@ import (
)
type Options struct {
LLMClient promptkit.LLMClient
LLMRuntime promptkit.RuntimeConfig
Catalog pipeline.ModuleCatalog
Registries pipeline.Registries
LLMClientFactory LLMClientFactory
RunIDGenerator RunIDGenerator
LookupEnv func(string) (string, bool)
Now func() time.Time
UserCacheDir func() (string, error)
ChunkPlanStoreFactory pipeline.ChunkPlanStoreFactory
DebugRecorderFactory func(string) (pipeline.DebugRecorder, error)
DebugTerminalFactory func(*debugbundle.SummaryWriter) DebugTerminalWriter
promptKitAssets *frameworkllm.AssetRegistry
LLMClient promptkit.LLMClient
LLMRuntime promptkit.RuntimeConfig
Catalog pipeline.ModuleCatalog
Registries pipeline.Registries
LLMClientFactory LLMClientFactory
RunIDGenerator RunIDGenerator
LookupEnv func(string) (string, bool)
Now func() time.Time
UserCacheDir func() (string, error)
DebugRecorderFactory func(string) (pipeline.DebugRecorder, error)
DebugTerminalFactory func(*debugbundle.SummaryWriter) DebugTerminalWriter
promptKitAssets *frameworkllm.AssetRegistry
}
type LLMRuntimeOverrides struct {
@@ -85,9 +83,6 @@ func normalizeOptions(opts Options) (Options, error) {
if opts.UserCacheDir == nil {
opts.UserCacheDir = os.UserCacheDir
}
if opts.ChunkPlanStoreFactory == nil {
opts.ChunkPlanStoreFactory = chunkplan.NewFilesystemStore
}
if opts.DebugRecorderFactory == nil {
opts.DebugRecorderFactory = frameworkdebug.NewFilesystemRecorder
}
@@ -130,19 +125,14 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
llmProfile := singleValueFlag{name: "--llm-profile"}
reasoningEffort := singleValueFlag{name: "--reasoning-effort"}
clearReasoningEffort := fs.Bool("clear-reasoning-effort", false, "clear the LLM profile reasoning effort")
resume := fs.Bool("resume", false, "reuse compatible recorded checkpoints")
recomputeStep := singleValueFlag{name: "--recompute-step"}
chunkCache := chunkCacheFlag{}
requestedSessionID := sessionIDFlag{}
referenceFlags := stringListFlag{}
withoutReferenceFlags := stringListFlag{}
fs.Var(&requestedSessionID, "session-id", "prompt session identifier")
fs.Var(&llmProfile, "llm-profile", "LLM profile override")
fs.Var(&reasoningEffort, "reasoning-effort", "reasoning effort override")
fs.Var(&chunkCache, "chunk_cache", "chunk plan cache mode: auto, bypass, or refresh")
fs.Var(&referenceFlags, "reference", "reference binding, as slot=path, chunk.slot=path, lane.slot=path, lane.extract.slot=path, lane.merge.slot=path, or lane.normalize.slot=path")
fs.Var(&withoutReferenceFlags, "without-reference", "unbind a reference, using the same selector forms as --reference")
fs.Var(&recomputeStep, "recompute-step", "recompute one ordered pipeline step and dependent lanes")
if err := validateRunFlagValues(args); err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 2
@@ -209,14 +199,6 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 2
}
if recomputeStep.set && strings.TrimSpace(recomputeStep.value) == "" {
fmt.Fprintln(stderr, "notarius: --recompute-step must not be empty")
return 2
}
if recomputeStep.set && len(only) > 0 {
fmt.Fprintln(stderr, "notarius: --recompute-step cannot be combined with --only")
return 2
}
referenceRequests, err := parseReferenceFlags(referenceFlags)
if err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
@@ -228,7 +210,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
return 2
}
loaded, loadedConfigPath, err := loadConfigKit(*configPath, *configOverlay, runConfigurationOverrides(chunkCache, *outputDir, *debugDir), opts)
loaded, loadedConfigPath, err := loadConfigKit(*configPath, *configOverlay, runConfigurationOverrides(*outputDir, *debugDir), opts)
if err != nil {
writeConfigLoadError(stderr, err)
return 1
@@ -238,19 +220,6 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 1
}
if *resume && !cfg.Cache.Checkpoints.Enabled {
fmt.Fprintln(stderr, "notarius: --resume requires cache.checkpoints.enabled: true")
return 1
}
if recomputeStep.set && !*resume {
fmt.Fprintln(stderr, "notarius: --recompute-step requires --resume")
return 2
}
if recomputeStep.set && !cfg.Cache.Checkpoints.Enabled {
fmt.Fprintln(stderr, "notarius: --recompute-step requires cache.checkpoints.enabled: true")
return 1
}
startedAt := opts.Now().UTC()
runID, err := opts.RunIDGenerator(startedAt)
if err != nil {
@@ -293,10 +262,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
ConfigSource: configSource(*configPath),
ConfigOverlay: effectiveConfigOverlay(*configOverlay),
OnlyLanes: append([]string(nil), only...),
ChunkCacheOverride: chunkCache.explicitValue(),
ReasoningEffortOverride: runtimeOverrides.ReasoningEffort,
Resume: *resume,
RecomputeStep: strings.TrimSpace(recomputeStep.value),
RunID: runID,
StartedAt: startedAt,
}
@@ -339,13 +305,6 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
return failPipelineCommand(stderr, commandState, terminalWriter, err)
}
effective.ResolvedPipeline = materialized
checkpointPolicy := pipeline.CheckpointExecutionPolicy{}
if recomputeStep.set {
checkpointPolicy, err = recomputePolicy(effective.ResolvedPipeline, recomputeStep.value)
if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, err)
}
}
invocation.PipelineDigest = effective.ResolvedPipeline.Digest
if err := writeSummary(summary, func() error { return summary.WriteInvocation(invocation) }); err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug invocation metadata: %w", err))
@@ -376,13 +335,6 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("create LLM client for profile %q: %w", factoryProfileID, err))
}
var llmFingerprints []checkpoint.Fingerprint
if effective.Config.Cache.Checkpoints.Enabled {
llmFingerprints, err = llmCheckpointFingerprints(llmClient)
if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("prepare LLM checkpoint identity: %w", err))
}
}
llmClient = pipeline.WithDebugLLMRecording(llmClient, debugRecorder)
prepared, err := pipeline.Prepare(effective.ResolvedPipeline, registries, pipeline.ModuleDependencies{LLM: llmClient})
if err != nil {
@@ -400,32 +352,24 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
if err := writeSummary(summary, func() error { return summary.WriteInvocation(invocation) }); err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug invocation metadata: %w", err))
}
chunkPlans, err := chunkPlanStoreForRun(effective.Config.Cache.ChunkPlans, opts)
if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, err)
}
checkpointRecorder, checkpointLoader, err := checkpointHandlersForRun(effective.Config.Cache.Checkpoints, opts, effective.ResolvedPipeline, prepared.CheckpointFingerprints(), llmFingerprints, rawInput, only, llmProfiles, strings.TrimSpace(llmProfile.value), effectiveSessionID, runtimeOverrides, *resume)
cache, err := artifactcache.NewFilesystemStore(effective.Config.CacheDirectory)
if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, err)
}
output, err := pipeline.New().Run(ctx, pipeline.RunInput{
Prepared: prepared,
Path: strings.TrimSpace(*inputPath),
RawInput: rawInput,
SessionID: effectiveSessionID,
RunID: runID,
StartedAt: startedAt,
LLMProfiles: llmProfiles,
Metadata: runMetadata(effective.Config.Output.Directory, debugPath),
Diagnostics: referenceDiagnostics,
ChunkCacheMode: effective.Config.Cache.ChunkPlans.Mode,
ChunkPlans: chunkPlans,
Checkpoints: checkpointRecorder,
Checkpoint: checkpointLoader,
CheckpointPolicy: checkpointPolicy,
Debug: debugRecorder,
Workers: cfg.Concurrency.Workers,
Prepared: prepared,
Path: strings.TrimSpace(*inputPath),
RawInput: rawInput,
SessionID: effectiveSessionID,
RunID: runID,
StartedAt: startedAt,
LLMProfiles: llmProfiles,
Metadata: runMetadata(effective.Config.Output.Directory, debugPath),
Diagnostics: referenceDiagnostics,
Cache: cache,
Debug: debugRecorder,
Workers: cfg.Concurrency.Workers,
})
diagnosticProjection, diagnosticErr := contracts.ProjectDiagnosticCollection(output.Diagnostics)
if diagnosticErr == nil {
@@ -511,196 +455,10 @@ func writePartialSummary(summary *debugbundle.SummaryWriter, output pipeline.Run
if err := summary.WriteRunManifest(output.Manifest); err != nil {
return err
}
if output.ChunkPlan != nil {
if err := summary.WriteChunkPlan(*output.ChunkPlan); err != nil {
return err
}
}
if err := summary.WriteDiagnostics(output.Diagnostics); err != nil {
return err
}
return summary.WriteCheckpointEvents(output.CheckpointEvents)
}
func checkpointHandlersForRun(
settings config.CheckpointCacheConfig,
opts Options,
resolved pipeline.ResolvedPipeline,
componentFingerprints []pipeline.CheckpointFingerprint,
llmFingerprints []checkpoint.Fingerprint,
rawInput []byte,
only []string,
llmProfiles []artifacts.LLMProfileManifest,
llmProfileOverride string,
sessionID string,
runtimeOverrides LLMRuntimeOverrides,
resume bool,
) (pipeline.CheckpointRecorder, pipeline.CheckpointLoader, error) {
if !settings.Enabled {
if resume {
return nil, nil, fmt.Errorf("--resume requires cache.checkpoints.enabled: true")
}
return pipeline.NoopCheckpointRecorder(), pipeline.NoopCheckpointLoader(), nil
}
identity, err := checkpoint.NewIdentity(checkpoint.IdentityInput{
Pipeline: resolved,
InputKey: resolved.Input.Module,
RawInputDigest: rawInputDigest(rawInput),
SelectedLanes: only,
RuntimeOverrides: runtimeOverrideFingerprints(llmProfileOverride, sessionID, runtimeOverrides),
References: pipeline.ReferenceProvenance(resolved),
ProvenanceFingerprints: combineCheckpointFingerprints(
llmProfileFingerprints(llmProfiles),
llmFingerprints,
checkpointIdentityFingerprints(componentFingerprints),
),
})
if err != nil {
return nil, nil, fmt.Errorf("create checkpoint identity: %w", err)
}
checkpointRoot := strings.TrimSpace(settings.Directory)
if checkpointRoot == "" {
checkpointRoot, err = config.DefaultCheckpointRoot(opts.UserCacheDir)
if err != nil {
return nil, nil, fmt.Errorf("resolve checkpoint root: %w", err)
}
}
recorder, err := checkpoint.NewFilesystemRecorder(checkpointRoot, identity)
if err != nil {
return nil, nil, fmt.Errorf("create checkpoint recorder: %w", err)
}
loader := pipeline.NoopCheckpointLoader()
if resume {
loader, err = checkpoint.NewFilesystemLoader(checkpointRoot, identity)
if err != nil {
return nil, nil, fmt.Errorf("create checkpoint loader: %w", err)
}
}
return recorder, loader, nil
}
func llmCheckpointFingerprints(client contracts.StructuredLLMClient) ([]checkpoint.Fingerprint, error) {
provider, ok := client.(frameworkllm.CheckpointFingerprintProvider)
if !ok {
return nil, nil
}
values, err := provider.LLMCheckpointFingerprints()
if err != nil {
return nil, err
}
out := make([]checkpoint.Fingerprint, 0, len(values))
for _, value := range values {
out = append(out, checkpoint.Fingerprint{Name: value.Name, Value: value.Value})
}
return out, nil
}
func combineCheckpointFingerprints(sources ...[]checkpoint.Fingerprint) []checkpoint.Fingerprint {
var out []checkpoint.Fingerprint
for _, source := range sources {
out = append(out, source...)
}
return out
}
func recomputePolicy(resolved pipeline.ResolvedPipeline, requestedStep string) (pipeline.CheckpointExecutionPolicy, error) {
requestedStep = strings.TrimSpace(requestedStep)
if requestedStep == "" {
return pipeline.CheckpointExecutionPolicy{}, fmt.Errorf("--recompute-step must not be empty")
}
var selected *pipeline.ResolvedPipelineStep
for index := range resolved.Steps {
if strings.TrimSpace(resolved.Steps[index].ID) == requestedStep {
selected = &resolved.Steps[index]
break
}
}
if selected == nil {
return pipeline.CheckpointExecutionPolicy{}, fmt.Errorf("unknown pipeline step %q", requestedStep)
}
policy := pipeline.CheckpointExecutionPolicy{ForcedLanes: make(map[string]struct{}), RequireReusableLanes: make(map[string]struct{})}
var forced []pipeline.ResolvedArtifactLane
for _, lane := range selected.ArtifactLanes {
lane.StepID = selected.ID
policy.ForcedLanes[pipeline.CheckpointLaneKey(selected.ID, lane.ID)] = struct{}{}
forced = append(forced, lane)
}
lanes := make(map[string]pipeline.ResolvedArtifactLane)
dependents := make(map[string][]pipeline.ResolvedArtifactLane)
for _, step := range resolved.Steps {
for _, lane := range step.ArtifactLanes {
lane.StepID = step.ID
lanes[pipeline.CheckpointLaneKey(step.ID, lane.ID)] = lane
for _, target := range []pipeline.ResolvedReferenceTarget{lane.ExtractReferences, lane.MergeReferences, lane.NormalizeReferences} {
for _, binding := range target.Bindings {
if binding.Artifact != nil {
producerKey := pipeline.CheckpointLaneKey(binding.Artifact.Step, binding.Artifact.Lane)
dependents[producerKey] = append(dependents[producerKey], lane)
}
}
}
}
}
processed := make(map[string]struct{})
queue := append([]pipeline.ResolvedArtifactLane(nil), forced...)
for len(queue) > 0 {
lane := queue[0]
queue = queue[1:]
key := pipeline.CheckpointLaneKey(lane.StepID, lane.ID)
if _, ok := processed[key]; ok {
continue
}
processed[key] = struct{}{}
for _, dependent := range dependents[key] {
dependentKey := pipeline.CheckpointLaneKey(dependent.StepID, dependent.ID)
if _, alreadyForced := policy.ForcedLanes[dependentKey]; alreadyForced {
continue
}
policy.ForcedLanes[dependentKey] = struct{}{}
forced = append(forced, dependent)
queue = append(queue, dependent)
}
}
processed = make(map[string]struct{})
queue = append([]pipeline.ResolvedArtifactLane(nil), forced...)
for len(queue) > 0 {
lane := queue[0]
queue = queue[1:]
key := pipeline.CheckpointLaneKey(lane.StepID, lane.ID)
if _, ok := processed[key]; ok {
continue
}
processed[key] = struct{}{}
for _, target := range []pipeline.ResolvedReferenceTarget{lane.ExtractReferences, lane.MergeReferences, lane.NormalizeReferences} {
for _, binding := range target.Bindings {
if binding.Artifact == nil {
continue
}
producerKey := pipeline.CheckpointLaneKey(binding.Artifact.Step, binding.Artifact.Lane)
if _, forcedAlready := policy.ForcedLanes[producerKey]; !forcedAlready {
policy.RequireReusableLanes[producerKey] = struct{}{}
}
if producer, ok := lanes[producerKey]; ok {
queue = append(queue, producer)
}
}
}
}
return policy, nil
}
func checkpointIdentityFingerprints(values []pipeline.CheckpointFingerprint) []checkpoint.Fingerprint {
if len(values) == 0 {
return nil
}
out := make([]checkpoint.Fingerprint, len(values))
for index, value := range values {
out[index] = checkpoint.Fingerprint{Name: "component:" + value.Name, Value: value.Value}
}
return out
return nil
}
func rawInputDigest(data []byte) string {
@@ -708,42 +466,6 @@ func rawInputDigest(data []byte) string {
return "sha256:" + hex.EncodeToString(sum[:])
}
func runtimeOverrideFingerprints(llmProfileOverride string, sessionID string, runtimeOverrides LLMRuntimeOverrides) []checkpoint.Fingerprint {
var values []checkpoint.Fingerprint
if strings.TrimSpace(llmProfileOverride) != "" {
values = append(values, checkpoint.Fingerprint{Name: "llm_profile_override", Value: strings.TrimSpace(llmProfileOverride)})
}
if strings.TrimSpace(sessionID) != "" {
values = append(values, checkpoint.Fingerprint{Name: "session_id", Value: strings.TrimSpace(sessionID)})
}
if runtimeOverrides.ReasoningEffort != nil {
value := strings.TrimSpace(*runtimeOverrides.ReasoningEffort)
if value == "" {
value = "<cleared>"
}
values = append(values, checkpoint.Fingerprint{Name: "reasoning_effort_override", Value: value})
}
return values
}
func llmProfileFingerprints(profiles []artifacts.LLMProfileManifest) []checkpoint.Fingerprint {
if len(profiles) == 0 {
return nil
}
values := make([]checkpoint.Fingerprint, 0, len(profiles))
for _, profile := range profiles {
id := strings.TrimSpace(profile.ID)
if id == "" {
continue
}
values = append(values, checkpoint.Fingerprint{
Name: "llm_profile:" + id,
Value: strings.TrimSpace(profile.Provider) + ":" + strings.TrimSpace(profile.Model),
})
}
return values
}
func configSource(configPath string) string {
if strings.TrimSpace(configPath) != "" {
return "flag"
@@ -837,22 +559,15 @@ func reorderRunArgs(args []string) []string {
func runFlagTakesValue(arg string) bool {
switch arg {
case "--config", "--config-overlay", "--input", "--only", "--output-dir", "--debug-dir", "--llm-profile", "--session-id", "--reasoning-effort", "--chunk_cache", "--reference", "--without-reference", "--recompute-step":
case "--config", "--config-overlay", "--input", "--only", "--output-dir", "--debug-dir", "--llm-profile", "--session-id", "--reasoning-effort", "--reference", "--without-reference":
return true
default:
return false
}
}
func runConfigurationOverrides(chunkCache chunkCacheFlag, outputDir, debugDir string) []configkit.Override {
func runConfigurationOverrides(outputDir, debugDir string) []configkit.Override {
overrides := make([]configkit.Override, 0, 3)
if chunkCache.set {
overrides = append(overrides, configkit.Override{
Path: configkit.Keys("cache", "chunk_plans", "mode"),
Value: string(chunkCache.value),
Flag: "--chunk_cache",
})
}
if outputDir = strings.TrimSpace(outputDir); outputDir != "" {
overrides = append(overrides, configkit.Override{
Path: configkit.Keys("output", "directory"),
@@ -870,57 +585,6 @@ func runConfigurationOverrides(chunkCache chunkCacheFlag, outputDir, debugDir st
return overrides
}
type chunkCacheFlag struct {
value pipeline.ChunkCacheMode
set bool
}
func (f *chunkCacheFlag) String() string {
if f == nil {
return ""
}
return string(f.value)
}
func (f *chunkCacheFlag) Set(raw string) error {
mode, err := pipeline.ParseChunkCacheMode(raw)
if err != nil {
return err
}
f.value = mode
f.set = true
return nil
}
func (f chunkCacheFlag) explicitValue() string {
if !f.set {
return ""
}
return string(f.value)
}
func chunkPlanStoreForRun(cfg config.ChunkPlanCacheConfig, opts Options) (pipeline.ChunkPlanStore, error) {
if cfg.Mode == pipeline.ChunkCacheBypass {
return nil, nil
}
root := strings.TrimSpace(cfg.Directory)
if root == "" {
var err error
root, err = config.DefaultChunkPlanRoot(opts.UserCacheDir)
if err != nil {
return nil, fmt.Errorf("resolve chunk plan root: %w", err)
}
}
store, err := opts.ChunkPlanStoreFactory(root)
if err != nil {
return nil, fmt.Errorf("create chunk plan store at %q: %w", root, err)
}
if store == nil {
return nil, fmt.Errorf("create chunk plan store at %q: factory returned nil", root)
}
return store, nil
}
func validateRunFlagValues(args []string) error {
for i, arg := range args {
if arg != "--config-overlay" && arg != "--session-id" && arg != "--reasoning-effort" {