Unify chunk-plan caching and remove checkpoint execution paths
This commit is contained in:
@@ -20,8 +20,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/fileio"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/checkpoint"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkplan"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/artifactcache"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
frameworkdebug "gitea.maximumdirect.net/eric/notarius/internal/framework/debug"
|
||||
frameworkllm "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
@@ -32,19 +31,18 @@ import (
|
||||
)
|
||||
|
||||
type Options struct {
|
||||
LLMClient promptkit.LLMClient
|
||||
LLMRuntime promptkit.RuntimeConfig
|
||||
Catalog pipeline.ModuleCatalog
|
||||
Registries pipeline.Registries
|
||||
LLMClientFactory LLMClientFactory
|
||||
RunIDGenerator RunIDGenerator
|
||||
LookupEnv func(string) (string, bool)
|
||||
Now func() time.Time
|
||||
UserCacheDir func() (string, error)
|
||||
ChunkPlanStoreFactory pipeline.ChunkPlanStoreFactory
|
||||
DebugRecorderFactory func(string) (pipeline.DebugRecorder, error)
|
||||
DebugTerminalFactory func(*debugbundle.SummaryWriter) DebugTerminalWriter
|
||||
promptKitAssets *frameworkllm.AssetRegistry
|
||||
LLMClient promptkit.LLMClient
|
||||
LLMRuntime promptkit.RuntimeConfig
|
||||
Catalog pipeline.ModuleCatalog
|
||||
Registries pipeline.Registries
|
||||
LLMClientFactory LLMClientFactory
|
||||
RunIDGenerator RunIDGenerator
|
||||
LookupEnv func(string) (string, bool)
|
||||
Now func() time.Time
|
||||
UserCacheDir func() (string, error)
|
||||
DebugRecorderFactory func(string) (pipeline.DebugRecorder, error)
|
||||
DebugTerminalFactory func(*debugbundle.SummaryWriter) DebugTerminalWriter
|
||||
promptKitAssets *frameworkllm.AssetRegistry
|
||||
}
|
||||
|
||||
type LLMRuntimeOverrides struct {
|
||||
@@ -85,9 +83,6 @@ func normalizeOptions(opts Options) (Options, error) {
|
||||
if opts.UserCacheDir == nil {
|
||||
opts.UserCacheDir = os.UserCacheDir
|
||||
}
|
||||
if opts.ChunkPlanStoreFactory == nil {
|
||||
opts.ChunkPlanStoreFactory = chunkplan.NewFilesystemStore
|
||||
}
|
||||
if opts.DebugRecorderFactory == nil {
|
||||
opts.DebugRecorderFactory = frameworkdebug.NewFilesystemRecorder
|
||||
}
|
||||
@@ -130,19 +125,14 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
llmProfile := singleValueFlag{name: "--llm-profile"}
|
||||
reasoningEffort := singleValueFlag{name: "--reasoning-effort"}
|
||||
clearReasoningEffort := fs.Bool("clear-reasoning-effort", false, "clear the LLM profile reasoning effort")
|
||||
resume := fs.Bool("resume", false, "reuse compatible recorded checkpoints")
|
||||
recomputeStep := singleValueFlag{name: "--recompute-step"}
|
||||
chunkCache := chunkCacheFlag{}
|
||||
requestedSessionID := sessionIDFlag{}
|
||||
referenceFlags := stringListFlag{}
|
||||
withoutReferenceFlags := stringListFlag{}
|
||||
fs.Var(&requestedSessionID, "session-id", "prompt session identifier")
|
||||
fs.Var(&llmProfile, "llm-profile", "LLM profile override")
|
||||
fs.Var(&reasoningEffort, "reasoning-effort", "reasoning effort override")
|
||||
fs.Var(&chunkCache, "chunk_cache", "chunk plan cache mode: auto, bypass, or refresh")
|
||||
fs.Var(&referenceFlags, "reference", "reference binding, as slot=path, chunk.slot=path, lane.slot=path, lane.extract.slot=path, lane.merge.slot=path, or lane.normalize.slot=path")
|
||||
fs.Var(&withoutReferenceFlags, "without-reference", "unbind a reference, using the same selector forms as --reference")
|
||||
fs.Var(&recomputeStep, "recompute-step", "recompute one ordered pipeline step and dependent lanes")
|
||||
if err := validateRunFlagValues(args); err != nil {
|
||||
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||
return 2
|
||||
@@ -209,14 +199,6 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||
return 2
|
||||
}
|
||||
if recomputeStep.set && strings.TrimSpace(recomputeStep.value) == "" {
|
||||
fmt.Fprintln(stderr, "notarius: --recompute-step must not be empty")
|
||||
return 2
|
||||
}
|
||||
if recomputeStep.set && len(only) > 0 {
|
||||
fmt.Fprintln(stderr, "notarius: --recompute-step cannot be combined with --only")
|
||||
return 2
|
||||
}
|
||||
referenceRequests, err := parseReferenceFlags(referenceFlags)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||
@@ -228,7 +210,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
return 2
|
||||
}
|
||||
|
||||
loaded, loadedConfigPath, err := loadConfigKit(*configPath, *configOverlay, runConfigurationOverrides(chunkCache, *outputDir, *debugDir), opts)
|
||||
loaded, loadedConfigPath, err := loadConfigKit(*configPath, *configOverlay, runConfigurationOverrides(*outputDir, *debugDir), opts)
|
||||
if err != nil {
|
||||
writeConfigLoadError(stderr, err)
|
||||
return 1
|
||||
@@ -238,19 +220,6 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
if *resume && !cfg.Cache.Checkpoints.Enabled {
|
||||
fmt.Fprintln(stderr, "notarius: --resume requires cache.checkpoints.enabled: true")
|
||||
return 1
|
||||
}
|
||||
if recomputeStep.set && !*resume {
|
||||
fmt.Fprintln(stderr, "notarius: --recompute-step requires --resume")
|
||||
return 2
|
||||
}
|
||||
if recomputeStep.set && !cfg.Cache.Checkpoints.Enabled {
|
||||
fmt.Fprintln(stderr, "notarius: --recompute-step requires cache.checkpoints.enabled: true")
|
||||
return 1
|
||||
}
|
||||
|
||||
startedAt := opts.Now().UTC()
|
||||
runID, err := opts.RunIDGenerator(startedAt)
|
||||
if err != nil {
|
||||
@@ -293,10 +262,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
ConfigSource: configSource(*configPath),
|
||||
ConfigOverlay: effectiveConfigOverlay(*configOverlay),
|
||||
OnlyLanes: append([]string(nil), only...),
|
||||
ChunkCacheOverride: chunkCache.explicitValue(),
|
||||
ReasoningEffortOverride: runtimeOverrides.ReasoningEffort,
|
||||
Resume: *resume,
|
||||
RecomputeStep: strings.TrimSpace(recomputeStep.value),
|
||||
RunID: runID,
|
||||
StartedAt: startedAt,
|
||||
}
|
||||
@@ -339,13 +305,6 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, err)
|
||||
}
|
||||
effective.ResolvedPipeline = materialized
|
||||
checkpointPolicy := pipeline.CheckpointExecutionPolicy{}
|
||||
if recomputeStep.set {
|
||||
checkpointPolicy, err = recomputePolicy(effective.ResolvedPipeline, recomputeStep.value)
|
||||
if err != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, err)
|
||||
}
|
||||
}
|
||||
invocation.PipelineDigest = effective.ResolvedPipeline.Digest
|
||||
if err := writeSummary(summary, func() error { return summary.WriteInvocation(invocation) }); err != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug invocation metadata: %w", err))
|
||||
@@ -376,13 +335,6 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
if err != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("create LLM client for profile %q: %w", factoryProfileID, err))
|
||||
}
|
||||
var llmFingerprints []checkpoint.Fingerprint
|
||||
if effective.Config.Cache.Checkpoints.Enabled {
|
||||
llmFingerprints, err = llmCheckpointFingerprints(llmClient)
|
||||
if err != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("prepare LLM checkpoint identity: %w", err))
|
||||
}
|
||||
}
|
||||
llmClient = pipeline.WithDebugLLMRecording(llmClient, debugRecorder)
|
||||
prepared, err := pipeline.Prepare(effective.ResolvedPipeline, registries, pipeline.ModuleDependencies{LLM: llmClient})
|
||||
if err != nil {
|
||||
@@ -400,32 +352,24 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
if err := writeSummary(summary, func() error { return summary.WriteInvocation(invocation) }); err != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug invocation metadata: %w", err))
|
||||
}
|
||||
chunkPlans, err := chunkPlanStoreForRun(effective.Config.Cache.ChunkPlans, opts)
|
||||
if err != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, err)
|
||||
}
|
||||
checkpointRecorder, checkpointLoader, err := checkpointHandlersForRun(effective.Config.Cache.Checkpoints, opts, effective.ResolvedPipeline, prepared.CheckpointFingerprints(), llmFingerprints, rawInput, only, llmProfiles, strings.TrimSpace(llmProfile.value), effectiveSessionID, runtimeOverrides, *resume)
|
||||
cache, err := artifactcache.NewFilesystemStore(effective.Config.CacheDirectory)
|
||||
if err != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, err)
|
||||
}
|
||||
|
||||
output, err := pipeline.New().Run(ctx, pipeline.RunInput{
|
||||
Prepared: prepared,
|
||||
Path: strings.TrimSpace(*inputPath),
|
||||
RawInput: rawInput,
|
||||
SessionID: effectiveSessionID,
|
||||
RunID: runID,
|
||||
StartedAt: startedAt,
|
||||
LLMProfiles: llmProfiles,
|
||||
Metadata: runMetadata(effective.Config.Output.Directory, debugPath),
|
||||
Diagnostics: referenceDiagnostics,
|
||||
ChunkCacheMode: effective.Config.Cache.ChunkPlans.Mode,
|
||||
ChunkPlans: chunkPlans,
|
||||
Checkpoints: checkpointRecorder,
|
||||
Checkpoint: checkpointLoader,
|
||||
CheckpointPolicy: checkpointPolicy,
|
||||
Debug: debugRecorder,
|
||||
Workers: cfg.Concurrency.Workers,
|
||||
Prepared: prepared,
|
||||
Path: strings.TrimSpace(*inputPath),
|
||||
RawInput: rawInput,
|
||||
SessionID: effectiveSessionID,
|
||||
RunID: runID,
|
||||
StartedAt: startedAt,
|
||||
LLMProfiles: llmProfiles,
|
||||
Metadata: runMetadata(effective.Config.Output.Directory, debugPath),
|
||||
Diagnostics: referenceDiagnostics,
|
||||
Cache: cache,
|
||||
Debug: debugRecorder,
|
||||
Workers: cfg.Concurrency.Workers,
|
||||
})
|
||||
diagnosticProjection, diagnosticErr := contracts.ProjectDiagnosticCollection(output.Diagnostics)
|
||||
if diagnosticErr == nil {
|
||||
@@ -511,196 +455,10 @@ func writePartialSummary(summary *debugbundle.SummaryWriter, output pipeline.Run
|
||||
if err := summary.WriteRunManifest(output.Manifest); err != nil {
|
||||
return err
|
||||
}
|
||||
if output.ChunkPlan != nil {
|
||||
if err := summary.WriteChunkPlan(*output.ChunkPlan); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := summary.WriteDiagnostics(output.Diagnostics); err != nil {
|
||||
return err
|
||||
}
|
||||
return summary.WriteCheckpointEvents(output.CheckpointEvents)
|
||||
}
|
||||
|
||||
func checkpointHandlersForRun(
|
||||
settings config.CheckpointCacheConfig,
|
||||
opts Options,
|
||||
resolved pipeline.ResolvedPipeline,
|
||||
componentFingerprints []pipeline.CheckpointFingerprint,
|
||||
llmFingerprints []checkpoint.Fingerprint,
|
||||
rawInput []byte,
|
||||
only []string,
|
||||
llmProfiles []artifacts.LLMProfileManifest,
|
||||
llmProfileOverride string,
|
||||
sessionID string,
|
||||
runtimeOverrides LLMRuntimeOverrides,
|
||||
resume bool,
|
||||
) (pipeline.CheckpointRecorder, pipeline.CheckpointLoader, error) {
|
||||
if !settings.Enabled {
|
||||
if resume {
|
||||
return nil, nil, fmt.Errorf("--resume requires cache.checkpoints.enabled: true")
|
||||
}
|
||||
return pipeline.NoopCheckpointRecorder(), pipeline.NoopCheckpointLoader(), nil
|
||||
}
|
||||
identity, err := checkpoint.NewIdentity(checkpoint.IdentityInput{
|
||||
Pipeline: resolved,
|
||||
InputKey: resolved.Input.Module,
|
||||
RawInputDigest: rawInputDigest(rawInput),
|
||||
SelectedLanes: only,
|
||||
RuntimeOverrides: runtimeOverrideFingerprints(llmProfileOverride, sessionID, runtimeOverrides),
|
||||
References: pipeline.ReferenceProvenance(resolved),
|
||||
ProvenanceFingerprints: combineCheckpointFingerprints(
|
||||
llmProfileFingerprints(llmProfiles),
|
||||
llmFingerprints,
|
||||
checkpointIdentityFingerprints(componentFingerprints),
|
||||
),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("create checkpoint identity: %w", err)
|
||||
}
|
||||
checkpointRoot := strings.TrimSpace(settings.Directory)
|
||||
if checkpointRoot == "" {
|
||||
checkpointRoot, err = config.DefaultCheckpointRoot(opts.UserCacheDir)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("resolve checkpoint root: %w", err)
|
||||
}
|
||||
}
|
||||
recorder, err := checkpoint.NewFilesystemRecorder(checkpointRoot, identity)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("create checkpoint recorder: %w", err)
|
||||
}
|
||||
loader := pipeline.NoopCheckpointLoader()
|
||||
if resume {
|
||||
loader, err = checkpoint.NewFilesystemLoader(checkpointRoot, identity)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("create checkpoint loader: %w", err)
|
||||
}
|
||||
}
|
||||
return recorder, loader, nil
|
||||
}
|
||||
|
||||
func llmCheckpointFingerprints(client contracts.StructuredLLMClient) ([]checkpoint.Fingerprint, error) {
|
||||
provider, ok := client.(frameworkllm.CheckpointFingerprintProvider)
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
values, err := provider.LLMCheckpointFingerprints()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]checkpoint.Fingerprint, 0, len(values))
|
||||
for _, value := range values {
|
||||
out = append(out, checkpoint.Fingerprint{Name: value.Name, Value: value.Value})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func combineCheckpointFingerprints(sources ...[]checkpoint.Fingerprint) []checkpoint.Fingerprint {
|
||||
var out []checkpoint.Fingerprint
|
||||
for _, source := range sources {
|
||||
out = append(out, source...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func recomputePolicy(resolved pipeline.ResolvedPipeline, requestedStep string) (pipeline.CheckpointExecutionPolicy, error) {
|
||||
requestedStep = strings.TrimSpace(requestedStep)
|
||||
if requestedStep == "" {
|
||||
return pipeline.CheckpointExecutionPolicy{}, fmt.Errorf("--recompute-step must not be empty")
|
||||
}
|
||||
var selected *pipeline.ResolvedPipelineStep
|
||||
for index := range resolved.Steps {
|
||||
if strings.TrimSpace(resolved.Steps[index].ID) == requestedStep {
|
||||
selected = &resolved.Steps[index]
|
||||
break
|
||||
}
|
||||
}
|
||||
if selected == nil {
|
||||
return pipeline.CheckpointExecutionPolicy{}, fmt.Errorf("unknown pipeline step %q", requestedStep)
|
||||
}
|
||||
|
||||
policy := pipeline.CheckpointExecutionPolicy{ForcedLanes: make(map[string]struct{}), RequireReusableLanes: make(map[string]struct{})}
|
||||
var forced []pipeline.ResolvedArtifactLane
|
||||
for _, lane := range selected.ArtifactLanes {
|
||||
lane.StepID = selected.ID
|
||||
policy.ForcedLanes[pipeline.CheckpointLaneKey(selected.ID, lane.ID)] = struct{}{}
|
||||
forced = append(forced, lane)
|
||||
}
|
||||
|
||||
lanes := make(map[string]pipeline.ResolvedArtifactLane)
|
||||
dependents := make(map[string][]pipeline.ResolvedArtifactLane)
|
||||
for _, step := range resolved.Steps {
|
||||
for _, lane := range step.ArtifactLanes {
|
||||
lane.StepID = step.ID
|
||||
lanes[pipeline.CheckpointLaneKey(step.ID, lane.ID)] = lane
|
||||
for _, target := range []pipeline.ResolvedReferenceTarget{lane.ExtractReferences, lane.MergeReferences, lane.NormalizeReferences} {
|
||||
for _, binding := range target.Bindings {
|
||||
if binding.Artifact != nil {
|
||||
producerKey := pipeline.CheckpointLaneKey(binding.Artifact.Step, binding.Artifact.Lane)
|
||||
dependents[producerKey] = append(dependents[producerKey], lane)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
processed := make(map[string]struct{})
|
||||
queue := append([]pipeline.ResolvedArtifactLane(nil), forced...)
|
||||
for len(queue) > 0 {
|
||||
lane := queue[0]
|
||||
queue = queue[1:]
|
||||
key := pipeline.CheckpointLaneKey(lane.StepID, lane.ID)
|
||||
if _, ok := processed[key]; ok {
|
||||
continue
|
||||
}
|
||||
processed[key] = struct{}{}
|
||||
for _, dependent := range dependents[key] {
|
||||
dependentKey := pipeline.CheckpointLaneKey(dependent.StepID, dependent.ID)
|
||||
if _, alreadyForced := policy.ForcedLanes[dependentKey]; alreadyForced {
|
||||
continue
|
||||
}
|
||||
policy.ForcedLanes[dependentKey] = struct{}{}
|
||||
forced = append(forced, dependent)
|
||||
queue = append(queue, dependent)
|
||||
}
|
||||
}
|
||||
|
||||
processed = make(map[string]struct{})
|
||||
queue = append([]pipeline.ResolvedArtifactLane(nil), forced...)
|
||||
for len(queue) > 0 {
|
||||
lane := queue[0]
|
||||
queue = queue[1:]
|
||||
key := pipeline.CheckpointLaneKey(lane.StepID, lane.ID)
|
||||
if _, ok := processed[key]; ok {
|
||||
continue
|
||||
}
|
||||
processed[key] = struct{}{}
|
||||
for _, target := range []pipeline.ResolvedReferenceTarget{lane.ExtractReferences, lane.MergeReferences, lane.NormalizeReferences} {
|
||||
for _, binding := range target.Bindings {
|
||||
if binding.Artifact == nil {
|
||||
continue
|
||||
}
|
||||
producerKey := pipeline.CheckpointLaneKey(binding.Artifact.Step, binding.Artifact.Lane)
|
||||
if _, forcedAlready := policy.ForcedLanes[producerKey]; !forcedAlready {
|
||||
policy.RequireReusableLanes[producerKey] = struct{}{}
|
||||
}
|
||||
if producer, ok := lanes[producerKey]; ok {
|
||||
queue = append(queue, producer)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return policy, nil
|
||||
}
|
||||
|
||||
func checkpointIdentityFingerprints(values []pipeline.CheckpointFingerprint) []checkpoint.Fingerprint {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]checkpoint.Fingerprint, len(values))
|
||||
for index, value := range values {
|
||||
out[index] = checkpoint.Fingerprint{Name: "component:" + value.Name, Value: value.Value}
|
||||
}
|
||||
return out
|
||||
return nil
|
||||
}
|
||||
|
||||
func rawInputDigest(data []byte) string {
|
||||
@@ -708,42 +466,6 @@ func rawInputDigest(data []byte) string {
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func runtimeOverrideFingerprints(llmProfileOverride string, sessionID string, runtimeOverrides LLMRuntimeOverrides) []checkpoint.Fingerprint {
|
||||
var values []checkpoint.Fingerprint
|
||||
if strings.TrimSpace(llmProfileOverride) != "" {
|
||||
values = append(values, checkpoint.Fingerprint{Name: "llm_profile_override", Value: strings.TrimSpace(llmProfileOverride)})
|
||||
}
|
||||
if strings.TrimSpace(sessionID) != "" {
|
||||
values = append(values, checkpoint.Fingerprint{Name: "session_id", Value: strings.TrimSpace(sessionID)})
|
||||
}
|
||||
if runtimeOverrides.ReasoningEffort != nil {
|
||||
value := strings.TrimSpace(*runtimeOverrides.ReasoningEffort)
|
||||
if value == "" {
|
||||
value = "<cleared>"
|
||||
}
|
||||
values = append(values, checkpoint.Fingerprint{Name: "reasoning_effort_override", Value: value})
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
func llmProfileFingerprints(profiles []artifacts.LLMProfileManifest) []checkpoint.Fingerprint {
|
||||
if len(profiles) == 0 {
|
||||
return nil
|
||||
}
|
||||
values := make([]checkpoint.Fingerprint, 0, len(profiles))
|
||||
for _, profile := range profiles {
|
||||
id := strings.TrimSpace(profile.ID)
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
values = append(values, checkpoint.Fingerprint{
|
||||
Name: "llm_profile:" + id,
|
||||
Value: strings.TrimSpace(profile.Provider) + ":" + strings.TrimSpace(profile.Model),
|
||||
})
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
func configSource(configPath string) string {
|
||||
if strings.TrimSpace(configPath) != "" {
|
||||
return "flag"
|
||||
@@ -837,22 +559,15 @@ func reorderRunArgs(args []string) []string {
|
||||
|
||||
func runFlagTakesValue(arg string) bool {
|
||||
switch arg {
|
||||
case "--config", "--config-overlay", "--input", "--only", "--output-dir", "--debug-dir", "--llm-profile", "--session-id", "--reasoning-effort", "--chunk_cache", "--reference", "--without-reference", "--recompute-step":
|
||||
case "--config", "--config-overlay", "--input", "--only", "--output-dir", "--debug-dir", "--llm-profile", "--session-id", "--reasoning-effort", "--reference", "--without-reference":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func runConfigurationOverrides(chunkCache chunkCacheFlag, outputDir, debugDir string) []configkit.Override {
|
||||
func runConfigurationOverrides(outputDir, debugDir string) []configkit.Override {
|
||||
overrides := make([]configkit.Override, 0, 3)
|
||||
if chunkCache.set {
|
||||
overrides = append(overrides, configkit.Override{
|
||||
Path: configkit.Keys("cache", "chunk_plans", "mode"),
|
||||
Value: string(chunkCache.value),
|
||||
Flag: "--chunk_cache",
|
||||
})
|
||||
}
|
||||
if outputDir = strings.TrimSpace(outputDir); outputDir != "" {
|
||||
overrides = append(overrides, configkit.Override{
|
||||
Path: configkit.Keys("output", "directory"),
|
||||
@@ -870,57 +585,6 @@ func runConfigurationOverrides(chunkCache chunkCacheFlag, outputDir, debugDir st
|
||||
return overrides
|
||||
}
|
||||
|
||||
type chunkCacheFlag struct {
|
||||
value pipeline.ChunkCacheMode
|
||||
set bool
|
||||
}
|
||||
|
||||
func (f *chunkCacheFlag) String() string {
|
||||
if f == nil {
|
||||
return ""
|
||||
}
|
||||
return string(f.value)
|
||||
}
|
||||
|
||||
func (f *chunkCacheFlag) Set(raw string) error {
|
||||
mode, err := pipeline.ParseChunkCacheMode(raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f.value = mode
|
||||
f.set = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f chunkCacheFlag) explicitValue() string {
|
||||
if !f.set {
|
||||
return ""
|
||||
}
|
||||
return string(f.value)
|
||||
}
|
||||
|
||||
func chunkPlanStoreForRun(cfg config.ChunkPlanCacheConfig, opts Options) (pipeline.ChunkPlanStore, error) {
|
||||
if cfg.Mode == pipeline.ChunkCacheBypass {
|
||||
return nil, nil
|
||||
}
|
||||
root := strings.TrimSpace(cfg.Directory)
|
||||
if root == "" {
|
||||
var err error
|
||||
root, err = config.DefaultChunkPlanRoot(opts.UserCacheDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve chunk plan root: %w", err)
|
||||
}
|
||||
}
|
||||
store, err := opts.ChunkPlanStoreFactory(root)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create chunk plan store at %q: %w", root, err)
|
||||
}
|
||||
if store == nil {
|
||||
return nil, fmt.Errorf("create chunk plan store at %q: factory returned nil", root)
|
||||
}
|
||||
return store, nil
|
||||
}
|
||||
|
||||
func validateRunFlagValues(args []string) error {
|
||||
for i, arg := range args {
|
||||
if arg != "--config-overlay" && arg != "--session-id" && arg != "--reasoning-effort" {
|
||||
|
||||
Reference in New Issue
Block a user