package cli import ( "context" "crypto/sha256" "encoding/hex" "errors" "flag" "fmt" "io" "os" "path" "path/filepath" "sort" "strings" "time" "gitea.maximumdirect.net/eric/notarius/internal/buildinfo" "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" "gitea.maximumdirect.net/eric/notarius/internal/core/config" "gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle" "gitea.maximumdirect.net/eric/notarius/internal/core/fileio" "gitea.maximumdirect.net/eric/notarius/internal/framework/artifactcache" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" frameworkdebug "gitea.maximumdirect.net/eric/notarius/internal/framework/debug" frameworkllm "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "go.maximumdirect.net/commandkit" configkit "go.maximumdirect.net/configkit" "go.maximumdirect.net/promptkit" ) type Options struct { LLMClient promptkit.LLMClient LLMRuntime promptkit.RuntimeConfig Catalog pipeline.ModuleCatalog Registries pipeline.Registries LLMClientFactory LLMClientFactory RunIDGenerator RunIDGenerator LookupEnv func(string) (string, bool) Now func() time.Time UserCacheDir func() (string, error) DebugRecorderFactory func(string) (pipeline.DebugRecorder, error) DebugTerminalFactory func(*debugbundle.SummaryWriter) DebugTerminalWriter promptKitAssets *frameworkllm.AssetRegistry } type LLMRuntimeOverrides struct { ReasoningEffort *string } type LLMClientFactory func(ctx context.Context, cfg config.Config, profileID string, overrides LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) // Run executes the command-line interface and returns a process exit code. func Run(args []string, stdout, stderr io.Writer) int { return RunWithOptions(args, stdout, stderr, Options{}) } func RunWithOptions(args []string, stdout, stderr io.Writer, opts Options) int { version, err := buildinfo.Version() if err != nil { fmt.Fprintf(stderr, "notarius: %v\n", err) return 1 } app, err := NewApplication(version, opts) if err != nil { fmt.Fprintf(stderr, "notarius: %v\n", err) return 1 } return app.Execute(context.Background(), args, commandkit.Streams{Out: stdout, Err: stderr}).Status } func normalizeOptions(opts Options) (Options, error) { if opts.LookupEnv == nil { opts.LookupEnv = os.LookupEnv } if opts.Now == nil { opts.Now = time.Now } if opts.RunIDGenerator == nil { opts.RunIDGenerator = defaultRunIDGenerator } if opts.UserCacheDir == nil { opts.UserCacheDir = os.UserCacheDir } if opts.DebugRecorderFactory == nil { opts.DebugRecorderFactory = frameworkdebug.NewFilesystemRecorder } if opts.DebugTerminalFactory == nil { opts.DebugTerminalFactory = func(writer *debugbundle.SummaryWriter) DebugTerminalWriter { return writer } } if isEmptyCatalog(opts.Catalog) && isEmptyRegistries(opts.Registries) { components, err := newProductionComponents() if err != nil { return Options{}, err } opts.Registries = components.registries opts.Catalog = catalogFromRegistries(components.registries) opts.promptKitAssets = components.assets } if opts.LLMClientFactory == nil { if opts.promptKitAssets == nil { assets, err := productionPromptAssets() if err != nil { return Options{}, err } opts.promptKitAssets = assets } opts.LLMClientFactory = productionLLMClientFactoryWithAssets(opts.promptKitAssets) } return opts, nil } func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) int { fs := flag.NewFlagSet("run", flag.ContinueOnError) fs.SetOutput(io.Discard) configPath := fs.String("config", "", "config file path") configOverlay := fs.String("config-overlay", "", "configuration overlay") inputPath := fs.String("input", "", "source input file path") onlyRaw := fs.String("only", "", "comma-separated artifact lanes") outputDir := fs.String("output-dir", "", "output directory") machineOutput := fs.Bool("json", false, "write the successful run result as JSON") debug := fs.Bool("debug", false, "write a debug bundle") debugDir := fs.String("debug-dir", "", "debug bundle directory") llmProfile := singleValueFlag{name: "--llm-profile"} reasoningEffort := singleValueFlag{name: "--reasoning-effort"} clearReasoningEffort := fs.Bool("clear-reasoning-effort", false, "clear the LLM profile reasoning effort") requestedSessionID := sessionIDFlag{} referenceFlags := stringListFlag{} withoutReferenceFlags := stringListFlag{} fs.Var(&requestedSessionID, "session-id", "prompt session identifier") fs.Var(&llmProfile, "llm-profile", "LLM profile override") fs.Var(&reasoningEffort, "reasoning-effort", "reasoning effort override") fs.Var(&referenceFlags, "reference", "reference binding, as slot=path, chunk.slot=path, lane.slot=path, lane.extract.slot=path, lane.merge.slot=path, or lane.normalize.slot=path") fs.Var(&withoutReferenceFlags, "without-reference", "unbind a reference, using the same selector forms as --reference") if err := validateRunFlagValues(args); err != nil { fmt.Fprintf(stderr, "notarius: %v\n", err) return 2 } if err := fs.Parse(reorderRunArgs(args)); err != nil { fmt.Fprintf(stderr, "notarius: %v\n", err) return 2 } if fs.NArg() == 0 { fmt.Fprintln(stderr, "notarius: run requires a pipeline ID") return 2 } if fs.NArg() > 1 { fmt.Fprintf(stderr, "notarius: unexpected argument %q\n", fs.Arg(1)) return 2 } pipelineID := strings.TrimSpace(fs.Arg(0)) if pipelineID == "" { fmt.Fprintln(stderr, "notarius: run requires a pipeline ID") return 2 } if strings.TrimSpace(*inputPath) == "" { fmt.Fprintln(stderr, "notarius: run requires --input") return 2 } if strings.TrimSpace(*debugDir) != "" && !*debug { fmt.Fprintln(stderr, "notarius: --debug-dir requires --debug") return 2 } if strings.TrimSpace(*outputDir) == "" && flagWasProvided(args, "--output-dir") { fmt.Fprintln(stderr, "notarius: --output-dir must not be empty") return 2 } if strings.TrimSpace(*debugDir) == "" && flagWasProvided(args, "--debug-dir") { fmt.Fprintln(stderr, "notarius: --debug-dir must not be empty") return 2 } if requestedSessionID.set && strings.TrimSpace(requestedSessionID.value) == "" { fmt.Fprintln(stderr, "notarius: --session-id must not be empty") return 2 } if llmProfile.set && strings.TrimSpace(llmProfile.value) == "" { fmt.Fprintln(stderr, "notarius: --llm-profile must not be empty") return 2 } if reasoningEffort.set && *clearReasoningEffort { fmt.Fprintln(stderr, "notarius: --reasoning-effort cannot be combined with --clear-reasoning-effort") return 2 } if reasoningEffort.set && strings.TrimSpace(reasoningEffort.value) == "" { fmt.Fprintln(stderr, "notarius: --reasoning-effort must not be empty") return 2 } runtimeOverrides := LLMRuntimeOverrides{} if reasoningEffort.set { value := strings.TrimSpace(reasoningEffort.value) runtimeOverrides.ReasoningEffort = &value } else if *clearReasoningEffort { value := "" runtimeOverrides.ReasoningEffort = &value } only, err := parseOnly(*onlyRaw) if err != nil { fmt.Fprintf(stderr, "notarius: %v\n", err) return 2 } referenceRequests, err := parseReferenceFlags(referenceFlags) if err != nil { fmt.Fprintf(stderr, "notarius: %v\n", err) return 2 } referenceUnbindRequests, err := parseReferenceUnbindFlags(withoutReferenceFlags) if err != nil { fmt.Fprintf(stderr, "notarius: %v\n", err) return 2 } loaded, loadedConfigPath, err := loadConfigKit(*configPath, *configOverlay, runConfigurationOverrides(*outputDir, *debugDir), opts) if err != nil { writeConfigLoadError(stderr, err) return 1 } cfg := loaded.RuntimeConfig() if err := cfg.Validate(); err != nil { fmt.Fprintf(stderr, "notarius: %v\n", err) return 1 } startedAt := opts.Now().UTC() runID, err := opts.RunIDGenerator(startedAt) if err != nil { fmt.Fprintf(stderr, "notarius: generate run ID: %v\n", err) return 1 } if err := validateRunID(runID); err != nil { fmt.Fprintf(stderr, "notarius: invalid generated run ID: %v\n", err) return 1 } runOutputDir := filepath.Join(cfg.Output.Directory, runID) commandState := newPipelineCommandState(runID, pipelineID, runOutputDir) var summary *debugbundle.SummaryWriter var terminalWriter DebugTerminalWriter debugPath := "" debugRecorder := pipeline.NoopDebugRecorder() if *debug { bundle, err := debugbundle.Allocate(cfg.Debug.Directory, runID, startedAt) if err != nil { fmt.Fprintf(stderr, "notarius: %v\n", err) return 1 } debugPath, summary = bundle.Path(), bundle.Summary() commandState.setDebugPath(debugPath) terminalWriter = opts.DebugTerminalFactory(summary) if terminalWriter == nil { terminalWriter = summary } debugRecorder, err = opts.DebugRecorderFactory(bundle.TraceRoot()) if err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("create debug recorder: %w", err)) } debugRecorder = pipeline.SynchronizedDebugRecorder(debugRecorder) } invocation := debugbundle.Invocation{ Operation: "run", PipelineID: pipelineID, InputPath: strings.TrimSpace(*inputPath), ConfigPath: loadedConfigPath, ConfigSource: configSource(*configPath), ConfigOverlay: effectiveConfigOverlay(*configOverlay), OnlyLanes: append([]string(nil), only...), ReasoningEffortOverride: runtimeOverrides.ReasoningEffort, RunID: runID, StartedAt: startedAt, } if err := writeSummary(summary, func() error { return summary.WriteInvocation(invocation) }); err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug invocation metadata: %w", err)) } catalog, err := effectiveCatalog(opts) if err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, err) } referenceOverrides, referenceUnbinds, err := resolveCLIReferenceRequests(cfg, pipelineID, only, catalog, referenceRequests, referenceUnbindRequests) if err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, err) } effective, err := cfg.Resolve(config.ResolveInput{ PipelineID: pipelineID, Only: only, Catalog: catalog, LLMProfileOverride: strings.TrimSpace(llmProfile.value), ReferenceOverrides: referenceOverrides, ReferenceUnbinds: referenceUnbinds, }) if err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, err) } profileIDs := effectiveLLMProfileIDs(effective.ResolvedPipeline) if err := validateExplicitPromptKitProfiles(context.Background(), effective.Config, profileIDs, opts.promptKitAssets); err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, err) } workingDir, err := os.Getwd() if err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("resolve working directory: %w", err)) } materialized, referenceDiagnostics, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{ ConfigPath: loadedConfigPath, WorkingDir: workingDir, }) if err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, err) } effective.ResolvedPipeline = materialized invocation.PipelineDigest = effective.ResolvedPipeline.Digest if err := writeSummary(summary, func() error { return summary.WriteInvocation(invocation) }); err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug invocation metadata: %w", err)) } if err := writeSummary(summary, func() error { return summary.WriteEffectiveConfig(loaded) }); err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug effective config: %w", err)) } if err := writeSummary(summary, func() error { return summary.WriteResolvedPipeline(effective) }); err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug resolved pipeline: %w", err)) } if err := writeSummary(summary, func() error { return summary.WriteResolvedReferences(pipeline.ReferenceProvenance(effective.ResolvedPipeline)) }); err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug resolved references: %w", err)) } registries, err := effectiveRegistries(opts) if err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, err) } ctx := context.Background() factoryProfileID := "" if len(profileIDs) == 1 { factoryProfileID = profileIDs[0] } llmClient, llmProfiles, err := opts.LLMClientFactory(ctx, effective.Config, factoryProfileID, runtimeOverrides) if err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("create LLM client for profile %q: %w", factoryProfileID, err)) } llmClient = pipeline.WithDebugLLMRecording(llmClient, debugRecorder) prepared, err := pipeline.Prepare(effective.ResolvedPipeline, registries, pipeline.ModuleDependencies{LLM: llmClient}) if err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("prepare pipeline %q: %w", pipelineID, err)) } rawInput, err := os.ReadFile(strings.TrimSpace(*inputPath)) if err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("read input %q: %w", strings.TrimSpace(*inputPath), err)) } effectiveSessionID, err := resolvePromptSessionID(requestedSessionID.value, effective.ResolvedPipeline.Input.Module, rawInput) if err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, err) } invocation.SessionID = effectiveSessionID if err := writeSummary(summary, func() error { return summary.WriteInvocation(invocation) }); err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug invocation metadata: %w", err)) } cache, err := artifactcache.NewFilesystemStore(effective.Config.CacheDirectory) if err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, err) } output, err := pipeline.New().Run(ctx, pipeline.RunInput{ Prepared: prepared, Path: strings.TrimSpace(*inputPath), RawInput: rawInput, SessionID: effectiveSessionID, RunID: runID, StartedAt: startedAt, LLMProfiles: llmProfiles, Metadata: runMetadata(effective.Config.Output.Directory, debugPath), Diagnostics: referenceDiagnostics, Cache: cache, Debug: debugRecorder, Workers: cfg.Concurrency.Workers, }) diagnosticProjection, diagnosticErr := contracts.ProjectDiagnosticCollection(output.Diagnostics) if diagnosticErr == nil { commandState.observeOutput(output, diagnosticProjection) } if err != nil { primaryErr := fmt.Errorf("run pipeline %q: %w", pipelineID, err) if output.Manifest.PipelineID != "" { if summaryErr := writePartialSummary(summary, output); summaryErr != nil { return failPipelineCommand(stderr, commandState, terminalWriter, primaryErr, fmt.Errorf("write debug summary: %w", summaryErr)) } } if diagnosticErr != nil { return failPipelineCommand(stderr, commandState, terminalWriter, primaryErr, fmt.Errorf("summarize run diagnostics: %w", diagnosticErr)) } return failPipelineCommand(stderr, commandState, terminalWriter, primaryErr) } if diagnosticErr != nil { return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("summarize run diagnostics: %w", diagnosticErr)) } if err := writePartialSummary(summary, output); err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug summary: %w", err)) } var encodedResult []byte if *machineOutput { result, err := newRunResultWithDiagnostics(effective.ResolvedPipeline, output, runOutputDir, debugPath, diagnosticProjection) if err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, err) } encodedResult, err = encodeRunResult(result) if err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, err) } } if err := writeOutputFiles(runOutputDir, output.OutputFiles); err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, err) } if primaryErr, persistenceErr := commandState.terminalize(terminalWriter, nil); primaryErr != nil { return writePipelineCommandFailure(stderr, commandState, primaryErr, persistenceErr) } if *machineOutput { if err := writeRunResult(stdout, encodedResult); err != nil { return writePipelineCommandFailure(stderr, commandState, errors.New("write run result"), nil) } } else { fmt.Fprintf(stdout, "pipeline %q complete: outputs=%d rejected=%d output=%s\n", effective.PipelineID, len(output.NormalizeOutputs), len(output.Rejected), runOutputDir) if debugPath != "" { fmt.Fprintf(stdout, "debug=%s\n", debugPath) } } if warningGroups := len(diagnosticProjection.Warnings); warningGroups > 0 { fmt.Fprintf(stderr, "notarius: run completed with %d warning group(s), %d occurrence(s)", warningGroups, diagnosticProjection.WarningOccurrenceCount) if warningFile, ok := logicalOutputFile(output.OutputFiles, "warnings.json"); ok { fmt.Fprintf(stderr, "; details=%s", filepath.Join(runOutputDir, warningFile)) } fmt.Fprintln(stderr) } return 0 } func logicalOutputFile(files []contracts.OutputFile, name string) (string, bool) { for _, file := range files { if file.Name == name { return file.Name, true } } return "", false } func writeSummary(summary *debugbundle.SummaryWriter, write func() error) error { if summary == nil { return nil } return write() } func writePartialSummary(summary *debugbundle.SummaryWriter, output pipeline.RunOutput) error { if summary == nil { return nil } if err := summary.WriteRunManifest(output.Manifest); err != nil { return err } if err := summary.WriteDiagnostics(output.Diagnostics); err != nil { return err } return nil } func rawInputDigest(data []byte) string { sum := sha256.Sum256(data) return "sha256:" + hex.EncodeToString(sum[:]) } func configSource(configPath string) string { if strings.TrimSpace(configPath) != "" { return "flag" } return "discovered" } func writeOutputFiles(runOutputDir string, files []contracts.OutputFile) error { for _, file := range files { if _, err := outputFilePath(runOutputDir, file.Name); err != nil { return err } } outputParent := filepath.Dir(runOutputDir) if err := os.MkdirAll(outputParent, 0o755); err != nil { return fmt.Errorf("create output parent %q: %w", outputParent, err) } if err := os.Mkdir(runOutputDir, 0o755); err != nil { if os.IsExist(err) { return fmt.Errorf("output run directory %q already exists", runOutputDir) } return fmt.Errorf("create output run directory %q: %w", runOutputDir, err) } for _, file := range files { if err := fileio.WriteBytes(runOutputDir, file.Name, file.Bytes, 0o755, 0o644); err != nil { return fmt.Errorf("write output file %q: %w", file.Name, err) } } return nil } func outputFilePath(runOutputDir, logicalName string) (string, error) { name := strings.TrimSpace(logicalName) if name == "" { return "", fmt.Errorf("output file name must not be empty") } if strings.Contains(name, `\`) { return "", fmt.Errorf("output file name %q must use slash-separated relative paths", name) } if path.IsAbs(name) || filepath.IsAbs(name) { return "", fmt.Errorf("output file name %q must be relative", name) } if strings.Contains(name, "..") { return "", fmt.Errorf("output file name %q must not contain ..", name) } cleaned := path.Clean(name) if cleaned == "." || cleaned != name { return "", fmt.Errorf("output file name %q must be clean", name) } root, err := filepath.Abs(runOutputDir) if err != nil { return "", fmt.Errorf("resolve output directory %q: %w", runOutputDir, err) } target, err := filepath.Abs(filepath.Join(root, filepath.FromSlash(cleaned))) if err != nil { return "", fmt.Errorf("resolve output file %q: %w", name, err) } rel, err := filepath.Rel(root, target) if err != nil { return "", fmt.Errorf("resolve output file %q: %w", name, err) } if rel == "." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) || rel == ".." { return "", fmt.Errorf("output file name %q resolves outside output directory", name) } return target, nil } func reorderRunArgs(args []string) []string { var flags []string var positionals []string for i := 0; i < len(args); i++ { arg := args[i] if arg == "--" { positionals = append(positionals, args[i+1:]...) break } if strings.HasPrefix(arg, "-") { flags = append(flags, arg) if runFlagTakesValue(arg) && !strings.Contains(arg, "=") && i+1 < len(args) { i++ flags = append(flags, args[i]) } continue } positionals = append(positionals, arg) } return append(flags, positionals...) } func runFlagTakesValue(arg string) bool { switch arg { case "--config", "--config-overlay", "--input", "--only", "--output-dir", "--debug-dir", "--llm-profile", "--session-id", "--reasoning-effort", "--reference", "--without-reference": return true default: return false } } func runConfigurationOverrides(outputDir, debugDir string) []configkit.Override { overrides := make([]configkit.Override, 0, 3) if outputDir = strings.TrimSpace(outputDir); outputDir != "" { overrides = append(overrides, configkit.Override{ Path: configkit.Keys("output", "directory"), Value: outputDir, Flag: "--output-dir", }) } if debugDir = strings.TrimSpace(debugDir); debugDir != "" { overrides = append(overrides, configkit.Override{ Path: configkit.Keys("debug", "directory"), Value: debugDir, Flag: "--debug-dir", }) } return overrides } func validateRunFlagValues(args []string) error { for i, arg := range args { if arg != "--config-overlay" && arg != "--session-id" && arg != "--reasoning-effort" { continue } if i+1 >= len(args) || strings.HasPrefix(args[i+1], "-") { return fmt.Errorf("flag needs an argument: %s", arg) } } return nil } func effectiveConfigOverlay(overlay string) string { if overlay == "" || overlay == "default" { return "" } return overlay } func flagWasProvided(args []string, name string) bool { for _, arg := range args { if arg == name || strings.HasPrefix(arg, name+"=") { return true } } return false } func effectiveLLMProfileIDs(resolved pipeline.ResolvedPipeline) []string { seen := make(map[string]struct{}) add := func(binding pipeline.ModuleBinding) { id := strings.TrimSpace(binding.LLMProfile) if id != "" { seen[id] = struct{}{} } } if resolved.InputExecutionClass == contracts.ExecutionClassLLMBacked { add(resolved.Input) } if resolved.ChunkExecutionClass == contracts.ExecutionClassLLMBacked { add(resolved.Chunk) } for _, lane := range resolved.AllArtifactLanes() { if lane.ExtractExecutionClass == contracts.ExecutionClassLLMBacked { add(lane.Extract) } if lane.MergeExecutionClass == contracts.ExecutionClassLLMBacked { add(lane.Merge) } if lane.NormalizeExecutionClass == contracts.ExecutionClassLLMBacked { add(lane.Normalize) } } for _, chain := range resolved.ValidatorChains { for _, validator := range chain.Validators { if validator.ExecutionClass == contracts.ExecutionClassLLMBacked { add(validator.Binding) } } } if resolved.OutputExecutionClass == contracts.ExecutionClassLLMBacked { add(resolved.Output) } ids := make([]string, 0, len(seen)) for id := range seen { ids = append(ids, id) } sort.Strings(ids) return ids } func runMetadata(outputDir, debugDir string) map[string]any { metadata := make(map[string]any) if dir := strings.TrimSpace(outputDir); dir != "" { metadata["output_dir"] = dir } if dir := strings.TrimSpace(debugDir); dir != "" { metadata["debug_dir"] = dir } if len(metadata) == 0 { return nil } return metadata } func parseOnly(raw string) ([]string, error) { if strings.TrimSpace(raw) == "" { return nil, nil } parts := strings.Split(raw, ",") result := make([]string, 0, len(parts)) for _, part := range parts { trimmed := strings.TrimSpace(part) if trimmed == "" { return nil, fmt.Errorf("--only must contain comma-separated non-empty artifact lane IDs") } result = append(result, trimmed) } return result, nil } type stringListFlag []string func (flag *stringListFlag) String() string { if flag == nil { return "" } return strings.Join(*flag, ",") } func (flag *stringListFlag) Set(value string) error { *flag = append(*flag, value) return nil } type sessionIDFlag struct { value string set bool } type singleValueFlag struct { name string value string set bool } func (flag *singleValueFlag) String() string { if flag == nil { return "" } return flag.value } func (flag *singleValueFlag) Set(value string) error { if flag.set { return fmt.Errorf("%s may be specified only once", flag.name) } flag.value = value flag.set = true return nil } func (flag *sessionIDFlag) String() string { if flag == nil { return "" } return flag.value } func (flag *sessionIDFlag) Set(value string) error { flag.value = value flag.set = true return nil } type cliReferenceRequest struct { Selector cliReferenceSelector Source string } type cliReferenceUnbindRequest struct { Selector cliReferenceSelector } type cliReferenceSelector struct { Scope cliReferenceSelectorScope LaneID string Stage pipeline.ModuleStage SlotName string } type cliReferenceSelectorScope uint8 const ( cliReferenceScopePipeline cliReferenceSelectorScope = iota cliReferenceScopeLane cliReferenceScopeChunk cliReferenceScopeBinding ) func parseReferenceFlags(values []string) ([]cliReferenceRequest, error) { if len(values) == 0 { return nil, nil } requests := make([]cliReferenceRequest, 0, len(values)) for _, raw := range values { name, source, ok := strings.Cut(raw, "=") if !ok { return nil, fmt.Errorf("--reference must use slot=path, lane.slot=path, or lane.stage.slot=path") } if strings.TrimSpace(source) == "" { return nil, fmt.Errorf("--reference path must not be empty; use --without-reference to unbind") } selector, err := parseReferenceSelector(name, "--reference") if err != nil { return nil, err } requests = append(requests, cliReferenceRequest{ Selector: selector, Source: strings.TrimSpace(source), }) } return requests, nil } func parseReferenceUnbindFlags(values []string) ([]cliReferenceUnbindRequest, error) { if len(values) == 0 { return nil, nil } requests := make([]cliReferenceUnbindRequest, 0, len(values)) for _, raw := range values { if strings.Contains(raw, "=") { return nil, fmt.Errorf("--without-reference must use a reference selector without =path") } selector, err := parseReferenceSelector(raw, "--without-reference") if err != nil { return nil, err } requests = append(requests, cliReferenceUnbindRequest{ Selector: selector, }) } return requests, nil } func parseReferenceSelector(raw string, flagName string) (cliReferenceSelector, error) { selector := strings.TrimSpace(raw) if selector == "" { return cliReferenceSelector{}, fmt.Errorf("%s reference slot must not be empty", flagName) } parts := strings.Split(selector, ".") for _, part := range parts { if strings.TrimSpace(part) == "" { return cliReferenceSelector{}, fmt.Errorf("%s must use non-empty reference selector values", flagName) } } switch len(parts) { case 1: return cliReferenceSelector{Scope: cliReferenceScopePipeline, SlotName: strings.TrimSpace(parts[0])}, nil case 2: first := strings.TrimSpace(parts[0]) slotName := strings.TrimSpace(parts[1]) if first == string(pipeline.StageChunk) { return cliReferenceSelector{Scope: cliReferenceScopeChunk, Stage: pipeline.StageChunk, SlotName: slotName}, nil } return cliReferenceSelector{Scope: cliReferenceScopeLane, LaneID: first, SlotName: slotName}, nil case 3: laneID := strings.TrimSpace(parts[0]) stage := pipeline.ModuleStage(strings.TrimSpace(parts[1])) slotName := strings.TrimSpace(parts[2]) if stage != pipeline.StageExtract && stage != pipeline.StageMerge && stage != pipeline.StageNormalize { return cliReferenceSelector{}, fmt.Errorf("%s lane-qualified selector must use lane.extract.slot, lane.merge.slot, or lane.normalize.slot", flagName) } return cliReferenceSelector{Scope: cliReferenceScopeBinding, LaneID: laneID, Stage: stage, SlotName: slotName}, nil default: return cliReferenceSelector{}, fmt.Errorf("%s must use slot, chunk.slot, lane.slot, lane.extract.slot, lane.merge.slot, or lane.normalize.slot", flagName) } } func resolveCLIReferenceRequests( cfg config.Config, pipelineID string, only []string, catalog pipeline.ModuleCatalog, referenceRequests []cliReferenceRequest, unbindRequests []cliReferenceUnbindRequest, ) ([]pipeline.ReferenceBinding, []pipeline.ReferenceUnbind, error) { if len(referenceRequests) == 0 && len(unbindRequests) == 0 { return nil, nil, nil } targets, err := selectedReferenceTargets(cfg, pipelineID, only, catalog) if err != nil { return nil, nil, err } // Broad CLI selectors are only presentation syntax. Collapse them into one // highest-specificity action per concrete framework target before pipeline // resolution so the generic reference contract stays stage-and-lane exact. actions := make(map[cliReferenceTargetKey]resolvedCLIReferenceAction) for _, request := range referenceRequests { matches, err := resolveCLIReferenceTargets(targets, request.Selector) if err != nil { return nil, nil, err } for _, target := range matches { candidate := resolvedCLIReferenceAction{ kind: cliReferenceActionBind, selector: request.Selector, target: target, slotName: request.Selector.SlotName, source: request.Source, specificity: request.Selector.specificity(), } if err := mergeCLIReferenceAction(actions, candidate); err != nil { return nil, nil, err } } } for _, request := range unbindRequests { matches, err := resolveCLIReferenceTargets(targets, request.Selector) if err != nil { return nil, nil, err } for _, target := range matches { candidate := resolvedCLIReferenceAction{ kind: cliReferenceActionUnbind, selector: request.Selector, target: target, slotName: request.Selector.SlotName, specificity: request.Selector.specificity(), } if err := mergeCLIReferenceAction(actions, candidate); err != nil { return nil, nil, err } } } resolved := make([]resolvedCLIReferenceAction, 0, len(actions)) for _, action := range actions { resolved = append(resolved, action) } sort.Slice(resolved, func(i, j int) bool { left, right := resolved[i], resolved[j] if left.target.laneID != right.target.laneID { return left.target.laneID < right.target.laneID } if left.target.stage != right.target.stage { return referenceStageOrder(left.target.stage) < referenceStageOrder(right.target.stage) } return left.slotName < right.slotName }) overrides := make([]pipeline.ReferenceBinding, 0, len(resolved)) unbinds := make([]pipeline.ReferenceUnbind, 0, len(resolved)) for _, action := range resolved { switch action.kind { case cliReferenceActionBind: overrides = append(overrides, pipeline.ReferenceBinding{ Stage: action.target.stage, LaneID: action.target.laneID, SlotName: action.slotName, Source: action.source, BindingSource: contracts.ReferenceBindingSourceCLI, }) case cliReferenceActionUnbind: unbinds = append(unbinds, pipeline.ReferenceUnbind{ Stage: action.target.stage, LaneID: action.target.laneID, SlotName: action.slotName, }) } } return overrides, unbinds, nil } type cliReferenceActionKind uint8 const ( cliReferenceActionBind cliReferenceActionKind = iota cliReferenceActionUnbind ) type cliReferenceTargetKey struct { stage pipeline.ModuleStage laneID string slotName string } type resolvedCLIReferenceAction struct { kind cliReferenceActionKind selector cliReferenceSelector target selectedReferenceTarget slotName string source string specificity int } func mergeCLIReferenceAction(actions map[cliReferenceTargetKey]resolvedCLIReferenceAction, candidate resolvedCLIReferenceAction) error { key := cliReferenceTargetKey{stage: candidate.target.stage, laneID: candidate.target.laneID, slotName: candidate.slotName} current, ok := actions[key] if !ok || candidate.specificity > current.specificity { actions[key] = candidate return nil } if candidate.specificity < current.specificity { return nil } if candidate.kind != current.kind { return fmt.Errorf("reference target %q slot %q is both bound by %q and unbound by %q at the same specificity", targetLabel(candidate.target), candidate.slotName, current.selector.String(), candidate.selector.String()) } actions[key] = candidate return nil } func (selector cliReferenceSelector) specificity() int { switch selector.Scope { case cliReferenceScopePipeline: return 0 case cliReferenceScopeLane: return 1 case cliReferenceScopeChunk, cliReferenceScopeBinding: return 2 default: return -1 } } func (selector cliReferenceSelector) String() string { switch selector.Scope { case cliReferenceScopePipeline: return selector.SlotName case cliReferenceScopeLane: return selector.LaneID + "." + selector.SlotName case cliReferenceScopeChunk: return "chunk." + selector.SlotName case cliReferenceScopeBinding: return selector.LaneID + "." + string(selector.Stage) + "." + selector.SlotName default: return selector.SlotName } } type selectedReferenceTarget struct { laneID string stage pipeline.ModuleStage module string slots map[string]struct{} } func selectedReferenceTargets(cfg config.Config, pipelineID string, only []string, catalog pipeline.ModuleCatalog) ([]selectedReferenceTarget, error) { profile, ok := lookupCLIReferencePipeline(cfg.Pipelines, pipelineID) if !ok { return nil, fmt.Errorf("pipeline %q is not configured", strings.TrimSpace(pipelineID)) } if profile.Steps != nil && len(only) > 0 { return nil, fmt.Errorf("pipeline %q --only is not supported for explicit ordered steps", strings.TrimSpace(pipelineID)) } lanesByID := make(map[string]pipeline.ArtifactLaneProfile, len(profile.Artifacts)) addLanes := func(values map[string]pipeline.ArtifactLaneProfile) error { for rawLaneID, lane := range values { laneID := strings.TrimSpace(rawLaneID) if laneID == "" { return fmt.Errorf("pipeline %q artifact lane id must not be empty", strings.TrimSpace(pipelineID)) } if _, ok := lanesByID[laneID]; ok { return fmt.Errorf("pipeline %q artifact lane %q is duplicated after trimming", strings.TrimSpace(pipelineID), laneID) } lanesByID[laneID] = lane } return nil } if profile.Steps == nil { if err := addLanes(profile.Artifacts); err != nil { return nil, err } } else { for _, step := range profile.Steps { if err := addLanes(step.Artifacts); err != nil { return nil, err } } } selectedIDs := make([]string, 0, len(lanesByID)) if len(only) == 0 { for laneID := range lanesByID { selectedIDs = append(selectedIDs, laneID) } } else { seen := make(map[string]struct{}, len(only)) for _, rawLaneID := range only { laneID := strings.TrimSpace(rawLaneID) if laneID == "" { return nil, fmt.Errorf("pipeline %q selected artifact lane id must not be empty", strings.TrimSpace(pipelineID)) } if _, ok := lanesByID[laneID]; !ok { return nil, fmt.Errorf("pipeline %q selected artifact lane %q is not declared", strings.TrimSpace(pipelineID), laneID) } if _, ok := seen[laneID]; !ok { selectedIDs = append(selectedIDs, laneID) seen[laneID] = struct{}{} } } } sort.Strings(selectedIDs) targets := make([]selectedReferenceTarget, 0, 1+len(selectedIDs)*3) chunk := pipeline.Binding(profile.Chunk.Module) chunk.Module = strings.TrimSpace(profile.Chunk.Module) if chunk.Module == "" { chunk.Module = pipeline.DefaultChunkModule } chunkSpec, err := cliReferenceChunkerSpec(catalog, chunk.Module) if err != nil { return nil, fmt.Errorf("pipeline %q chunk module %q: %w", strings.TrimSpace(pipelineID), chunk.Module, err) } targets = append(targets, selectedReferenceTarget{ stage: pipeline.StageChunk, module: chunk.Module, slots: referenceSlotSet(chunkSpec.ReferenceSlots), }) for _, laneID := range selectedIDs { lane := lanesByID[laneID] extractModule := strings.TrimSpace(lane.Extract.Module) if extractModule == "" { return nil, fmt.Errorf("pipeline %q lane %q extract module must not be empty", strings.TrimSpace(pipelineID), laneID) } extractSpec, err := cliReferenceExtractorSpec(catalog, extractModule) if err != nil { return nil, fmt.Errorf("pipeline %q lane %q extract module %q: %w", strings.TrimSpace(pipelineID), laneID, extractModule, err) } artifactKind := extractSpec.ArtifactKind if artifactKind == "" { return nil, fmt.Errorf("pipeline %q lane %q extract module %q does not declare an artifact kind", strings.TrimSpace(pipelineID), laneID, extractModule) } targets = append(targets, selectedReferenceTarget{ laneID: laneID, stage: pipeline.StageExtract, module: extractModule, slots: referenceSlotSet(extractSpec.ReferenceSlots), }) mergeModule := strings.TrimSpace(lane.Merge.Module) if mergeModule == "" { mergeModule = pipeline.DefaultMergeModule } mergeSpec, err := cliReferenceMergerSpec(catalog, mergeModule, artifactKind) if err != nil { return nil, fmt.Errorf("pipeline %q lane %q merge module %q: %w", strings.TrimSpace(pipelineID), laneID, mergeModule, err) } targets = append(targets, selectedReferenceTarget{ laneID: laneID, stage: pipeline.StageMerge, module: mergeModule, slots: referenceSlotSet(mergeSpec.ReferenceSlots), }) normalizeModule := strings.TrimSpace(lane.Normalize.Module) if normalizeModule == "" { normalizeModule = pipeline.DefaultNormalizeModule } normalizeSpec, err := cliReferenceNormalizerSpec(catalog, normalizeModule, artifactKind) if err != nil { return nil, fmt.Errorf("pipeline %q lane %q normalize module %q: %w", strings.TrimSpace(pipelineID), laneID, normalizeModule, err) } targets = append(targets, selectedReferenceTarget{ laneID: laneID, stage: pipeline.StageNormalize, module: normalizeModule, slots: referenceSlotSet(normalizeSpec.ReferenceSlots), }) } return targets, nil } func lookupCLIReferencePipeline(profiles map[string]pipeline.PipelineProfile, pipelineID string) (pipeline.PipelineProfile, bool) { pipelineID = strings.TrimSpace(pipelineID) for rawID, profile := range profiles { if strings.TrimSpace(rawID) == pipelineID { return profile, true } } return pipeline.PipelineProfile{}, false } func cliReferenceChunkerSpec(catalog pipeline.ModuleCatalog, module string) (pipeline.ModuleSpec, error) { if catalog.Chunkers == nil { return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module) } spec, ok := catalog.Chunkers.Spec(module) if !ok { return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module) } return spec, nil } func cliReferenceExtractorSpec(catalog pipeline.ModuleCatalog, module string) (pipeline.ModuleSpec, error) { if catalog.Extractors == nil { return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module) } spec, ok := catalog.Extractors.Spec(module) if !ok { return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module) } return spec, nil } func cliReferenceMergerSpec(catalog pipeline.ModuleCatalog, module string, kind contracts.ArtifactKind) (pipeline.ModuleSpec, error) { if catalog.Mergers == nil { return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module) } registered := catalog.Mergers.RegisteredArtifactKinds(module) if len(registered) == 0 { return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module) } spec, ok := catalog.Mergers.SpecForArtifact(module, kind) if !ok { return pipeline.ModuleSpec{}, cliReferenceArtifactVariantError("merger", module, kind, registered) } return spec, nil } func cliReferenceNormalizerSpec(catalog pipeline.ModuleCatalog, module string, kind contracts.ArtifactKind) (pipeline.ModuleSpec, error) { if catalog.Normalizers == nil { return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module) } registered := catalog.Normalizers.RegisteredArtifactKinds(module) if len(registered) == 0 { return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module) } spec, ok := catalog.Normalizers.SpecForArtifact(module, kind) if !ok { return pipeline.ModuleSpec{}, cliReferenceArtifactVariantError("normalizer", module, kind, registered) } return spec, nil } func cliReferenceArtifactVariantError(moduleType string, module string, kind contracts.ArtifactKind, registered []contracts.ArtifactKind) error { values := make([]string, len(registered)) for i, value := range registered { values[i] = string(value) } if len(values) == 0 { return fmt.Errorf("%s %q has no typed variant for artifact kind %q", moduleType, module, kind) } return fmt.Errorf("%s %q has no typed variant for artifact kind %q; registered kinds: %s", moduleType, module, kind, strings.Join(values, ", ")) } func referenceSlotSet(slots []contracts.ReferenceSlot) map[string]struct{} { slotSet := make(map[string]struct{}, len(slots)) for _, slot := range slots { slotSet[slot.Name] = struct{}{} } return slotSet } func resolveCLIReferenceTargets(targets []selectedReferenceTarget, selector cliReferenceSelector) ([]selectedReferenceTarget, error) { slotName := strings.TrimSpace(selector.SlotName) if slotName == "" { return nil, fmt.Errorf("reference slot must not be empty") } switch selector.Scope { case cliReferenceScopePipeline: matches := make([]selectedReferenceTarget, 0, len(targets)) for _, target := range targets { if _, ok := target.slots[slotName]; ok { matches = append(matches, target) } } if len(matches) == 0 { return nil, fmt.Errorf("reference slot %q is not declared by any selected target", slotName) } return matches, nil case cliReferenceScopeChunk: for _, target := range targets { if target.stage != pipeline.StageChunk { continue } if _, ok := target.slots[slotName]; !ok { return nil, fmt.Errorf("reference slot %q is not declared by chunk module %q", slotName, target.module) } return []selectedReferenceTarget{target}, nil } return nil, fmt.Errorf("reference chunk target is not selected") case cliReferenceScopeLane: laneSelected := false matches := make([]selectedReferenceTarget, 0, 3) for _, target := range targets { if target.laneID != selector.LaneID { continue } laneSelected = true if _, ok := target.slots[slotName]; ok { matches = append(matches, target) } } if !laneSelected { return nil, fmt.Errorf("reference lane %q is not selected", selector.LaneID) } if len(matches) == 0 { return nil, fmt.Errorf("reference slot %q is not declared by selected lane %q", slotName, selector.LaneID) } return matches, nil case cliReferenceScopeBinding: laneSelected := false for _, target := range targets { if target.laneID != selector.LaneID { continue } laneSelected = true if target.stage != selector.Stage { continue } if _, ok := target.slots[slotName]; !ok { return nil, fmt.Errorf("reference slot %q is not declared by selected %s target %q", slotName, selector.Stage, targetLabel(target)) } return []selectedReferenceTarget{target}, nil } if !laneSelected { return nil, fmt.Errorf("reference lane %q is not selected", selector.LaneID) } return nil, fmt.Errorf("reference %s target is not selected for lane %q", selector.Stage, selector.LaneID) default: return nil, fmt.Errorf("reference selector has unknown scope") } } func targetLabel(target selectedReferenceTarget) string { if target.stage == pipeline.StageChunk { return "chunk" } return target.laneID + "." + string(target.stage) } func referenceStageOrder(stage pipeline.ModuleStage) int { switch stage { case pipeline.StageChunk: return 0 case pipeline.StageExtract: return 1 case pipeline.StageMerge: return 2 case pipeline.StageNormalize: return 3 default: return 4 } } func sortedPipelineIDs(cfg config.Config) []string { ids := make([]string, 0, len(cfg.Pipelines)) for id := range cfg.Pipelines { ids = append(ids, strings.TrimSpace(id)) } sort.Strings(ids) return ids }