Record debug evidence in one run-scoped capture session

This commit is contained in:
2026-11-22 02:49:48 +00:00
parent 584b041e57
commit 8680060400
10 changed files with 700 additions and 676 deletions

View File

@@ -18,7 +18,6 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/buildinfo"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle"
"gitea.maximumdirect.net/eric/notarius/internal/core/fileio"
"gitea.maximumdirect.net/eric/notarius/internal/framework/artifactcache"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
@@ -31,18 +30,16 @@ import (
)
type Options struct {
LLMClient promptkit.LLMClient
LLMRuntime promptkit.RuntimeConfig
Catalog pipeline.ModuleCatalog
Registries pipeline.Registries
LLMClientFactory LLMClientFactory
RunIDGenerator RunIDGenerator
LookupEnv func(string) (string, bool)
Now func() time.Time
UserCacheDir func() (string, error)
DebugRecorderFactory func(string) (pipeline.DebugRecorder, error)
DebugTerminalFactory func(*debugbundle.SummaryWriter) DebugTerminalWriter
promptKitAssets *frameworkllm.AssetRegistry
LLMClient promptkit.LLMClient
LLMRuntime promptkit.RuntimeConfig
Catalog pipeline.ModuleCatalog
Registries pipeline.Registries
LLMClientFactory LLMClientFactory
RunIDGenerator RunIDGenerator
LookupEnv func(string) (string, bool)
Now func() time.Time
UserCacheDir func() (string, error)
promptKitAssets *frameworkllm.AssetRegistry
}
type LLMRuntimeOverrides struct {
@@ -83,12 +80,6 @@ func normalizeOptions(opts Options) (Options, error) {
if opts.UserCacheDir == nil {
opts.UserCacheDir = os.UserCacheDir
}
if opts.DebugRecorderFactory == nil {
opts.DebugRecorderFactory = frameworkdebug.NewFilesystemRecorder
}
if opts.DebugTerminalFactory == nil {
opts.DebugTerminalFactory = func(writer *debugbundle.SummaryWriter) DebugTerminalWriter { return writer }
}
if isEmptyCatalog(opts.Catalog) && isEmptyRegistries(opts.Registries) {
components, err := newProductionComponents()
if err != nil {
@@ -158,18 +149,10 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
fmt.Fprintln(stderr, "notarius: run requires --input")
return 2
}
if strings.TrimSpace(*debugDir) != "" && !*debug {
fmt.Fprintln(stderr, "notarius: --debug-dir requires --debug")
return 2
}
if strings.TrimSpace(*outputDir) == "" && flagWasProvided(args, "--output-dir") {
fmt.Fprintln(stderr, "notarius: --output-dir must not be empty")
return 2
}
if strings.TrimSpace(*debugDir) == "" && flagWasProvided(args, "--debug-dir") {
fmt.Fprintln(stderr, "notarius: --debug-dir must not be empty")
return 2
}
if requestedSessionID.set && strings.TrimSpace(requestedSessionID.value) == "" {
fmt.Fprintln(stderr, "notarius: --session-id must not be empty")
return 2
@@ -210,7 +193,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
return 2
}
loaded, loadedConfigPath, err := loadConfigKit(*configPath, *configOverlay, runConfigurationOverrides(*outputDir, *debugDir), opts)
loaded, loadedConfigPath, err := loadConfigKit(*configPath, *configOverlay, runConfigurationOverrides(*outputDir), opts)
if err != nil {
writeConfigLoadError(stderr, err)
return 1
@@ -231,52 +214,48 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
return 1
}
runOutputDir := filepath.Join(cfg.Output.Directory, runID)
commandState := newPipelineCommandState(runID, pipelineID, runOutputDir)
var summary *debugbundle.SummaryWriter
var terminalWriter DebugTerminalWriter
debugPath := ""
debugRecorder := pipeline.NoopDebugRecorder()
if *debug {
bundle, err := debugbundle.Allocate(cfg.Debug.Directory, runID, startedAt)
if err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 1
}
debugPath, summary = bundle.Path(), bundle.Summary()
commandState.setDebugPath(debugPath)
terminalWriter = opts.DebugTerminalFactory(summary)
if terminalWriter == nil {
terminalWriter = summary
}
debugRecorder, err = opts.DebugRecorderFactory(bundle.TraceRoot())
if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("create debug recorder: %w", err))
}
debugRecorder = pipeline.SynchronizedDebugRecorder(debugRecorder)
var capture *frameworkdebug.Recorder
var output pipeline.RunOutput
published := false
captureOutcome := func() any {
return map[string]any{"pipeline": pipelineID, "output_directory": runOutputDir, "artifact_count": output.ArtifactCount(), "fallback_count": output.FallbackCount, "published": published}
}
invocation := debugbundle.Invocation{
Operation: "run",
PipelineID: pipelineID,
InputPath: strings.TrimSpace(*inputPath),
ConfigPath: loadedConfigPath,
ConfigSource: configSource(*configPath),
ConfigOverlay: effectiveConfigOverlay(*configOverlay),
OnlyLanes: append([]string(nil), only...),
ReasoningEffortOverride: runtimeOverrides.ReasoningEffort,
RunID: runID,
StartedAt: startedAt,
fail := func(primary error, extra ...error) int {
cause := errors.Join(append([]error{primary}, extra...)...)
cause = capture.Finalize(captureOutcome(), cause)
fmt.Fprintf(stderr, "notarius: %v\n", cause)
if published {
fmt.Fprintf(stderr, "notarius: artifacts=%s\n", runOutputDir)
}
if capture.Path() != "" {
fmt.Fprintf(stderr, "notarius: debug=%s\n", capture.Path())
}
return 1
}
if err := writeSummary(summary, func() error { return summary.WriteInvocation(invocation) }); err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug invocation metadata: %w", err))
capture, err = startRunDebug(runDebugInput{
Enabled: commandkit.Supplied[bool]{Value: *debug, Set: flagWasProvided(args, "--debug")},
Directory: commandkit.Supplied[string]{Value: *debugDir, Set: flagWasProvided(args, "--debug-dir")},
}, cfg.Debug.Directory, runID, startedAt)
if err != nil {
return fail(err)
}
debugPath := capture.Path()
var debugRecorder pipeline.DebugRecorder = pipeline.NoopDebugRecorder()
if capture.Enabled() {
debugRecorder = capture
}
if err := capture.WriteJSON("context.json", map[string]any{"pipeline": pipelineID, "input_path": *inputPath, "config_path": loadedConfigPath, "run_id": runID, "started_at": startedAt}); err != nil {
return fail(err)
}
catalog, err := effectiveCatalog(opts)
if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, err)
return fail(err)
}
referenceOverrides, referenceUnbinds, err := resolveCLIReferenceRequests(cfg, pipelineID, only, catalog, referenceRequests, referenceUnbindRequests)
if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, err)
return fail(err)
}
effective, err := cfg.Resolve(config.ResolveInput{
PipelineID: pipelineID,
@@ -287,43 +266,28 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
ReferenceUnbinds: referenceUnbinds,
})
if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, err)
return fail(err)
}
profileIDs := effectiveLLMProfileIDs(effective.ResolvedPipeline)
if err := validateExplicitPromptKitProfiles(context.Background(), effective.Config, profileIDs, opts.promptKitAssets); err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, err)
return fail(err)
}
workingDir, err := os.Getwd()
if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("resolve working directory: %w", err))
return fail(fmt.Errorf("resolve working directory: %w", err))
}
materialized, referenceDiagnostics, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{
ConfigPath: loadedConfigPath,
WorkingDir: workingDir,
})
if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, err)
return fail(err)
}
effective.ResolvedPipeline = materialized
invocation.PipelineDigest = effective.ResolvedPipeline.Digest
if err := writeSummary(summary, func() error { return summary.WriteInvocation(invocation) }); err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug invocation metadata: %w", err))
}
if err := writeSummary(summary, func() error { return summary.WriteEffectiveConfig(loaded) }); err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug effective config: %w", err))
}
if err := writeSummary(summary, func() error { return summary.WriteResolvedPipeline(effective) }); err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug resolved pipeline: %w", err))
}
if err := writeSummary(summary, func() error {
return summary.WriteResolvedReferences(pipeline.ReferenceProvenance(effective.ResolvedPipeline))
}); err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug resolved references: %w", err))
}
registries, err := effectiveRegistries(opts)
if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, err)
return fail(err)
}
ctx := context.Background()
@@ -333,31 +297,27 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
}
llmClient, llmProfiles, err := opts.LLMClientFactory(ctx, effective.Config, factoryProfileID, runtimeOverrides)
if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("create LLM client for profile %q: %w", factoryProfileID, err))
return fail(fmt.Errorf("create LLM client for profile %q: %w", factoryProfileID, err))
}
llmClient = pipeline.WithDebugLLMRecording(llmClient, debugRecorder)
prepared, err := pipeline.Prepare(effective.ResolvedPipeline, registries, pipeline.ModuleDependencies{LLM: llmClient})
if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("prepare pipeline %q: %w", pipelineID, err))
return fail(fmt.Errorf("prepare pipeline %q: %w", pipelineID, err))
}
rawInput, err := os.ReadFile(strings.TrimSpace(*inputPath))
if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("read input %q: %w", strings.TrimSpace(*inputPath), err))
return fail(fmt.Errorf("read input %q: %w", strings.TrimSpace(*inputPath), err))
}
effectiveSessionID, err := resolvePromptSessionID(requestedSessionID.value, effective.ResolvedPipeline.Input.Module, rawInput)
if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, err)
}
invocation.SessionID = effectiveSessionID
if err := writeSummary(summary, func() error { return summary.WriteInvocation(invocation) }); err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug invocation metadata: %w", err))
return fail(err)
}
cache, err := artifactcache.NewFilesystemStore(effective.Config.CacheDirectory)
if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, err)
return fail(err)
}
output, err := pipeline.New().Run(ctx, pipeline.RunInput{
output, err = pipeline.New().Run(ctx, pipeline.RunInput{
Prepared: prepared,
Path: strings.TrimSpace(*inputPath),
RawInput: rawInput,
@@ -372,52 +332,42 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
Workers: cfg.Concurrency.Workers,
})
diagnosticProjection, diagnosticErr := contracts.ProjectDiagnosticCollection(output.Diagnostics)
if diagnosticErr == nil {
commandState.observeOutput(output, diagnosticProjection)
}
if err != nil {
primaryErr := fmt.Errorf("run pipeline %q: %w", pipelineID, err)
if output.Manifest.PipelineID != "" {
if summaryErr := writePartialSummary(summary, output); summaryErr != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, primaryErr, fmt.Errorf("write debug summary: %w", summaryErr))
}
}
if diagnosticErr != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, primaryErr, fmt.Errorf("summarize run diagnostics: %w", diagnosticErr))
return fail(primaryErr, fmt.Errorf("summarize run diagnostics: %w", diagnosticErr))
}
return failPipelineCommand(stderr, commandState, terminalWriter, primaryErr)
return fail(primaryErr)
}
if diagnosticErr != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("summarize run diagnostics: %w", diagnosticErr))
return fail(fmt.Errorf("summarize run diagnostics: %w", diagnosticErr))
}
if err := writePartialSummary(summary, output); err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug summary: %w", err))
}
var encodedResult []byte
if *machineOutput {
result, err := newRunResultWithDiagnostics(effective.ResolvedPipeline, output, runOutputDir, debugPath, diagnosticProjection)
if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, err)
return fail(err)
}
encodedResult, err = encodeRunResult(result)
if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, err)
return fail(err)
}
}
if err := writeOutputFiles(runOutputDir, output.OutputFiles); err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, err)
return fail(err)
}
if primaryErr, persistenceErr := commandState.terminalize(terminalWriter, nil); primaryErr != nil {
return writePipelineCommandFailure(stderr, commandState, primaryErr, persistenceErr)
published = true
if err := capture.Finalize(captureOutcome(), nil); err != nil {
return fail(err)
}
if *machineOutput {
if err := writeRunResult(stdout, encodedResult); err != nil {
return writePipelineCommandFailure(stderr, commandState, errors.New("write run result"), nil)
return fail(errors.New("write run result"))
}
} else {
fmt.Fprintf(stdout, "pipeline %q complete: outputs=%d rejected=%d output=%s\n", effective.PipelineID, len(output.NormalizeOutputs), len(output.Rejected), runOutputDir)
fmt.Fprintf(stdout, "pipeline %q complete: artifacts=%d output=%s\n", effective.PipelineID, output.ArtifactCount(), runOutputDir)
if debugPath != "" {
fmt.Fprintf(stdout, "debug=%s\n", debugPath)
}
@@ -441,26 +391,6 @@ func logicalOutputFile(files []contracts.OutputFile, name string) (string, bool)
return "", false
}
func writeSummary(summary *debugbundle.SummaryWriter, write func() error) error {
if summary == nil {
return nil
}
return write()
}
func writePartialSummary(summary *debugbundle.SummaryWriter, output pipeline.RunOutput) error {
if summary == nil {
return nil
}
if err := summary.WriteRunManifest(output.Manifest); err != nil {
return err
}
if err := summary.WriteDiagnostics(output.Diagnostics); err != nil {
return err
}
return nil
}
func rawInputDigest(data []byte) string {
sum := sha256.Sum256(data)
return "sha256:" + hex.EncodeToString(sum[:])
@@ -566,7 +496,7 @@ func runFlagTakesValue(arg string) bool {
}
}
func runConfigurationOverrides(outputDir, debugDir string) []configkit.Override {
func runConfigurationOverrides(outputDir string) []configkit.Override {
overrides := make([]configkit.Override, 0, 3)
if outputDir = strings.TrimSpace(outputDir); outputDir != "" {
overrides = append(overrides, configkit.Override{
@@ -575,13 +505,6 @@ func runConfigurationOverrides(outputDir, debugDir string) []configkit.Override
Flag: "--output-dir",
})
}
if debugDir = strings.TrimSpace(debugDir); debugDir != "" {
overrides = append(overrides, configkit.Override{
Path: configkit.Keys("debug", "directory"),
Value: debugDir,
Flag: "--debug-dir",
})
}
return overrides
}