Record debug evidence in one run-scoped capture session
This commit is contained in:
@@ -18,7 +18,6 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/buildinfo"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/fileio"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/artifactcache"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
@@ -31,18 +30,16 @@ import (
|
||||
)
|
||||
|
||||
type Options struct {
|
||||
LLMClient promptkit.LLMClient
|
||||
LLMRuntime promptkit.RuntimeConfig
|
||||
Catalog pipeline.ModuleCatalog
|
||||
Registries pipeline.Registries
|
||||
LLMClientFactory LLMClientFactory
|
||||
RunIDGenerator RunIDGenerator
|
||||
LookupEnv func(string) (string, bool)
|
||||
Now func() time.Time
|
||||
UserCacheDir func() (string, error)
|
||||
DebugRecorderFactory func(string) (pipeline.DebugRecorder, error)
|
||||
DebugTerminalFactory func(*debugbundle.SummaryWriter) DebugTerminalWriter
|
||||
promptKitAssets *frameworkllm.AssetRegistry
|
||||
LLMClient promptkit.LLMClient
|
||||
LLMRuntime promptkit.RuntimeConfig
|
||||
Catalog pipeline.ModuleCatalog
|
||||
Registries pipeline.Registries
|
||||
LLMClientFactory LLMClientFactory
|
||||
RunIDGenerator RunIDGenerator
|
||||
LookupEnv func(string) (string, bool)
|
||||
Now func() time.Time
|
||||
UserCacheDir func() (string, error)
|
||||
promptKitAssets *frameworkllm.AssetRegistry
|
||||
}
|
||||
|
||||
type LLMRuntimeOverrides struct {
|
||||
@@ -83,12 +80,6 @@ func normalizeOptions(opts Options) (Options, error) {
|
||||
if opts.UserCacheDir == nil {
|
||||
opts.UserCacheDir = os.UserCacheDir
|
||||
}
|
||||
if opts.DebugRecorderFactory == nil {
|
||||
opts.DebugRecorderFactory = frameworkdebug.NewFilesystemRecorder
|
||||
}
|
||||
if opts.DebugTerminalFactory == nil {
|
||||
opts.DebugTerminalFactory = func(writer *debugbundle.SummaryWriter) DebugTerminalWriter { return writer }
|
||||
}
|
||||
if isEmptyCatalog(opts.Catalog) && isEmptyRegistries(opts.Registries) {
|
||||
components, err := newProductionComponents()
|
||||
if err != nil {
|
||||
@@ -158,18 +149,10 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
fmt.Fprintln(stderr, "notarius: run requires --input")
|
||||
return 2
|
||||
}
|
||||
if strings.TrimSpace(*debugDir) != "" && !*debug {
|
||||
fmt.Fprintln(stderr, "notarius: --debug-dir requires --debug")
|
||||
return 2
|
||||
}
|
||||
if strings.TrimSpace(*outputDir) == "" && flagWasProvided(args, "--output-dir") {
|
||||
fmt.Fprintln(stderr, "notarius: --output-dir must not be empty")
|
||||
return 2
|
||||
}
|
||||
if strings.TrimSpace(*debugDir) == "" && flagWasProvided(args, "--debug-dir") {
|
||||
fmt.Fprintln(stderr, "notarius: --debug-dir must not be empty")
|
||||
return 2
|
||||
}
|
||||
if requestedSessionID.set && strings.TrimSpace(requestedSessionID.value) == "" {
|
||||
fmt.Fprintln(stderr, "notarius: --session-id must not be empty")
|
||||
return 2
|
||||
@@ -210,7 +193,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
return 2
|
||||
}
|
||||
|
||||
loaded, loadedConfigPath, err := loadConfigKit(*configPath, *configOverlay, runConfigurationOverrides(*outputDir, *debugDir), opts)
|
||||
loaded, loadedConfigPath, err := loadConfigKit(*configPath, *configOverlay, runConfigurationOverrides(*outputDir), opts)
|
||||
if err != nil {
|
||||
writeConfigLoadError(stderr, err)
|
||||
return 1
|
||||
@@ -231,52 +214,48 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
return 1
|
||||
}
|
||||
runOutputDir := filepath.Join(cfg.Output.Directory, runID)
|
||||
commandState := newPipelineCommandState(runID, pipelineID, runOutputDir)
|
||||
var summary *debugbundle.SummaryWriter
|
||||
var terminalWriter DebugTerminalWriter
|
||||
debugPath := ""
|
||||
debugRecorder := pipeline.NoopDebugRecorder()
|
||||
if *debug {
|
||||
bundle, err := debugbundle.Allocate(cfg.Debug.Directory, runID, startedAt)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
debugPath, summary = bundle.Path(), bundle.Summary()
|
||||
commandState.setDebugPath(debugPath)
|
||||
terminalWriter = opts.DebugTerminalFactory(summary)
|
||||
if terminalWriter == nil {
|
||||
terminalWriter = summary
|
||||
}
|
||||
debugRecorder, err = opts.DebugRecorderFactory(bundle.TraceRoot())
|
||||
if err != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("create debug recorder: %w", err))
|
||||
}
|
||||
debugRecorder = pipeline.SynchronizedDebugRecorder(debugRecorder)
|
||||
|
||||
var capture *frameworkdebug.Recorder
|
||||
var output pipeline.RunOutput
|
||||
published := false
|
||||
captureOutcome := func() any {
|
||||
return map[string]any{"pipeline": pipelineID, "output_directory": runOutputDir, "artifact_count": output.ArtifactCount(), "fallback_count": output.FallbackCount, "published": published}
|
||||
}
|
||||
invocation := debugbundle.Invocation{
|
||||
Operation: "run",
|
||||
PipelineID: pipelineID,
|
||||
InputPath: strings.TrimSpace(*inputPath),
|
||||
ConfigPath: loadedConfigPath,
|
||||
ConfigSource: configSource(*configPath),
|
||||
ConfigOverlay: effectiveConfigOverlay(*configOverlay),
|
||||
OnlyLanes: append([]string(nil), only...),
|
||||
ReasoningEffortOverride: runtimeOverrides.ReasoningEffort,
|
||||
RunID: runID,
|
||||
StartedAt: startedAt,
|
||||
fail := func(primary error, extra ...error) int {
|
||||
cause := errors.Join(append([]error{primary}, extra...)...)
|
||||
cause = capture.Finalize(captureOutcome(), cause)
|
||||
fmt.Fprintf(stderr, "notarius: %v\n", cause)
|
||||
if published {
|
||||
fmt.Fprintf(stderr, "notarius: artifacts=%s\n", runOutputDir)
|
||||
}
|
||||
if capture.Path() != "" {
|
||||
fmt.Fprintf(stderr, "notarius: debug=%s\n", capture.Path())
|
||||
}
|
||||
return 1
|
||||
}
|
||||
if err := writeSummary(summary, func() error { return summary.WriteInvocation(invocation) }); err != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug invocation metadata: %w", err))
|
||||
capture, err = startRunDebug(runDebugInput{
|
||||
Enabled: commandkit.Supplied[bool]{Value: *debug, Set: flagWasProvided(args, "--debug")},
|
||||
Directory: commandkit.Supplied[string]{Value: *debugDir, Set: flagWasProvided(args, "--debug-dir")},
|
||||
}, cfg.Debug.Directory, runID, startedAt)
|
||||
if err != nil {
|
||||
return fail(err)
|
||||
}
|
||||
debugPath := capture.Path()
|
||||
var debugRecorder pipeline.DebugRecorder = pipeline.NoopDebugRecorder()
|
||||
if capture.Enabled() {
|
||||
debugRecorder = capture
|
||||
}
|
||||
if err := capture.WriteJSON("context.json", map[string]any{"pipeline": pipelineID, "input_path": *inputPath, "config_path": loadedConfigPath, "run_id": runID, "started_at": startedAt}); err != nil {
|
||||
return fail(err)
|
||||
}
|
||||
|
||||
catalog, err := effectiveCatalog(opts)
|
||||
if err != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, err)
|
||||
return fail(err)
|
||||
}
|
||||
referenceOverrides, referenceUnbinds, err := resolveCLIReferenceRequests(cfg, pipelineID, only, catalog, referenceRequests, referenceUnbindRequests)
|
||||
if err != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, err)
|
||||
return fail(err)
|
||||
}
|
||||
effective, err := cfg.Resolve(config.ResolveInput{
|
||||
PipelineID: pipelineID,
|
||||
@@ -287,43 +266,28 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
ReferenceUnbinds: referenceUnbinds,
|
||||
})
|
||||
if err != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, err)
|
||||
return fail(err)
|
||||
}
|
||||
profileIDs := effectiveLLMProfileIDs(effective.ResolvedPipeline)
|
||||
if err := validateExplicitPromptKitProfiles(context.Background(), effective.Config, profileIDs, opts.promptKitAssets); err != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, err)
|
||||
return fail(err)
|
||||
}
|
||||
workingDir, err := os.Getwd()
|
||||
if err != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("resolve working directory: %w", err))
|
||||
return fail(fmt.Errorf("resolve working directory: %w", err))
|
||||
}
|
||||
materialized, referenceDiagnostics, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{
|
||||
ConfigPath: loadedConfigPath,
|
||||
WorkingDir: workingDir,
|
||||
})
|
||||
if err != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, err)
|
||||
return fail(err)
|
||||
}
|
||||
effective.ResolvedPipeline = materialized
|
||||
invocation.PipelineDigest = effective.ResolvedPipeline.Digest
|
||||
if err := writeSummary(summary, func() error { return summary.WriteInvocation(invocation) }); err != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug invocation metadata: %w", err))
|
||||
}
|
||||
if err := writeSummary(summary, func() error { return summary.WriteEffectiveConfig(loaded) }); err != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug effective config: %w", err))
|
||||
}
|
||||
if err := writeSummary(summary, func() error { return summary.WriteResolvedPipeline(effective) }); err != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug resolved pipeline: %w", err))
|
||||
}
|
||||
if err := writeSummary(summary, func() error {
|
||||
return summary.WriteResolvedReferences(pipeline.ReferenceProvenance(effective.ResolvedPipeline))
|
||||
}); err != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug resolved references: %w", err))
|
||||
}
|
||||
|
||||
registries, err := effectiveRegistries(opts)
|
||||
if err != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, err)
|
||||
return fail(err)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
@@ -333,31 +297,27 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
}
|
||||
llmClient, llmProfiles, err := opts.LLMClientFactory(ctx, effective.Config, factoryProfileID, runtimeOverrides)
|
||||
if err != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("create LLM client for profile %q: %w", factoryProfileID, err))
|
||||
return fail(fmt.Errorf("create LLM client for profile %q: %w", factoryProfileID, err))
|
||||
}
|
||||
llmClient = pipeline.WithDebugLLMRecording(llmClient, debugRecorder)
|
||||
prepared, err := pipeline.Prepare(effective.ResolvedPipeline, registries, pipeline.ModuleDependencies{LLM: llmClient})
|
||||
if err != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("prepare pipeline %q: %w", pipelineID, err))
|
||||
return fail(fmt.Errorf("prepare pipeline %q: %w", pipelineID, err))
|
||||
}
|
||||
rawInput, err := os.ReadFile(strings.TrimSpace(*inputPath))
|
||||
if err != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("read input %q: %w", strings.TrimSpace(*inputPath), err))
|
||||
return fail(fmt.Errorf("read input %q: %w", strings.TrimSpace(*inputPath), err))
|
||||
}
|
||||
effectiveSessionID, err := resolvePromptSessionID(requestedSessionID.value, effective.ResolvedPipeline.Input.Module, rawInput)
|
||||
if err != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, err)
|
||||
}
|
||||
invocation.SessionID = effectiveSessionID
|
||||
if err := writeSummary(summary, func() error { return summary.WriteInvocation(invocation) }); err != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug invocation metadata: %w", err))
|
||||
return fail(err)
|
||||
}
|
||||
cache, err := artifactcache.NewFilesystemStore(effective.Config.CacheDirectory)
|
||||
if err != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, err)
|
||||
return fail(err)
|
||||
}
|
||||
|
||||
output, err := pipeline.New().Run(ctx, pipeline.RunInput{
|
||||
output, err = pipeline.New().Run(ctx, pipeline.RunInput{
|
||||
Prepared: prepared,
|
||||
Path: strings.TrimSpace(*inputPath),
|
||||
RawInput: rawInput,
|
||||
@@ -372,52 +332,42 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
Workers: cfg.Concurrency.Workers,
|
||||
})
|
||||
diagnosticProjection, diagnosticErr := contracts.ProjectDiagnosticCollection(output.Diagnostics)
|
||||
if diagnosticErr == nil {
|
||||
commandState.observeOutput(output, diagnosticProjection)
|
||||
}
|
||||
if err != nil {
|
||||
primaryErr := fmt.Errorf("run pipeline %q: %w", pipelineID, err)
|
||||
if output.Manifest.PipelineID != "" {
|
||||
if summaryErr := writePartialSummary(summary, output); summaryErr != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, primaryErr, fmt.Errorf("write debug summary: %w", summaryErr))
|
||||
}
|
||||
}
|
||||
if diagnosticErr != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, primaryErr, fmt.Errorf("summarize run diagnostics: %w", diagnosticErr))
|
||||
return fail(primaryErr, fmt.Errorf("summarize run diagnostics: %w", diagnosticErr))
|
||||
}
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, primaryErr)
|
||||
return fail(primaryErr)
|
||||
}
|
||||
if diagnosticErr != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("summarize run diagnostics: %w", diagnosticErr))
|
||||
return fail(fmt.Errorf("summarize run diagnostics: %w", diagnosticErr))
|
||||
}
|
||||
|
||||
if err := writePartialSummary(summary, output); err != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug summary: %w", err))
|
||||
}
|
||||
var encodedResult []byte
|
||||
if *machineOutput {
|
||||
result, err := newRunResultWithDiagnostics(effective.ResolvedPipeline, output, runOutputDir, debugPath, diagnosticProjection)
|
||||
if err != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, err)
|
||||
return fail(err)
|
||||
}
|
||||
encodedResult, err = encodeRunResult(result)
|
||||
if err != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, err)
|
||||
return fail(err)
|
||||
}
|
||||
}
|
||||
if err := writeOutputFiles(runOutputDir, output.OutputFiles); err != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, err)
|
||||
return fail(err)
|
||||
}
|
||||
if primaryErr, persistenceErr := commandState.terminalize(terminalWriter, nil); primaryErr != nil {
|
||||
return writePipelineCommandFailure(stderr, commandState, primaryErr, persistenceErr)
|
||||
published = true
|
||||
if err := capture.Finalize(captureOutcome(), nil); err != nil {
|
||||
return fail(err)
|
||||
}
|
||||
|
||||
if *machineOutput {
|
||||
if err := writeRunResult(stdout, encodedResult); err != nil {
|
||||
return writePipelineCommandFailure(stderr, commandState, errors.New("write run result"), nil)
|
||||
return fail(errors.New("write run result"))
|
||||
}
|
||||
} else {
|
||||
fmt.Fprintf(stdout, "pipeline %q complete: outputs=%d rejected=%d output=%s\n", effective.PipelineID, len(output.NormalizeOutputs), len(output.Rejected), runOutputDir)
|
||||
fmt.Fprintf(stdout, "pipeline %q complete: artifacts=%d output=%s\n", effective.PipelineID, output.ArtifactCount(), runOutputDir)
|
||||
if debugPath != "" {
|
||||
fmt.Fprintf(stdout, "debug=%s\n", debugPath)
|
||||
}
|
||||
@@ -441,26 +391,6 @@ func logicalOutputFile(files []contracts.OutputFile, name string) (string, bool)
|
||||
return "", false
|
||||
}
|
||||
|
||||
func writeSummary(summary *debugbundle.SummaryWriter, write func() error) error {
|
||||
if summary == nil {
|
||||
return nil
|
||||
}
|
||||
return write()
|
||||
}
|
||||
|
||||
func writePartialSummary(summary *debugbundle.SummaryWriter, output pipeline.RunOutput) error {
|
||||
if summary == nil {
|
||||
return nil
|
||||
}
|
||||
if err := summary.WriteRunManifest(output.Manifest); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := summary.WriteDiagnostics(output.Diagnostics); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func rawInputDigest(data []byte) string {
|
||||
sum := sha256.Sum256(data)
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
@@ -566,7 +496,7 @@ func runFlagTakesValue(arg string) bool {
|
||||
}
|
||||
}
|
||||
|
||||
func runConfigurationOverrides(outputDir, debugDir string) []configkit.Override {
|
||||
func runConfigurationOverrides(outputDir string) []configkit.Override {
|
||||
overrides := make([]configkit.Override, 0, 3)
|
||||
if outputDir = strings.TrimSpace(outputDir); outputDir != "" {
|
||||
overrides = append(overrides, configkit.Override{
|
||||
@@ -575,13 +505,6 @@ func runConfigurationOverrides(outputDir, debugDir string) []configkit.Override
|
||||
Flag: "--output-dir",
|
||||
})
|
||||
}
|
||||
if debugDir = strings.TrimSpace(debugDir); debugDir != "" {
|
||||
overrides = append(overrides, configkit.Override{
|
||||
Path: configkit.Keys("debug", "directory"),
|
||||
Value: debugDir,
|
||||
Flag: "--debug-dir",
|
||||
})
|
||||
}
|
||||
return overrides
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user