eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Skip report and debug work for disabled outputs

Browse Source
This commit is contained in:
Eric Rakestraw
2026-11-10 20:48:32 +00:00
parent a9b7c37a7e
commit 71a9b1f560
11 changed files with 161 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
internal/cli/process_inputs.go
View File

@@ -87,24 +87,31 @@ func prepareProcess(ctx context.Context, input processInput, loaded *processInpu
return nil, err
}
started := time.Now().UTC()
snapshot := reporting.Snapshot{RunID: runID, Status: reporting.StatusInProgress, StartedAt: started, TranscriptPath: input.Transcript, OutputPath: input.Output.Value}
for index, item := range cfg.Pipeline.Stages {
snapshot.StageNames = append(snapshot.StageNames, reporting.StageLabel(index, item.Module, item.ID))
var secrets []string
if cfg.Reporting.Enabled || input.Debug.Value {
secrets = loaded.Runtime.ConfiguredSecrets(ctx)
}
secrets := loaded.Runtime.ConfiguredSecrets(ctx)
initial, err := renderReportBundle(snapshot, secrets...)
if err != nil {
return nil, err
}
failed := func(primary error, failures []diagnostics.FileFailure) diagnostics.ReportBundle {
failedSnapshot := snapshot
completed := time.Now().UTC()
failedSnapshot.CompletedAt = &completed
if primary != nil {
failedSnapshot.Errors = []reporting.ErrorRecord{{Scope: "run", Phase: "initialization", Reason: "canceled", Message: "Processing was canceled."}}
var initial diagnostics.ReportBundle
var failed func(error, []diagnostics.FileFailure) diagnostics.ReportBundle
if cfg.Reporting.Enabled {
snapshot := reporting.Snapshot{RunID: runID, Status: reporting.StatusInProgress, StartedAt: started, TranscriptPath: input.Transcript, OutputPath: input.Output.Value}
for index, item := range cfg.Pipeline.Stages {
snapshot.StageNames = append(snapshot.StageNames, reporting.StageLabel(index, item.Module, item.ID))
}
initial, err = renderReportBundle(snapshot, secrets...)
if err != nil {
return nil, err
}
failed = func(primary error, failures []diagnostics.FileFailure) diagnostics.ReportBundle {
failedSnapshot := snapshot
completed := time.Now().UTC()
failedSnapshot.CompletedAt = &completed
if primary != nil {
failedSnapshot.Errors = []reporting.ErrorRecord{{Scope: "run", Phase: "initialization", Reason: "canceled", Message: "Processing was canceled."}}
}
bundle, _ := renderReportBundle(failedOutputSnapshot(failedSnapshot, failures), secrets...)
return bundle
}
bundle, _ := renderReportBundle(failedOutputSnapshot(failedSnapshot, failures), secrets...)
return bundle
}
var debug *preflight.DebugInput
var warnings []error

5
internal/cli/process_ordered_integration_test.go
View File

@@ -73,7 +73,6 @@ func TestProcessingUsesPerEntryProfilesInOrder(t *testing.T) {
dir := t.TempDir()
files := map[string]string{
"config.yml": `version: 3
reporting: {enabled: false}
glossary_collections:
party:
entries: [{id: alice, name: Alice, category: person}]
@@ -117,6 +116,7 @@ pipeline:
}
inputs := &processInputs{Configuration: loaded, Runtime: runtime, Source: source}
loaded.Config.Diagnostics.WorkDir = dir
loaded.Config.Reporting.StdoutDir = dir
prepared, err := prepareProcess(context.Background(), processInput{Transcript: filepath.Join(dir, "transcript.json"), Debug: commandkit.Supplied[bool]{Value: true}}, inputs, &bytes.Buffer{})
if err != nil {
t.Fatal(err)
@@ -159,7 +159,7 @@ pipeline:
func TestRecoverableModelFailureKeepsNormalizedTranscript(t *testing.T) {
dir := t.TempDir()
files := map[string]string{
"config.yml": "version: 3\nreporting: {enabled: false}\nllm: {proposal_profile: local, transient_retries: 0}\npipeline:\n stages: [{module: grammar, id: failing}]\n",
"config.yml": "version: 3\nllm: {proposal_profile: local, transient_retries: 0}\npipeline:\n stages: [{module: grammar, id: failing}]\n",
"prompts.yml": "profiles:\n - {id: local, endpoint: 'https://example.invalid/v1', model: fixture, api_key_required: false}\n",
"transcript.json": `[{"speaker":"Alice","start":0,"end":1,"text":"hello"}]`,
}
@@ -184,6 +184,7 @@ func TestRecoverableModelFailureKeepsNormalizedTranscript(t *testing.T) {
t.Fatal(err)
}
inputs := &processInputs{Configuration: loaded, Runtime: runtime, Source: source}
loaded.Config.Reporting.StdoutDir = dir
prepared, err := prepareProcess(context.Background(), processInput{Transcript: filepath.Join(dir, "transcript.json")}, inputs, &bytes.Buffer{})
if err != nil {
t.Fatal(err)

34
internal/cli/process_output.go
View File

@@ -31,31 +31,39 @@ func finishProcess(inv commandkit.Invocation, input processInput, loaded *proces
if primary != nil {
snapshot = failedProcessingSnapshot(snapshot, primary, inv.Context.Err())
}
reports, err := renderReportBundle(snapshot, secrets...)
if err != nil {
if primary == nil {
primary = err
reportPaths := prepared.Prepared.Output.ReportPaths()
var reports diagnostics.ReportBundle
var failed func([]diagnostics.FileFailure) diagnostics.ReportBundle
if len(reportPaths) > 0 {
var err error
reports, err = renderReportBundle(snapshot, secrets...)
if err != nil {
if primary == nil {
primary = err
}
snapshot.Status = reporting.StatusFailed
}
snapshot.Status = reporting.StatusFailed
}
failed := func(failures []diagnostics.FileFailure) diagnostics.ReportBundle {
result, bundleErr := renderReportBundle(failedOutputSnapshot(snapshot, failures), secrets...)
if bundleErr != nil {
return reports
failed = func(failures []diagnostics.FileFailure) diagnostics.ReportBundle {
result, bundleErr := renderReportBundle(failedOutputSnapshot(snapshot, failures), secrets...)
if bundleErr != nil {
return reports
}
return result
}
return result
}
if canceled := inv.Context.Err(); primary == nil && canceled != nil {
primary = canceled
snapshot = failedProcessingSnapshot(snapshot, canceled, canceled)
reports = failed(nil)
if failed != nil {
reports = failed(nil)
}
}
result := prepared.Prepared.Output.Finish(primary, transcript, reports, failed)
if result != nil {
return commandkit.Failure(safe.String(result.Error())).WithCause(result)
}
_ = reporter.AtEnd().Info(reporting.CompletionLine(snapshot))
for _, path := range prepared.Prepared.Output.ReportPaths() {
for _, path := range reportPaths {
_ = reporter.AtEnd().Info(fmt.Sprintf("Report: %s", safe.String(path)))
}
if prepared.Prepared.Debug != nil {

17
internal/cli/process_pipeline.go
View File

@@ -20,14 +20,20 @@ import (
func executePipeline(ctx context.Context, loaded *processInputs, prepared *preparedProcess, normalized *schema.Transcript, warn func(error)) (processResult, error) {
cfg := &loaded.Configuration.Config
specs := contracts.ResolveModuleRunSpecs(cfg)
observer := llm.NewObservationCollector()
debug := prepared.Prepared.Debug
var observer *llm.ObservationCollector
var hooks llm.CallHooks
if debug != nil {
observer = llm.NewObservationCollector()
hooks = observer.Hooks()
}
for i := range specs {
item := &specs[i]
proposal, err := loaded.Runtime.NewRoleClient("proposal", *item.Settings.LLM.ProposalProfile, item.Settings.LLM, observer.Hooks())
proposal, err := loaded.Runtime.NewRoleClient("proposal", *item.Settings.LLM.ProposalProfile, item.Settings.LLM, hooks)
if err != nil {
return processResult{}, fmt.Errorf("%s proposal client: %w", item.InstanceName, err)
}
validation, err := loaded.Runtime.NewRoleClient("validation", *item.Settings.LLM.ValidationProfile, item.Settings.LLM, observer.Hooks())
validation, err := loaded.Runtime.NewRoleClient("validation", *item.Settings.LLM.ValidationProfile, item.Settings.LLM, hooks)
if err != nil {
return processResult{}, fmt.Errorf("%s validation client: %w", item.InstanceName, err)
}
@@ -39,7 +45,7 @@ func executePipeline(ctx context.Context, loaded *processInputs, prepared *prepa
ProposalDiagnosticsDir: debugPath, ValidationDiagnosticsDir: debugPath,
CaptureCandidates: debugPath != "", ReportDiagnosticError: warn,
})
if debug := prepared.Prepared.Debug; debug != nil {
if debug != nil {
warn(debug.WriteJSONArtifact(diagnostics.ArtifactLLMExecution, processreport.BuildLLMExecution(observer.Snapshot())))
warn(debug.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{RunDirectoryPath: debug.Path(), RunOutput: &output})))
}
@@ -61,6 +67,7 @@ func executePipeline(ctx context.Context, loaded *processInputs, prepared *prepa
}
failures = append(failures, failure)
}
snapshot := processreport.Build(processreport.BuildInput{RunID: prepared.RunID, Status: reporting.StatusInProgress, StartedAt: prepared.Started, ModuleSpecs: specs, RunOutput: &output, RunFailures: failures})
snapshot := processreport.Build(processreport.BuildInput{RunID: prepared.RunID, Status: reporting.StatusInProgress, StartedAt: prepared.Started,
ModuleSpecs: specs, RunOutput: &output, RunFailures: failures, SummaryOnly: !cfg.Reporting.Enabled})
return processResult{Transcript: output.FinalTranscript, Snapshot: snapshot}, runErr
}