From 11f314d99572af28a46e04fd4213eb7467cf0470 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Mon, 16 Nov 2026 15:12:06 +0000 Subject: [PATCH] Report live correction progress through CommandKit --- .../correction_repair_integration_test.go | 5 + cmd/audita/main_integration_test.go | 111 ++++++++++++++++++ docs/cli.md | 8 +- docs/integrations/subprocess.md | 13 +- docs/operations.md | 20 ++++ docs/roadmap/implementation.md | 8 ++ internal/cli/examples_integration_test.go | 2 +- internal/cli/process.go | 5 +- .../cli/process_ordered_integration_test.go | 4 +- internal/cli/process_output_test.go | 2 +- internal/cli/process_pipeline.go | 14 ++- internal/cli/process_progress.go | 65 ++++++++++ internal/framework/runner/progress.go | 18 +++ internal/framework/runner/runner.go | 25 +++- 14 files changed, 289 insertions(+), 11 deletions(-) create mode 100644 internal/cli/process_progress.go create mode 100644 internal/framework/runner/progress.go diff --git a/cmd/audita/correction_repair_integration_test.go b/cmd/audita/correction_repair_integration_test.go index 8a40f3a..b23b2bb 100644 --- a/cmd/audita/correction_repair_integration_test.go +++ b/cmd/audita/correction_repair_integration_test.go @@ -239,6 +239,11 @@ func TestSubprocessCorrectionRepairConversation(t *testing.T) { t.Fatal("replacement not used") } if tc.debug { + for _, activity := range []string{"correction repair turn", "validation", "provider attempts"} { + if !strings.Contains(stderr, activity) { + t.Fatalf("missing repair progress %q: %s", activity, stderr) + } + } _, ledger := f.debugArtifacts(t) if len(ledger) != 3 { t.Fatalf("ledger: %+v", ledger) diff --git a/cmd/audita/main_integration_test.go b/cmd/audita/main_integration_test.go index e222313..7656183 100644 --- a/cmd/audita/main_integration_test.go +++ b/cmd/audita/main_integration_test.go @@ -1,9 +1,11 @@ package main import ( + "bufio" "bytes" "context" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -1474,3 +1476,112 @@ func TestSubprocessProfileSources(t *testing.T) { t.Fatalf("consumed message file not protected as report output: %d %s", code, stderr) } } + +func TestSubprocessProgressReportingModes(t *testing.T) { + for _, tc := range []struct { + name string + flags []string + visible bool + file bool + }{ + {name: "ordinary"}, + {name: "explicit", flags: []string{"--progress"}, visible: true}, + {name: "debug", flags: []string{"--debug"}, visible: true}, + {name: "debug with progress off", flags: []string{"--debug", "--progress=false"}}, + {name: "quiet", flags: []string{"--quiet", "--progress"}, file: true}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newProcessFixture(t, "normal", "[grammar]") + code, stdout, stderr := f.run(t, tc.file, tc.flags...) + if code != 0 { + t.Fatalf("exit %d: %s", code, stderr) + } + for _, text := range []string{"Stage 1: grammar", "chunk 1", "proposal", "validation", "provider attempts"} { + if strings.Contains(stderr, text) != tc.visible { + t.Fatalf("progress fragment %q, expected visibility %v: %s", text, tc.visible, stderr) + } + } + if tc.file { + if stdout != "" { + t.Fatalf("unexpected stdout with file output: %q", stdout) + } + } else if !json.Valid([]byte(stdout)) || !strings.Contains(stdout, "Hello world") { + t.Fatalf("stdout is not the transcript: %q", stdout) + } + }) + } +} + +func TestSubprocessEmitsProgressBeforeProviderReturns(t *testing.T) { + f := newProcessFixture(t, "wait-cancel", "[grammar]") + ctx, stop := context.WithTimeout(context.Background(), 5*time.Second) + defer stop() + cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestProcessSubprocessHelper$", "--", + "process", f.transcript, "--config", f.config, "--output", f.output, "--progress") + cmd.Env = append(os.Environ(), "AUDITA_HELPER_PROCESS=1", "AUDITA_TEST_PROVIDER_KEY="+f.provider.secret) + stderr, err := cmd.StderrPipe() + if err != nil { + t.Fatal(err) + } + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + defer func() { + stop() + _ = cmd.Wait() + }() + progress := make(chan string, 1) + go func() { + scanner := bufio.NewScanner(stderr) + for scanner.Scan() { + line := scanner.Text() + if strings.Contains(line, "proposal") && strings.Contains(line, "chunk 1") { + select { + case progress <- line: + default: + } + } + } + }() + select { + case <-f.provider.entered: + case <-ctx.Done(): + t.Fatal("provider was not reached") + } + select { + case line := <-progress: + if !strings.Contains(line, "grammar") || !strings.Contains(line, "started") { + t.Fatalf("progress omitted the running operation: %q", line) + } + case <-ctx.Done(): + t.Fatal("proposal progress did not reach stderr while the provider was blocked") + } + if err := cmd.Process.Signal(os.Interrupt); err != nil { + t.Fatal(err) + } + if err := cmd.Wait(); err == nil || ctx.Err() != nil { + t.Fatalf("interrupt did not complete the invocation: %v", err) + } +} + +type unavailableDiagnostics struct{} + +func (unavailableDiagnostics) Write([]byte) (int, error) { + return 0, errors.New("progress destination failed") +} + +func TestProgressDeliveryFailureKeepsCompletedProcessing(t *testing.T) { + f := newProcessFixture(t, "normal", "[grammar]") + t.Setenv("AUDITA_TEST_PROVIDER_KEY", f.provider.secret) + code := cli.Run(context.Background(), []string{"process", f.transcript, "--config", f.config, "--output", f.output, "--progress"}, io.Discard, unavailableDiagnostics{}) + if code != 1 { + t.Fatalf("expected CommandKit presentation failure, got %d", code) + } + output, err := os.ReadFile(f.output) + if err != nil || !bytes.Contains(output, []byte("Hello world")) { + t.Fatalf("transcript did not finish: %s, %v", output, err) + } + if doc := f.errors(t); doc.Status != reporting.StatusCompleted { + t.Fatalf("presentation changed report status: %+v", doc) + } +} diff --git a/docs/cli.md b/docs/cli.md index 165157c..11f5154 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -20,6 +20,12 @@ audita process [flags] transcript.json messages. Quiet processing requires an output file or `/dev/null`. See [operations](operations.md) for output and report lifecycle. +Progress for processing goes to stderr. Use `--progress` to show stage and +chunk activity, model proposal, validation and repair operations, and observed +counts as work completes. Debug mode enables progress by default. +`--progress=false` disables it, and quiet mode suppresses it even when +`--progress` is explicit. + Explicit process flags override root settings only. `--glossaries` and `--protected-terms` each accept one JSON array of collection names or `null`; `[]` clears the selection. Set pipeline order and per-entry overrides in YAML. @@ -35,7 +41,7 @@ The application configuration scope uses ConfigKit's native effective values, source evidence, explanation, comparison, and redaction. PromptKit supplies its own configuration and resource scopes and request preparation, validation, and explicit smoke diagnostics. Inspection does not require a processing-ready -pipeline or contact a provider. Smoke tests can contact providers and are run +pipeline or contact a provider. Processing preflight separately checks the selected\nprofiles and glossary entries. Smoke tests can contact providers and are run only on explicit request. Process stdout contains transcript JSON when no output file is selected. Ordinary diff --git a/docs/integrations/subprocess.md b/docs/integrations/subprocess.md index bfc3518..09795f2 100644 --- a/docs/integrations/subprocess.md +++ b/docs/integrations/subprocess.md @@ -18,10 +18,21 @@ not redirect file-output reports. `--work-dir` selects a debug root, while Supplying a configuration path also makes file selection predictable; only explicitly referenced credential variables are read from the environment. +Choose a trusted configuration bundle. An explicit main path still allows +its configured imports and automatic overlay, glossary, and prompt companions. +Without a main path, Audita uses the configured discovery locations. The +parent controls who can write those sources and which report destinations and +retained debug directories the invocation will use. Native inspection is +separate from processing preflight. + Resolve the executable deliberately and set the child's working directory so relative input/output paths and reported artifact paths have a known base. Read stdout and stderr concurrently to avoid pipe blocking, and impose timeout -or cancellation in the parent process when needed. +or cancellation in the parent process when needed. Optional `--progress` +messages arrive on stderr during stage and chunk execution, including model +operations and completions. Debug enables progress by default, and +`--progress=false` or quiet mode suppresses it. Consume progress as +human-readable telemetry. Stdout remains the transcript stream. ## Result Handling diff --git a/docs/operations.md b/docs/operations.md index ceae8f7..3a908ce 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -17,6 +17,17 @@ history to the selected repair provider. A provider rejection ends that branch as unavailable; other approved edits can still be applied. See [configuration](config.md#settings) for the attempt budget and profile selection. +## Configuration trust + +Treat the selected main file, imports, overlay, and automatically loaded +glossary and prompt companions as one trusted bundle. Choose the working +directory deliberately when using automatic discovery. A bundle can select +the provider, prompt content, and local files consumed by processing. Review +these sources before running with sensitive transcript data. Native inspection +describes the loaded configuration and resources. Processing readiness also +depends on the profiles and glossaries selected by the pipeline. See +[configuration loading](config.md#loading-and-composition) for discovery rules. + ## Outputs and lifecycle Reporting is enabled by default. For transcript file output, `errors.yml`, @@ -37,6 +48,8 @@ delivery failure while preserving the new destination. Parent directories for required outputs must already exist. Publication uses the destinations resolved during preparation. Give concurrent invocations separate destinations, and choose separate report directories when retaining reports from multiple runs. +The caller owns these directories and their retention. Later runs can replace +ordinary report files at the same destination. Enabled reports are initialized as `in_progress` before processing. The transcript is published before final reports; the files do not form a single @@ -49,6 +62,13 @@ the reader. If failed reports cannot be rendered, available report evidence remains. An uncatchable termination can leave `in_progress` evidence. Use the exit status and report status to determine the outcome. +## Live progress + +Use `--progress` to follow stage, chunk, proposal, validation, and repair +activity on stderr. Report and transcript files provide the retained outcome. +Debug mode enables live progress by default. Use `--progress=false` to keep +debug evidence with progress disabled. + ## Debug evidence The shared `--debug` flag enables retained artifacts in a run directory beneath diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index d7bce46..dccf160 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -700,6 +700,14 @@ asserting useful identities and stream separation without exact wording or concurrent ordering. Keep a progress-specific check separate from delegated inspection availability. +**Complete.** Runner lifecycle and native model events flow through +CommandKit progress reporting. Subprocess tests observed proposal progress +while a provider was blocked, checked repair activity, verified stdout +separation and reporting-mode combinations, and confirmed that a failed +progress destination leaves completed output intact. The CLI, operations, +and subprocess guides cover live progress, destination ownership, and +configuration trust. The full offline Go suite passed. + ## Stage 20 — Verify the integrated sprint and release bootstrap Audit #28, #29, #58, #60–62 and sprint closure. diff --git a/internal/cli/examples_integration_test.go b/internal/cli/examples_integration_test.go index 311048d..6569862 100644 --- a/internal/cli/examples_integration_test.go +++ b/internal/cli/examples_integration_test.go @@ -80,7 +80,7 @@ func TestMaintainedProjectsWithControlledProvider(t *testing.T) { t.Fatal(err) } defer prepared.Prepared.Output.Close() - if _, err := executePipeline(context.Background(), inputs, prepared, source.Canonical(), func(error) {}); err != nil { + if _, err := executePipeline(context.Background(), inputs, prepared, source.Canonical(), func(error) {}, nil); err != nil { t.Fatal(err) } if int(calls.Load()) != len(loaded.Config.Pipeline.Stages) { diff --git a/internal/cli/process.go b/internal/cli/process.go index 3c3e270..595dedb 100644 --- a/internal/cli/process.go +++ b/internal/cli/process.go @@ -74,7 +74,10 @@ func runProcess(inv commandkit.Invocation, input processInput) commandkit.Outcom if err := inv.Context.Err(); err != nil { return finishProcess(inv, input, prepared, reporting.Snapshot{}, nil, &runner.ExecutionFailure{Phase: "normalization", Cause: err}) } - result, runErr := executePipeline(inv.Context, loaded, prepared, normalized, warn) + reporter := inv.Reporter() + result, runErr := executePipeline(inv.Context, loaded, prepared, normalized, warn, func(message string) { + _ = reporter.Progress(message) + }) if runErr != nil { return finishProcess(inv, input, prepared, result.Snapshot, nil, runErr) } diff --git a/internal/cli/process_ordered_integration_test.go b/internal/cli/process_ordered_integration_test.go index 08c79ce..6575b76 100644 --- a/internal/cli/process_ordered_integration_test.go +++ b/internal/cli/process_ordered_integration_test.go @@ -122,7 +122,7 @@ pipeline: t.Fatal(err) } defer prepared.Prepared.Output.Close() - result, err := executePipeline(context.Background(), inputs, prepared, source.Canonical(), func(error) {}) + result, err := executePipeline(context.Background(), inputs, prepared, source.Canonical(), func(error) {}, nil) if err != nil { t.Fatal(err) } @@ -194,7 +194,7 @@ func TestRecoverableModelFailureKeepsNormalizedTranscript(t *testing.T) { t.Fatal(err) } defer prepared.Prepared.Output.Close() - result, err := executePipeline(context.Background(), inputs, prepared, source.Canonical(), func(error) {}) + result, err := executePipeline(context.Background(), inputs, prepared, source.Canonical(), func(error) {}, nil) if err != nil { t.Fatal(err) } diff --git a/internal/cli/process_output_test.go b/internal/cli/process_output_test.go index bd8f932..1ae0be5 100644 --- a/internal/cli/process_output_test.go +++ b/internal/cli/process_output_test.go @@ -156,7 +156,7 @@ func TestCanceledEmptyPipelineLeavesTranscriptAndFinalizesReports(t *testing.T) t.Fatal(err) } defer prepared.Prepared.Output.Close() - result, err := executePipeline(inv.Context, loaded, prepared, loaded.Source.Canonical(), func(error) {}) + result, err := executePipeline(inv.Context, loaded, prepared, loaded.Source.Canonical(), func(error) {}, nil) if err != nil { t.Fatal(err) } diff --git a/internal/cli/process_pipeline.go b/internal/cli/process_pipeline.go index c8ac138..586808c 100644 --- a/internal/cli/process_pipeline.go +++ b/internal/cli/process_pipeline.go @@ -13,9 +13,10 @@ import ( "gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput" "gitea.maximumdirect.net/eric/audita/v3/internal/modules" + pk "go.maximumdirect.net/promptkit" ) -func executePipeline(ctx context.Context, loaded *processInputs, prepared *preparedProcess, normalized *schema.Transcript, warn func(error)) (processResult, error) { +func executePipeline(ctx context.Context, loaded *processInputs, prepared *preparedProcess, normalized *schema.Transcript, warn func(error), progress func(string)) (processResult, error) { cfg := &loaded.Configuration.Config specs := prepared.Prepared.Specs debug := prepared.Prepared.Debug @@ -26,6 +27,15 @@ func executePipeline(ctx context.Context, loaded *processInputs, prepared *prepa observer = llm.NewObservationCollector() hooks = observer.Hooks() } + if progress != nil { + observe := hooks.OnEventWithCall + hooks.OnEventWithCall = func(callID uint64, coordinates structuredoutput.DiagnosticContext, event pk.ExecutionEvent) { + if observe != nil { + observe(callID, coordinates, event) + } + reportModelProgress(progress, coordinates, event) + } + } for i := range specs { item := &specs[i] setupFailure := func(role string, cause error) error { @@ -58,7 +68,7 @@ func executePipeline(ctx context.Context, loaded *processInputs, prepared *prepa } output, runErr := runner.New(modules.NewFactory()).Run(ctx, runner.RunInput{ Config: cfg, Transcript: normalized, ModuleSpecs: specs, - Interactions: interactions, Evidence: &evidence, + Interactions: interactions, Evidence: &evidence, Progress: runnerProgress(progress), }) if debug != nil { for _, warning := range writer.Warnings() { diff --git a/internal/cli/process_progress.go b/internal/cli/process_progress.go new file mode 100644 index 0000000..d8ee14c --- /dev/null +++ b/internal/cli/process_progress.go @@ -0,0 +1,65 @@ +package cli + +import ( + "fmt" + "strings" + + "gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting" + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner" + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput" + pk "go.maximumdirect.net/promptkit" +) + +func runnerProgress(progress func(string)) func(runner.Progress) { + if progress == nil { + return nil + } + return func(event runner.Progress) { + location := progressLocation(event.ModuleIndex, event.ModuleKey, event.ModuleID, event.SectionIndex) + message := location + " " + event.State + if event.Count != nil { + unit := "proposals" + if event.SectionIndex == nil { + unit = "corrections" + if event.State == "started" { + unit = "chunks" + } + } + message += fmt.Sprintf(" (%d %s)", *event.Count, unit) + } + progress(message) + } +} + +func reportModelProgress(progress func(string), coordinates structuredoutput.DiagnosticContext, event pk.ExecutionEvent) { + if coordinates.StageIndex == nil || (event.Kind != pk.ExecutionStarted && event.Kind != pk.ExecutionFinished) { + return + } + location := progressLocation(*coordinates.StageIndex, coordinates.ModuleKey, coordinates.StageID, coordinates.SectionIndex) + operation := strings.ReplaceAll(coordinates.Role, "_", " ") + if coordinates.ValidatorName != "" { + operation += " " + coordinates.ValidatorName + } + if coordinates.BatchIndex != nil { + operation += fmt.Sprintf(" batch %d", *coordinates.BatchIndex+1) + } + if coordinates.RepairAttempt != nil { + if coordinates.Role != "correction_repair" { + operation += " repair" + } + operation += fmt.Sprintf(" turn %d", *coordinates.RepairAttempt) + } + if event.Kind == pk.ExecutionStarted { + progress(fmt.Sprintf("%s %s started", location, operation)) + } else { + progress(fmt.Sprintf("%s %s finished (%s, %d provider attempts)", location, operation, event.Outcome, event.Attempts)) + } +} + +func progressLocation(index int, module string, id *string, section *int) string { + location := "Stage " + reporting.StageLabel(index, module, id) + if section != nil { + location += fmt.Sprintf(", chunk %d", *section+1) + } + return location +} diff --git a/internal/framework/runner/progress.go b/internal/framework/runner/progress.go new file mode 100644 index 0000000..4b17f96 --- /dev/null +++ b/internal/framework/runner/progress.go @@ -0,0 +1,18 @@ +package runner + +// Progress identifies work admitted or completed by the runner. +type Progress struct { + ModuleIndex int + ModuleKey string + ModuleID *string + SectionIndex *int + State string + Count *int +} + +func (event Progress) emit(callback func(Progress), state string, section *int, count *int) { + if callback != nil { + event.State, event.SectionIndex, event.Count = state, section, count + callback(event) + } +} diff --git a/internal/framework/runner/runner.go b/internal/framework/runner/runner.go index d5fb2e3..ad3f435 100644 --- a/internal/framework/runner/runner.go +++ b/internal/framework/runner/runner.go @@ -119,6 +119,7 @@ type RunInput struct { ModuleSpecs []contracts.ModuleRunSpec Interactions structuredoutput.InteractionSink Evidence *EvidenceSelection + Progress func(Progress) } // RunOutput is the deterministic runner output. @@ -175,11 +176,14 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) { } for moduleIndex, spec := range input.ModuleSpecs { + progress := Progress{ModuleIndex: spec.Index, ModuleKey: spec.ModuleKey, ModuleID: spec.ID} + sectionCount := len(sections) + progress.emit(input.Progress, "started", nil, §ionCount) var reportUnits []WorkUnitOutcome if evidence.reports { reportUnits = workUnits[moduleIndex*len(sections) : (moduleIndex+1)*len(sections)] } - execution, err := r.executeModule(ctx, spec, sections, input.Interactions, evidence, reportUnits) + execution, err := r.executeModule(ctx, spec, sections, input.Interactions, evidence, reportUnits, input.Progress) units := execution.pipeline.WorkUnits if err == nil { filters := findApplyFiltering(sections, execution.pipeline.Approved, execution.skipped, execution.policy) @@ -252,8 +256,11 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) { } results = append(results, result) if err != nil { + progress.emit(input.Progress, "failed", nil, nil) return snapshot(), fmt.Errorf("module %q failed: %w", spec.InstanceName, err) } + corrections := len(execution.applied) + progress.emit(input.Progress, "completed", nil, &corrections) } if err := ctx.Err(); err != nil { @@ -271,7 +278,7 @@ type moduleExecution struct { appliedBySection []int } -func (r *Runner) executeModule(ctx context.Context, spec contracts.ModuleRunSpec, sections []chunking.Section, interactions structuredoutput.InteractionSink, evidence evidenceNeeds, reportUnits []WorkUnitOutcome) (moduleExecution, error) { +func (r *Runner) executeModule(ctx context.Context, spec contracts.ModuleRunSpec, sections []chunking.Section, interactions structuredoutput.InteractionSink, evidence evidenceNeeds, reportUnits []WorkUnitOutcome, progress func(Progress)) (moduleExecution, error) { var execution moduleExecution origin := failureReference(spec, nil, "") if err := ctx.Err(); err != nil { @@ -289,6 +296,7 @@ func (r *Runner) executeModule(ctx context.Context, spec contracts.ModuleRunSpec Module: module, Evidence: &evidence, ReportUnits: reportUnits, + Progress: progress, Spec: spec, Policy: execution.policy, Sections: sections, @@ -359,6 +367,7 @@ func applyWithinSections(sections []chunking.Section, approved []proposals.Enric } type collectSectionProposalsInput struct { + Progress func(Progress) Evidence *evidenceNeeds ReportUnits []WorkUnitOutcome Interactions structuredoutput.InteractionSink @@ -421,6 +430,8 @@ func proposeSection(ctx context.Context, input collectSectionProposalsInput, sec if ctx.Err() != nil { return result } + progress := Progress{ModuleIndex: input.Spec.Index, ModuleKey: input.Spec.ModuleKey, ModuleID: input.Spec.ID} + progress.emit(input.Progress, "started", §ion.Index, nil) meta := contracts.SectionMetadataFromSection(section) proposalResult, err := input.Module.Propose(ctx, contracts.ProposalRequest{ ExecutionContext: contracts.ExecutionContext{ @@ -615,6 +626,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) } } + progress := Progress{ModuleIndex: input.Spec.Index, ModuleKey: input.Spec.ModuleKey, ModuleID: input.Spec.ID} nextAdmission, active := 0, 0 for { for work := pending[nextSectionToRecord]; work != nil && work.finished; work = pending[nextSectionToRecord] { @@ -672,11 +684,20 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) } } work.finished = true + if result.started { + progress.emit(input.Progress, "interrupted", &work.unit.Section.Index, nil) + } case validated := <-validationResults: active-- work := pending[validated.sectionPos] work.validation = &validated work.finished = true + state := "completed" + if validated.failure != nil { + state = "interrupted" + } + proposals := len(work.candidates) + progress.emit(input.Progress, state, &work.unit.Section.Index, &proposals) } } workers.Wait()