Report live correction progress through CommandKit

This commit is contained in:
2026-11-16 15:12:06 +00:00
parent 891d4aac2c
commit 11f314d995
14 changed files with 289 additions and 11 deletions

View File

@@ -80,7 +80,7 @@ func TestMaintainedProjectsWithControlledProvider(t *testing.T) {
t.Fatal(err)
}
defer prepared.Prepared.Output.Close()
if _, err := executePipeline(context.Background(), inputs, prepared, source.Canonical(), func(error) {}); err != nil {
if _, err := executePipeline(context.Background(), inputs, prepared, source.Canonical(), func(error) {}, nil); err != nil {
t.Fatal(err)
}
if int(calls.Load()) != len(loaded.Config.Pipeline.Stages) {

View File

@@ -74,7 +74,10 @@ func runProcess(inv commandkit.Invocation, input processInput) commandkit.Outcom
if err := inv.Context.Err(); err != nil {
return finishProcess(inv, input, prepared, reporting.Snapshot{}, nil, &runner.ExecutionFailure{Phase: "normalization", Cause: err})
}
result, runErr := executePipeline(inv.Context, loaded, prepared, normalized, warn)
reporter := inv.Reporter()
result, runErr := executePipeline(inv.Context, loaded, prepared, normalized, warn, func(message string) {
_ = reporter.Progress(message)
})
if runErr != nil {
return finishProcess(inv, input, prepared, result.Snapshot, nil, runErr)
}

View File

@@ -122,7 +122,7 @@ pipeline:
t.Fatal(err)
}
defer prepared.Prepared.Output.Close()
result, err := executePipeline(context.Background(), inputs, prepared, source.Canonical(), func(error) {})
result, err := executePipeline(context.Background(), inputs, prepared, source.Canonical(), func(error) {}, nil)
if err != nil {
t.Fatal(err)
}
@@ -194,7 +194,7 @@ func TestRecoverableModelFailureKeepsNormalizedTranscript(t *testing.T) {
t.Fatal(err)
}
defer prepared.Prepared.Output.Close()
result, err := executePipeline(context.Background(), inputs, prepared, source.Canonical(), func(error) {})
result, err := executePipeline(context.Background(), inputs, prepared, source.Canonical(), func(error) {}, nil)
if err != nil {
t.Fatal(err)
}

View File

@@ -156,7 +156,7 @@ func TestCanceledEmptyPipelineLeavesTranscriptAndFinalizesReports(t *testing.T)
t.Fatal(err)
}
defer prepared.Prepared.Output.Close()
result, err := executePipeline(inv.Context, loaded, prepared, loaded.Source.Canonical(), func(error) {})
result, err := executePipeline(inv.Context, loaded, prepared, loaded.Source.Canonical(), func(error) {}, nil)
if err != nil {
t.Fatal(err)
}

View File

@@ -13,9 +13,10 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
"gitea.maximumdirect.net/eric/audita/v3/internal/modules"
pk "go.maximumdirect.net/promptkit"
)
func executePipeline(ctx context.Context, loaded *processInputs, prepared *preparedProcess, normalized *schema.Transcript, warn func(error)) (processResult, error) {
func executePipeline(ctx context.Context, loaded *processInputs, prepared *preparedProcess, normalized *schema.Transcript, warn func(error), progress func(string)) (processResult, error) {
cfg := &loaded.Configuration.Config
specs := prepared.Prepared.Specs
debug := prepared.Prepared.Debug
@@ -26,6 +27,15 @@ func executePipeline(ctx context.Context, loaded *processInputs, prepared *prepa
observer = llm.NewObservationCollector()
hooks = observer.Hooks()
}
if progress != nil {
observe := hooks.OnEventWithCall
hooks.OnEventWithCall = func(callID uint64, coordinates structuredoutput.DiagnosticContext, event pk.ExecutionEvent) {
if observe != nil {
observe(callID, coordinates, event)
}
reportModelProgress(progress, coordinates, event)
}
}
for i := range specs {
item := &specs[i]
setupFailure := func(role string, cause error) error {
@@ -58,7 +68,7 @@ func executePipeline(ctx context.Context, loaded *processInputs, prepared *prepa
}
output, runErr := runner.New(modules.NewFactory()).Run(ctx, runner.RunInput{
Config: cfg, Transcript: normalized, ModuleSpecs: specs,
Interactions: interactions, Evidence: &evidence,
Interactions: interactions, Evidence: &evidence, Progress: runnerProgress(progress),
})
if debug != nil {
for _, warning := range writer.Warnings() {

View File

@@ -0,0 +1,65 @@
package cli
import (
"fmt"
"strings"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
pk "go.maximumdirect.net/promptkit"
)
func runnerProgress(progress func(string)) func(runner.Progress) {
if progress == nil {
return nil
}
return func(event runner.Progress) {
location := progressLocation(event.ModuleIndex, event.ModuleKey, event.ModuleID, event.SectionIndex)
message := location + " " + event.State
if event.Count != nil {
unit := "proposals"
if event.SectionIndex == nil {
unit = "corrections"
if event.State == "started" {
unit = "chunks"
}
}
message += fmt.Sprintf(" (%d %s)", *event.Count, unit)
}
progress(message)
}
}
func reportModelProgress(progress func(string), coordinates structuredoutput.DiagnosticContext, event pk.ExecutionEvent) {
if coordinates.StageIndex == nil || (event.Kind != pk.ExecutionStarted && event.Kind != pk.ExecutionFinished) {
return
}
location := progressLocation(*coordinates.StageIndex, coordinates.ModuleKey, coordinates.StageID, coordinates.SectionIndex)
operation := strings.ReplaceAll(coordinates.Role, "_", " ")
if coordinates.ValidatorName != "" {
operation += " " + coordinates.ValidatorName
}
if coordinates.BatchIndex != nil {
operation += fmt.Sprintf(" batch %d", *coordinates.BatchIndex+1)
}
if coordinates.RepairAttempt != nil {
if coordinates.Role != "correction_repair" {
operation += " repair"
}
operation += fmt.Sprintf(" turn %d", *coordinates.RepairAttempt)
}
if event.Kind == pk.ExecutionStarted {
progress(fmt.Sprintf("%s %s started", location, operation))
} else {
progress(fmt.Sprintf("%s %s finished (%s, %d provider attempts)", location, operation, event.Outcome, event.Attempts))
}
}
func progressLocation(index int, module string, id *string, section *int) string {
location := "Stage " + reporting.StageLabel(index, module, id)
if section != nil {
location += fmt.Sprintf(", chunk %d", *section+1)
}
return location
}

View File

@@ -0,0 +1,18 @@
package runner
// Progress identifies work admitted or completed by the runner.
type Progress struct {
ModuleIndex int
ModuleKey string
ModuleID *string
SectionIndex *int
State string
Count *int
}
func (event Progress) emit(callback func(Progress), state string, section *int, count *int) {
if callback != nil {
event.State, event.SectionIndex, event.Count = state, section, count
callback(event)
}
}

View File

@@ -119,6 +119,7 @@ type RunInput struct {
ModuleSpecs []contracts.ModuleRunSpec
Interactions structuredoutput.InteractionSink
Evidence *EvidenceSelection
Progress func(Progress)
}
// RunOutput is the deterministic runner output.
@@ -175,11 +176,14 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
}
for moduleIndex, spec := range input.ModuleSpecs {
progress := Progress{ModuleIndex: spec.Index, ModuleKey: spec.ModuleKey, ModuleID: spec.ID}
sectionCount := len(sections)
progress.emit(input.Progress, "started", nil, &sectionCount)
var reportUnits []WorkUnitOutcome
if evidence.reports {
reportUnits = workUnits[moduleIndex*len(sections) : (moduleIndex+1)*len(sections)]
}
execution, err := r.executeModule(ctx, spec, sections, input.Interactions, evidence, reportUnits)
execution, err := r.executeModule(ctx, spec, sections, input.Interactions, evidence, reportUnits, input.Progress)
units := execution.pipeline.WorkUnits
if err == nil {
filters := findApplyFiltering(sections, execution.pipeline.Approved, execution.skipped, execution.policy)
@@ -252,8 +256,11 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
}
results = append(results, result)
if err != nil {
progress.emit(input.Progress, "failed", nil, nil)
return snapshot(), fmt.Errorf("module %q failed: %w", spec.InstanceName, err)
}
corrections := len(execution.applied)
progress.emit(input.Progress, "completed", nil, &corrections)
}
if err := ctx.Err(); err != nil {
@@ -271,7 +278,7 @@ type moduleExecution struct {
appliedBySection []int
}
func (r *Runner) executeModule(ctx context.Context, spec contracts.ModuleRunSpec, sections []chunking.Section, interactions structuredoutput.InteractionSink, evidence evidenceNeeds, reportUnits []WorkUnitOutcome) (moduleExecution, error) {
func (r *Runner) executeModule(ctx context.Context, spec contracts.ModuleRunSpec, sections []chunking.Section, interactions structuredoutput.InteractionSink, evidence evidenceNeeds, reportUnits []WorkUnitOutcome, progress func(Progress)) (moduleExecution, error) {
var execution moduleExecution
origin := failureReference(spec, nil, "")
if err := ctx.Err(); err != nil {
@@ -289,6 +296,7 @@ func (r *Runner) executeModule(ctx context.Context, spec contracts.ModuleRunSpec
Module: module,
Evidence: &evidence,
ReportUnits: reportUnits,
Progress: progress,
Spec: spec,
Policy: execution.policy,
Sections: sections,
@@ -359,6 +367,7 @@ func applyWithinSections(sections []chunking.Section, approved []proposals.Enric
}
type collectSectionProposalsInput struct {
Progress func(Progress)
Evidence *evidenceNeeds
ReportUnits []WorkUnitOutcome
Interactions structuredoutput.InteractionSink
@@ -421,6 +430,8 @@ func proposeSection(ctx context.Context, input collectSectionProposalsInput, sec
if ctx.Err() != nil {
return result
}
progress := Progress{ModuleIndex: input.Spec.Index, ModuleKey: input.Spec.ModuleKey, ModuleID: input.Spec.ID}
progress.emit(input.Progress, "started", &section.Index, nil)
meta := contracts.SectionMetadataFromSection(section)
proposalResult, err := input.Module.Propose(ctx, contracts.ProposalRequest{
ExecutionContext: contracts.ExecutionContext{
@@ -615,6 +626,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
}
}
progress := Progress{ModuleIndex: input.Spec.Index, ModuleKey: input.Spec.ModuleKey, ModuleID: input.Spec.ID}
nextAdmission, active := 0, 0
for {
for work := pending[nextSectionToRecord]; work != nil && work.finished; work = pending[nextSectionToRecord] {
@@ -672,11 +684,20 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
}
}
work.finished = true
if result.started {
progress.emit(input.Progress, "interrupted", &work.unit.Section.Index, nil)
}
case validated := <-validationResults:
active--
work := pending[validated.sectionPos]
work.validation = &validated
work.finished = true
state := "completed"
if validated.failure != nil {
state = "interrupted"
}
proposals := len(work.candidates)
progress.emit(input.Progress, state, &work.unit.Section.Index, &proposals)
}
}
workers.Wait()