Report live correction progress through CommandKit

This commit is contained in:
2026-11-16 15:12:06 +00:00
parent 891d4aac2c
commit 11f314d995
14 changed files with 289 additions and 11 deletions

View File

@@ -80,7 +80,7 @@ func TestMaintainedProjectsWithControlledProvider(t *testing.T) {
t.Fatal(err)
}
defer prepared.Prepared.Output.Close()
if _, err := executePipeline(context.Background(), inputs, prepared, source.Canonical(), func(error) {}); err != nil {
if _, err := executePipeline(context.Background(), inputs, prepared, source.Canonical(), func(error) {}, nil); err != nil {
t.Fatal(err)
}
if int(calls.Load()) != len(loaded.Config.Pipeline.Stages) {

View File

@@ -74,7 +74,10 @@ func runProcess(inv commandkit.Invocation, input processInput) commandkit.Outcom
if err := inv.Context.Err(); err != nil {
return finishProcess(inv, input, prepared, reporting.Snapshot{}, nil, &runner.ExecutionFailure{Phase: "normalization", Cause: err})
}
result, runErr := executePipeline(inv.Context, loaded, prepared, normalized, warn)
reporter := inv.Reporter()
result, runErr := executePipeline(inv.Context, loaded, prepared, normalized, warn, func(message string) {
_ = reporter.Progress(message)
})
if runErr != nil {
return finishProcess(inv, input, prepared, result.Snapshot, nil, runErr)
}

View File

@@ -122,7 +122,7 @@ pipeline:
t.Fatal(err)
}
defer prepared.Prepared.Output.Close()
result, err := executePipeline(context.Background(), inputs, prepared, source.Canonical(), func(error) {})
result, err := executePipeline(context.Background(), inputs, prepared, source.Canonical(), func(error) {}, nil)
if err != nil {
t.Fatal(err)
}
@@ -194,7 +194,7 @@ func TestRecoverableModelFailureKeepsNormalizedTranscript(t *testing.T) {
t.Fatal(err)
}
defer prepared.Prepared.Output.Close()
result, err := executePipeline(context.Background(), inputs, prepared, source.Canonical(), func(error) {})
result, err := executePipeline(context.Background(), inputs, prepared, source.Canonical(), func(error) {}, nil)
if err != nil {
t.Fatal(err)
}

View File

@@ -156,7 +156,7 @@ func TestCanceledEmptyPipelineLeavesTranscriptAndFinalizesReports(t *testing.T)
t.Fatal(err)
}
defer prepared.Prepared.Output.Close()
result, err := executePipeline(inv.Context, loaded, prepared, loaded.Source.Canonical(), func(error) {})
result, err := executePipeline(inv.Context, loaded, prepared, loaded.Source.Canonical(), func(error) {}, nil)
if err != nil {
t.Fatal(err)
}

View File

@@ -13,9 +13,10 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
"gitea.maximumdirect.net/eric/audita/v3/internal/modules"
pk "go.maximumdirect.net/promptkit"
)
func executePipeline(ctx context.Context, loaded *processInputs, prepared *preparedProcess, normalized *schema.Transcript, warn func(error)) (processResult, error) {
func executePipeline(ctx context.Context, loaded *processInputs, prepared *preparedProcess, normalized *schema.Transcript, warn func(error), progress func(string)) (processResult, error) {
cfg := &loaded.Configuration.Config
specs := prepared.Prepared.Specs
debug := prepared.Prepared.Debug
@@ -26,6 +27,15 @@ func executePipeline(ctx context.Context, loaded *processInputs, prepared *prepa
observer = llm.NewObservationCollector()
hooks = observer.Hooks()
}
if progress != nil {
observe := hooks.OnEventWithCall
hooks.OnEventWithCall = func(callID uint64, coordinates structuredoutput.DiagnosticContext, event pk.ExecutionEvent) {
if observe != nil {
observe(callID, coordinates, event)
}
reportModelProgress(progress, coordinates, event)
}
}
for i := range specs {
item := &specs[i]
setupFailure := func(role string, cause error) error {
@@ -58,7 +68,7 @@ func executePipeline(ctx context.Context, loaded *processInputs, prepared *prepa
}
output, runErr := runner.New(modules.NewFactory()).Run(ctx, runner.RunInput{
Config: cfg, Transcript: normalized, ModuleSpecs: specs,
Interactions: interactions, Evidence: &evidence,
Interactions: interactions, Evidence: &evidence, Progress: runnerProgress(progress),
})
if debug != nil {
for _, warning := range writer.Warnings() {

View File

@@ -0,0 +1,65 @@
package cli
import (
"fmt"
"strings"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
pk "go.maximumdirect.net/promptkit"
)
func runnerProgress(progress func(string)) func(runner.Progress) {
if progress == nil {
return nil
}
return func(event runner.Progress) {
location := progressLocation(event.ModuleIndex, event.ModuleKey, event.ModuleID, event.SectionIndex)
message := location + " " + event.State
if event.Count != nil {
unit := "proposals"
if event.SectionIndex == nil {
unit = "corrections"
if event.State == "started" {
unit = "chunks"
}
}
message += fmt.Sprintf(" (%d %s)", *event.Count, unit)
}
progress(message)
}
}
func reportModelProgress(progress func(string), coordinates structuredoutput.DiagnosticContext, event pk.ExecutionEvent) {
if coordinates.StageIndex == nil || (event.Kind != pk.ExecutionStarted && event.Kind != pk.ExecutionFinished) {
return
}
location := progressLocation(*coordinates.StageIndex, coordinates.ModuleKey, coordinates.StageID, coordinates.SectionIndex)
operation := strings.ReplaceAll(coordinates.Role, "_", " ")
if coordinates.ValidatorName != "" {
operation += " " + coordinates.ValidatorName
}
if coordinates.BatchIndex != nil {
operation += fmt.Sprintf(" batch %d", *coordinates.BatchIndex+1)
}
if coordinates.RepairAttempt != nil {
if coordinates.Role != "correction_repair" {
operation += " repair"
}
operation += fmt.Sprintf(" turn %d", *coordinates.RepairAttempt)
}
if event.Kind == pk.ExecutionStarted {
progress(fmt.Sprintf("%s %s started", location, operation))
} else {
progress(fmt.Sprintf("%s %s finished (%s, %d provider attempts)", location, operation, event.Outcome, event.Attempts))
}
}
func progressLocation(index int, module string, id *string, section *int) string {
location := "Stage " + reporting.StageLabel(index, module, id)
if section != nil {
location += fmt.Sprintf(", chunk %d", *section+1)
}
return location
}