Report live correction progress through CommandKit
This commit is contained in:
@@ -80,7 +80,7 @@ func TestMaintainedProjectsWithControlledProvider(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer prepared.Prepared.Output.Close()
|
||||
if _, err := executePipeline(context.Background(), inputs, prepared, source.Canonical(), func(error) {}); err != nil {
|
||||
if _, err := executePipeline(context.Background(), inputs, prepared, source.Canonical(), func(error) {}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if int(calls.Load()) != len(loaded.Config.Pipeline.Stages) {
|
||||
|
||||
@@ -74,7 +74,10 @@ func runProcess(inv commandkit.Invocation, input processInput) commandkit.Outcom
|
||||
if err := inv.Context.Err(); err != nil {
|
||||
return finishProcess(inv, input, prepared, reporting.Snapshot{}, nil, &runner.ExecutionFailure{Phase: "normalization", Cause: err})
|
||||
}
|
||||
result, runErr := executePipeline(inv.Context, loaded, prepared, normalized, warn)
|
||||
reporter := inv.Reporter()
|
||||
result, runErr := executePipeline(inv.Context, loaded, prepared, normalized, warn, func(message string) {
|
||||
_ = reporter.Progress(message)
|
||||
})
|
||||
if runErr != nil {
|
||||
return finishProcess(inv, input, prepared, result.Snapshot, nil, runErr)
|
||||
}
|
||||
|
||||
@@ -122,7 +122,7 @@ pipeline:
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer prepared.Prepared.Output.Close()
|
||||
result, err := executePipeline(context.Background(), inputs, prepared, source.Canonical(), func(error) {})
|
||||
result, err := executePipeline(context.Background(), inputs, prepared, source.Canonical(), func(error) {}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -194,7 +194,7 @@ func TestRecoverableModelFailureKeepsNormalizedTranscript(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer prepared.Prepared.Output.Close()
|
||||
result, err := executePipeline(context.Background(), inputs, prepared, source.Canonical(), func(error) {})
|
||||
result, err := executePipeline(context.Background(), inputs, prepared, source.Canonical(), func(error) {}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -156,7 +156,7 @@ func TestCanceledEmptyPipelineLeavesTranscriptAndFinalizesReports(t *testing.T)
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer prepared.Prepared.Output.Close()
|
||||
result, err := executePipeline(inv.Context, loaded, prepared, loaded.Source.Canonical(), func(error) {})
|
||||
result, err := executePipeline(inv.Context, loaded, prepared, loaded.Source.Canonical(), func(error) {}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -13,9 +13,10 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/modules"
|
||||
pk "go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func executePipeline(ctx context.Context, loaded *processInputs, prepared *preparedProcess, normalized *schema.Transcript, warn func(error)) (processResult, error) {
|
||||
func executePipeline(ctx context.Context, loaded *processInputs, prepared *preparedProcess, normalized *schema.Transcript, warn func(error), progress func(string)) (processResult, error) {
|
||||
cfg := &loaded.Configuration.Config
|
||||
specs := prepared.Prepared.Specs
|
||||
debug := prepared.Prepared.Debug
|
||||
@@ -26,6 +27,15 @@ func executePipeline(ctx context.Context, loaded *processInputs, prepared *prepa
|
||||
observer = llm.NewObservationCollector()
|
||||
hooks = observer.Hooks()
|
||||
}
|
||||
if progress != nil {
|
||||
observe := hooks.OnEventWithCall
|
||||
hooks.OnEventWithCall = func(callID uint64, coordinates structuredoutput.DiagnosticContext, event pk.ExecutionEvent) {
|
||||
if observe != nil {
|
||||
observe(callID, coordinates, event)
|
||||
}
|
||||
reportModelProgress(progress, coordinates, event)
|
||||
}
|
||||
}
|
||||
for i := range specs {
|
||||
item := &specs[i]
|
||||
setupFailure := func(role string, cause error) error {
|
||||
@@ -58,7 +68,7 @@ func executePipeline(ctx context.Context, loaded *processInputs, prepared *prepa
|
||||
}
|
||||
output, runErr := runner.New(modules.NewFactory()).Run(ctx, runner.RunInput{
|
||||
Config: cfg, Transcript: normalized, ModuleSpecs: specs,
|
||||
Interactions: interactions, Evidence: &evidence,
|
||||
Interactions: interactions, Evidence: &evidence, Progress: runnerProgress(progress),
|
||||
})
|
||||
if debug != nil {
|
||||
for _, warning := range writer.Warnings() {
|
||||
|
||||
65
internal/cli/process_progress.go
Normal file
65
internal/cli/process_progress.go
Normal file
@@ -0,0 +1,65 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||
pk "go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func runnerProgress(progress func(string)) func(runner.Progress) {
|
||||
if progress == nil {
|
||||
return nil
|
||||
}
|
||||
return func(event runner.Progress) {
|
||||
location := progressLocation(event.ModuleIndex, event.ModuleKey, event.ModuleID, event.SectionIndex)
|
||||
message := location + " " + event.State
|
||||
if event.Count != nil {
|
||||
unit := "proposals"
|
||||
if event.SectionIndex == nil {
|
||||
unit = "corrections"
|
||||
if event.State == "started" {
|
||||
unit = "chunks"
|
||||
}
|
||||
}
|
||||
message += fmt.Sprintf(" (%d %s)", *event.Count, unit)
|
||||
}
|
||||
progress(message)
|
||||
}
|
||||
}
|
||||
|
||||
func reportModelProgress(progress func(string), coordinates structuredoutput.DiagnosticContext, event pk.ExecutionEvent) {
|
||||
if coordinates.StageIndex == nil || (event.Kind != pk.ExecutionStarted && event.Kind != pk.ExecutionFinished) {
|
||||
return
|
||||
}
|
||||
location := progressLocation(*coordinates.StageIndex, coordinates.ModuleKey, coordinates.StageID, coordinates.SectionIndex)
|
||||
operation := strings.ReplaceAll(coordinates.Role, "_", " ")
|
||||
if coordinates.ValidatorName != "" {
|
||||
operation += " " + coordinates.ValidatorName
|
||||
}
|
||||
if coordinates.BatchIndex != nil {
|
||||
operation += fmt.Sprintf(" batch %d", *coordinates.BatchIndex+1)
|
||||
}
|
||||
if coordinates.RepairAttempt != nil {
|
||||
if coordinates.Role != "correction_repair" {
|
||||
operation += " repair"
|
||||
}
|
||||
operation += fmt.Sprintf(" turn %d", *coordinates.RepairAttempt)
|
||||
}
|
||||
if event.Kind == pk.ExecutionStarted {
|
||||
progress(fmt.Sprintf("%s %s started", location, operation))
|
||||
} else {
|
||||
progress(fmt.Sprintf("%s %s finished (%s, %d provider attempts)", location, operation, event.Outcome, event.Attempts))
|
||||
}
|
||||
}
|
||||
|
||||
func progressLocation(index int, module string, id *string, section *int) string {
|
||||
location := "Stage " + reporting.StageLabel(index, module, id)
|
||||
if section != nil {
|
||||
location += fmt.Sprintf(", chunk %d", *section+1)
|
||||
}
|
||||
return location
|
||||
}
|
||||
Reference in New Issue
Block a user