Report live correction progress through CommandKit
This commit is contained in:
@@ -80,7 +80,7 @@ func TestMaintainedProjectsWithControlledProvider(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer prepared.Prepared.Output.Close()
|
||||
if _, err := executePipeline(context.Background(), inputs, prepared, source.Canonical(), func(error) {}); err != nil {
|
||||
if _, err := executePipeline(context.Background(), inputs, prepared, source.Canonical(), func(error) {}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if int(calls.Load()) != len(loaded.Config.Pipeline.Stages) {
|
||||
|
||||
@@ -74,7 +74,10 @@ func runProcess(inv commandkit.Invocation, input processInput) commandkit.Outcom
|
||||
if err := inv.Context.Err(); err != nil {
|
||||
return finishProcess(inv, input, prepared, reporting.Snapshot{}, nil, &runner.ExecutionFailure{Phase: "normalization", Cause: err})
|
||||
}
|
||||
result, runErr := executePipeline(inv.Context, loaded, prepared, normalized, warn)
|
||||
reporter := inv.Reporter()
|
||||
result, runErr := executePipeline(inv.Context, loaded, prepared, normalized, warn, func(message string) {
|
||||
_ = reporter.Progress(message)
|
||||
})
|
||||
if runErr != nil {
|
||||
return finishProcess(inv, input, prepared, result.Snapshot, nil, runErr)
|
||||
}
|
||||
|
||||
@@ -122,7 +122,7 @@ pipeline:
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer prepared.Prepared.Output.Close()
|
||||
result, err := executePipeline(context.Background(), inputs, prepared, source.Canonical(), func(error) {})
|
||||
result, err := executePipeline(context.Background(), inputs, prepared, source.Canonical(), func(error) {}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -194,7 +194,7 @@ func TestRecoverableModelFailureKeepsNormalizedTranscript(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer prepared.Prepared.Output.Close()
|
||||
result, err := executePipeline(context.Background(), inputs, prepared, source.Canonical(), func(error) {})
|
||||
result, err := executePipeline(context.Background(), inputs, prepared, source.Canonical(), func(error) {}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -156,7 +156,7 @@ func TestCanceledEmptyPipelineLeavesTranscriptAndFinalizesReports(t *testing.T)
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer prepared.Prepared.Output.Close()
|
||||
result, err := executePipeline(inv.Context, loaded, prepared, loaded.Source.Canonical(), func(error) {})
|
||||
result, err := executePipeline(inv.Context, loaded, prepared, loaded.Source.Canonical(), func(error) {}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -13,9 +13,10 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/modules"
|
||||
pk "go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func executePipeline(ctx context.Context, loaded *processInputs, prepared *preparedProcess, normalized *schema.Transcript, warn func(error)) (processResult, error) {
|
||||
func executePipeline(ctx context.Context, loaded *processInputs, prepared *preparedProcess, normalized *schema.Transcript, warn func(error), progress func(string)) (processResult, error) {
|
||||
cfg := &loaded.Configuration.Config
|
||||
specs := prepared.Prepared.Specs
|
||||
debug := prepared.Prepared.Debug
|
||||
@@ -26,6 +27,15 @@ func executePipeline(ctx context.Context, loaded *processInputs, prepared *prepa
|
||||
observer = llm.NewObservationCollector()
|
||||
hooks = observer.Hooks()
|
||||
}
|
||||
if progress != nil {
|
||||
observe := hooks.OnEventWithCall
|
||||
hooks.OnEventWithCall = func(callID uint64, coordinates structuredoutput.DiagnosticContext, event pk.ExecutionEvent) {
|
||||
if observe != nil {
|
||||
observe(callID, coordinates, event)
|
||||
}
|
||||
reportModelProgress(progress, coordinates, event)
|
||||
}
|
||||
}
|
||||
for i := range specs {
|
||||
item := &specs[i]
|
||||
setupFailure := func(role string, cause error) error {
|
||||
@@ -58,7 +68,7 @@ func executePipeline(ctx context.Context, loaded *processInputs, prepared *prepa
|
||||
}
|
||||
output, runErr := runner.New(modules.NewFactory()).Run(ctx, runner.RunInput{
|
||||
Config: cfg, Transcript: normalized, ModuleSpecs: specs,
|
||||
Interactions: interactions, Evidence: &evidence,
|
||||
Interactions: interactions, Evidence: &evidence, Progress: runnerProgress(progress),
|
||||
})
|
||||
if debug != nil {
|
||||
for _, warning := range writer.Warnings() {
|
||||
|
||||
65
internal/cli/process_progress.go
Normal file
65
internal/cli/process_progress.go
Normal file
@@ -0,0 +1,65 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||
pk "go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func runnerProgress(progress func(string)) func(runner.Progress) {
|
||||
if progress == nil {
|
||||
return nil
|
||||
}
|
||||
return func(event runner.Progress) {
|
||||
location := progressLocation(event.ModuleIndex, event.ModuleKey, event.ModuleID, event.SectionIndex)
|
||||
message := location + " " + event.State
|
||||
if event.Count != nil {
|
||||
unit := "proposals"
|
||||
if event.SectionIndex == nil {
|
||||
unit = "corrections"
|
||||
if event.State == "started" {
|
||||
unit = "chunks"
|
||||
}
|
||||
}
|
||||
message += fmt.Sprintf(" (%d %s)", *event.Count, unit)
|
||||
}
|
||||
progress(message)
|
||||
}
|
||||
}
|
||||
|
||||
func reportModelProgress(progress func(string), coordinates structuredoutput.DiagnosticContext, event pk.ExecutionEvent) {
|
||||
if coordinates.StageIndex == nil || (event.Kind != pk.ExecutionStarted && event.Kind != pk.ExecutionFinished) {
|
||||
return
|
||||
}
|
||||
location := progressLocation(*coordinates.StageIndex, coordinates.ModuleKey, coordinates.StageID, coordinates.SectionIndex)
|
||||
operation := strings.ReplaceAll(coordinates.Role, "_", " ")
|
||||
if coordinates.ValidatorName != "" {
|
||||
operation += " " + coordinates.ValidatorName
|
||||
}
|
||||
if coordinates.BatchIndex != nil {
|
||||
operation += fmt.Sprintf(" batch %d", *coordinates.BatchIndex+1)
|
||||
}
|
||||
if coordinates.RepairAttempt != nil {
|
||||
if coordinates.Role != "correction_repair" {
|
||||
operation += " repair"
|
||||
}
|
||||
operation += fmt.Sprintf(" turn %d", *coordinates.RepairAttempt)
|
||||
}
|
||||
if event.Kind == pk.ExecutionStarted {
|
||||
progress(fmt.Sprintf("%s %s started", location, operation))
|
||||
} else {
|
||||
progress(fmt.Sprintf("%s %s finished (%s, %d provider attempts)", location, operation, event.Outcome, event.Attempts))
|
||||
}
|
||||
}
|
||||
|
||||
func progressLocation(index int, module string, id *string, section *int) string {
|
||||
location := "Stage " + reporting.StageLabel(index, module, id)
|
||||
if section != nil {
|
||||
location += fmt.Sprintf(", chunk %d", *section+1)
|
||||
}
|
||||
return location
|
||||
}
|
||||
18
internal/framework/runner/progress.go
Normal file
18
internal/framework/runner/progress.go
Normal file
@@ -0,0 +1,18 @@
|
||||
package runner
|
||||
|
||||
// Progress identifies work admitted or completed by the runner.
|
||||
type Progress struct {
|
||||
ModuleIndex int
|
||||
ModuleKey string
|
||||
ModuleID *string
|
||||
SectionIndex *int
|
||||
State string
|
||||
Count *int
|
||||
}
|
||||
|
||||
func (event Progress) emit(callback func(Progress), state string, section *int, count *int) {
|
||||
if callback != nil {
|
||||
event.State, event.SectionIndex, event.Count = state, section, count
|
||||
callback(event)
|
||||
}
|
||||
}
|
||||
@@ -119,6 +119,7 @@ type RunInput struct {
|
||||
ModuleSpecs []contracts.ModuleRunSpec
|
||||
Interactions structuredoutput.InteractionSink
|
||||
Evidence *EvidenceSelection
|
||||
Progress func(Progress)
|
||||
}
|
||||
|
||||
// RunOutput is the deterministic runner output.
|
||||
@@ -175,11 +176,14 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
|
||||
}
|
||||
|
||||
for moduleIndex, spec := range input.ModuleSpecs {
|
||||
progress := Progress{ModuleIndex: spec.Index, ModuleKey: spec.ModuleKey, ModuleID: spec.ID}
|
||||
sectionCount := len(sections)
|
||||
progress.emit(input.Progress, "started", nil, §ionCount)
|
||||
var reportUnits []WorkUnitOutcome
|
||||
if evidence.reports {
|
||||
reportUnits = workUnits[moduleIndex*len(sections) : (moduleIndex+1)*len(sections)]
|
||||
}
|
||||
execution, err := r.executeModule(ctx, spec, sections, input.Interactions, evidence, reportUnits)
|
||||
execution, err := r.executeModule(ctx, spec, sections, input.Interactions, evidence, reportUnits, input.Progress)
|
||||
units := execution.pipeline.WorkUnits
|
||||
if err == nil {
|
||||
filters := findApplyFiltering(sections, execution.pipeline.Approved, execution.skipped, execution.policy)
|
||||
@@ -252,8 +256,11 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
|
||||
}
|
||||
results = append(results, result)
|
||||
if err != nil {
|
||||
progress.emit(input.Progress, "failed", nil, nil)
|
||||
return snapshot(), fmt.Errorf("module %q failed: %w", spec.InstanceName, err)
|
||||
}
|
||||
corrections := len(execution.applied)
|
||||
progress.emit(input.Progress, "completed", nil, &corrections)
|
||||
}
|
||||
|
||||
if err := ctx.Err(); err != nil {
|
||||
@@ -271,7 +278,7 @@ type moduleExecution struct {
|
||||
appliedBySection []int
|
||||
}
|
||||
|
||||
func (r *Runner) executeModule(ctx context.Context, spec contracts.ModuleRunSpec, sections []chunking.Section, interactions structuredoutput.InteractionSink, evidence evidenceNeeds, reportUnits []WorkUnitOutcome) (moduleExecution, error) {
|
||||
func (r *Runner) executeModule(ctx context.Context, spec contracts.ModuleRunSpec, sections []chunking.Section, interactions structuredoutput.InteractionSink, evidence evidenceNeeds, reportUnits []WorkUnitOutcome, progress func(Progress)) (moduleExecution, error) {
|
||||
var execution moduleExecution
|
||||
origin := failureReference(spec, nil, "")
|
||||
if err := ctx.Err(); err != nil {
|
||||
@@ -289,6 +296,7 @@ func (r *Runner) executeModule(ctx context.Context, spec contracts.ModuleRunSpec
|
||||
Module: module,
|
||||
Evidence: &evidence,
|
||||
ReportUnits: reportUnits,
|
||||
Progress: progress,
|
||||
Spec: spec,
|
||||
Policy: execution.policy,
|
||||
Sections: sections,
|
||||
@@ -359,6 +367,7 @@ func applyWithinSections(sections []chunking.Section, approved []proposals.Enric
|
||||
}
|
||||
|
||||
type collectSectionProposalsInput struct {
|
||||
Progress func(Progress)
|
||||
Evidence *evidenceNeeds
|
||||
ReportUnits []WorkUnitOutcome
|
||||
Interactions structuredoutput.InteractionSink
|
||||
@@ -421,6 +430,8 @@ func proposeSection(ctx context.Context, input collectSectionProposalsInput, sec
|
||||
if ctx.Err() != nil {
|
||||
return result
|
||||
}
|
||||
progress := Progress{ModuleIndex: input.Spec.Index, ModuleKey: input.Spec.ModuleKey, ModuleID: input.Spec.ID}
|
||||
progress.emit(input.Progress, "started", §ion.Index, nil)
|
||||
meta := contracts.SectionMetadataFromSection(section)
|
||||
proposalResult, err := input.Module.Propose(ctx, contracts.ProposalRequest{
|
||||
ExecutionContext: contracts.ExecutionContext{
|
||||
@@ -615,6 +626,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
||||
}
|
||||
}
|
||||
|
||||
progress := Progress{ModuleIndex: input.Spec.Index, ModuleKey: input.Spec.ModuleKey, ModuleID: input.Spec.ID}
|
||||
nextAdmission, active := 0, 0
|
||||
for {
|
||||
for work := pending[nextSectionToRecord]; work != nil && work.finished; work = pending[nextSectionToRecord] {
|
||||
@@ -672,11 +684,20 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
||||
}
|
||||
}
|
||||
work.finished = true
|
||||
if result.started {
|
||||
progress.emit(input.Progress, "interrupted", &work.unit.Section.Index, nil)
|
||||
}
|
||||
case validated := <-validationResults:
|
||||
active--
|
||||
work := pending[validated.sectionPos]
|
||||
work.validation = &validated
|
||||
work.finished = true
|
||||
state := "completed"
|
||||
if validated.failure != nil {
|
||||
state = "interrupted"
|
||||
}
|
||||
proposals := len(work.candidates)
|
||||
progress.emit(input.Progress, state, &work.unit.Section.Index, &proposals)
|
||||
}
|
||||
}
|
||||
workers.Wait()
|
||||
|
||||
Reference in New Issue
Block a user