Report live correction progress through CommandKit
This commit is contained in:
@@ -239,6 +239,11 @@ func TestSubprocessCorrectionRepairConversation(t *testing.T) {
|
||||
t.Fatal("replacement not used")
|
||||
}
|
||||
if tc.debug {
|
||||
for _, activity := range []string{"correction repair turn", "validation", "provider attempts"} {
|
||||
if !strings.Contains(stderr, activity) {
|
||||
t.Fatalf("missing repair progress %q: %s", activity, stderr)
|
||||
}
|
||||
}
|
||||
_, ledger := f.debugArtifacts(t)
|
||||
if len(ledger) != 3 {
|
||||
t.Fatalf("ledger: %+v", ledger)
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -1474,3 +1476,112 @@ func TestSubprocessProfileSources(t *testing.T) {
|
||||
t.Fatalf("consumed message file not protected as report output: %d %s", code, stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessProgressReportingModes(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
flags []string
|
||||
visible bool
|
||||
file bool
|
||||
}{
|
||||
{name: "ordinary"},
|
||||
{name: "explicit", flags: []string{"--progress"}, visible: true},
|
||||
{name: "debug", flags: []string{"--debug"}, visible: true},
|
||||
{name: "debug with progress off", flags: []string{"--debug", "--progress=false"}},
|
||||
{name: "quiet", flags: []string{"--quiet", "--progress"}, file: true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
f := newProcessFixture(t, "normal", "[grammar]")
|
||||
code, stdout, stderr := f.run(t, tc.file, tc.flags...)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit %d: %s", code, stderr)
|
||||
}
|
||||
for _, text := range []string{"Stage 1: grammar", "chunk 1", "proposal", "validation", "provider attempts"} {
|
||||
if strings.Contains(stderr, text) != tc.visible {
|
||||
t.Fatalf("progress fragment %q, expected visibility %v: %s", text, tc.visible, stderr)
|
||||
}
|
||||
}
|
||||
if tc.file {
|
||||
if stdout != "" {
|
||||
t.Fatalf("unexpected stdout with file output: %q", stdout)
|
||||
}
|
||||
} else if !json.Valid([]byte(stdout)) || !strings.Contains(stdout, "Hello world") {
|
||||
t.Fatalf("stdout is not the transcript: %q", stdout)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessEmitsProgressBeforeProviderReturns(t *testing.T) {
|
||||
f := newProcessFixture(t, "wait-cancel", "[grammar]")
|
||||
ctx, stop := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer stop()
|
||||
cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestProcessSubprocessHelper$", "--",
|
||||
"process", f.transcript, "--config", f.config, "--output", f.output, "--progress")
|
||||
cmd.Env = append(os.Environ(), "AUDITA_HELPER_PROCESS=1", "AUDITA_TEST_PROVIDER_KEY="+f.provider.secret)
|
||||
stderr, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := cmd.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
stop()
|
||||
_ = cmd.Wait()
|
||||
}()
|
||||
progress := make(chan string, 1)
|
||||
go func() {
|
||||
scanner := bufio.NewScanner(stderr)
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
if strings.Contains(line, "proposal") && strings.Contains(line, "chunk 1") {
|
||||
select {
|
||||
case progress <- line:
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
select {
|
||||
case <-f.provider.entered:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("provider was not reached")
|
||||
}
|
||||
select {
|
||||
case line := <-progress:
|
||||
if !strings.Contains(line, "grammar") || !strings.Contains(line, "started") {
|
||||
t.Fatalf("progress omitted the running operation: %q", line)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
t.Fatal("proposal progress did not reach stderr while the provider was blocked")
|
||||
}
|
||||
if err := cmd.Process.Signal(os.Interrupt); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := cmd.Wait(); err == nil || ctx.Err() != nil {
|
||||
t.Fatalf("interrupt did not complete the invocation: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
type unavailableDiagnostics struct{}
|
||||
|
||||
func (unavailableDiagnostics) Write([]byte) (int, error) {
|
||||
return 0, errors.New("progress destination failed")
|
||||
}
|
||||
|
||||
func TestProgressDeliveryFailureKeepsCompletedProcessing(t *testing.T) {
|
||||
f := newProcessFixture(t, "normal", "[grammar]")
|
||||
t.Setenv("AUDITA_TEST_PROVIDER_KEY", f.provider.secret)
|
||||
code := cli.Run(context.Background(), []string{"process", f.transcript, "--config", f.config, "--output", f.output, "--progress"}, io.Discard, unavailableDiagnostics{})
|
||||
if code != 1 {
|
||||
t.Fatalf("expected CommandKit presentation failure, got %d", code)
|
||||
}
|
||||
output, err := os.ReadFile(f.output)
|
||||
if err != nil || !bytes.Contains(output, []byte("Hello world")) {
|
||||
t.Fatalf("transcript did not finish: %s, %v", output, err)
|
||||
}
|
||||
if doc := f.errors(t); doc.Status != reporting.StatusCompleted {
|
||||
t.Fatalf("presentation changed report status: %+v", doc)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user