320 lines
15 KiB
Go
320 lines
15 KiB
Go
package cli
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
|
|
"go.maximumdirect.net/commandkit"
|
|
"go.maximumdirect.net/configkit"
|
|
)
|
|
|
|
func decodeCompletionReport(t *testing.T, format commandkit.Format, content []byte) map[string]any {
|
|
t.Helper()
|
|
if format == commandkit.FormatYAML {
|
|
value, err := commandkit.DecodeYAMLReport(bytes.NewReader(content))
|
|
if err != nil {
|
|
t.Fatalf("decode native YAML report: %v\n%s", err, content)
|
|
}
|
|
content, err = json.Marshal(value)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
var report map[string]any
|
|
if err := json.Unmarshal(content, &report); err != nil {
|
|
t.Fatalf("decode native report: %v\n%s", err, content)
|
|
}
|
|
return report
|
|
}
|
|
|
|
func assertCompletedRunFiles(t *testing.T, file string, captured bool) {
|
|
t.Helper()
|
|
run := filepath.Join(filepath.Dir(file), "output-must-not-exist", "run-1")
|
|
if content, err := os.ReadFile(filepath.Join(run, "lanes", "notes.json")); err != nil || len(content) == 0 {
|
|
t.Fatalf("published domain artifact: %s %v", content, err)
|
|
}
|
|
files, err := os.ReadDir(run)
|
|
if err != nil || len(files) != 1 || files[0].Name() != "lanes" {
|
|
t.Fatalf("unexpected reporting sidecars: %#v %v", files, err)
|
|
}
|
|
if captured {
|
|
debug := filepath.Join(filepath.Dir(file), "debug-must-not-exist", "run-1")
|
|
content, err := os.ReadFile(filepath.Join(debug, "outcome.json"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var terminal struct {
|
|
Outcome struct {
|
|
Published bool `json:"published"`
|
|
} `json:"outcome"`
|
|
Error string `json:"error"`
|
|
CaptureError string `json:"capture_error"`
|
|
}
|
|
if err := json.Unmarshal(content, &terminal); err != nil || !terminal.Outcome.Published || terminal.Error != "" || terminal.CaptureError != "" {
|
|
t.Fatalf("capture was not completed before reporting: %s %v", content, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
type completedRunWriter struct {
|
|
t *testing.T
|
|
file string
|
|
captured bool
|
|
content bytes.Buffer
|
|
}
|
|
|
|
func (writer *completedRunWriter) Write(content []byte) (int, error) {
|
|
assertCompletedRunFiles(writer.t, writer.file, writer.captured)
|
|
return writer.content.Write(content)
|
|
}
|
|
|
|
func TestNativeRunCompletionFormatsFollowPublicationAndCapture(t *testing.T) {
|
|
for _, format := range []commandkit.Format{commandkit.FormatText, commandkit.FormatJSON, commandkit.FormatYAML} {
|
|
t.Run(string(format), func(t *testing.T) {
|
|
probe := &runProbe{}
|
|
app, file, input, _ := runFixture(t, probe)
|
|
output := &completedRunWriter{t: t, file: file, captured: true}
|
|
var diagnostics bytes.Buffer
|
|
outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--format", string(format)}, commandkit.Streams{Out: output, Err: &diagnostics})
|
|
if outcome.Status != 0 || !strings.Contains(diagnostics.String(), "Running pipeline") {
|
|
t.Fatalf("completion: %#v stderr=%s", outcome, diagnostics.String())
|
|
}
|
|
directory := filepath.Join(filepath.Dir(file), "output-must-not-exist", "run-1")
|
|
debug := filepath.Join(filepath.Dir(file), "debug-must-not-exist", "run-1")
|
|
if format == commandkit.FormatText {
|
|
if !strings.Contains(output.content.String(), "Success: demo produced 1 artifacts in "+directory) || !strings.Contains(output.content.String(), "Debug capture: "+debug) || strings.Contains(output.content.String(), "fallback") {
|
|
t.Fatalf("native completion text: %s", output.content.String())
|
|
}
|
|
} else {
|
|
report := decodeCompletionReport(t, format, output.content.Bytes())
|
|
if len(report) != 4 || report["pipeline"] != "demo" || report["artifact_count"] != float64(1) || report["output_directory"] != directory || report["debug_directory"] != debug {
|
|
t.Fatalf("native report: %#v", report)
|
|
}
|
|
}
|
|
if probe.produced != 1 {
|
|
t.Fatalf("reporting reran a producer: %d", probe.produced)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestNativeRunDisclosesFallbackIndependentlyOfDiagnosticVisibility(t *testing.T) {
|
|
for _, format := range []commandkit.Format{commandkit.FormatText, commandkit.FormatJSON, commandkit.FormatYAML} {
|
|
for _, verbose := range []bool{false, true} {
|
|
probe := &runProbe{correctFirst: true}
|
|
app, file, input, _ := runFixture(t, probe)
|
|
args := []string{"pipeline", "run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--set", `/pipelines/demo/mode="permissive"`, "--set", "/pipelines/demo/max_additional_turns=0", "--format", string(format)}
|
|
if verbose {
|
|
args = append(args, "--verbose")
|
|
}
|
|
var output, diagnostics bytes.Buffer
|
|
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: &diagnostics})
|
|
if outcome.Status != 0 || probe.produced != 1 {
|
|
t.Fatalf("permissive completion: %#v %s", outcome, diagnostics.String())
|
|
}
|
|
if verbose && !strings.Contains(diagnostics.String(), "WARN:") || !verbose && diagnostics.Len() != 0 {
|
|
t.Fatalf("native warning visibility: %s", diagnostics.String())
|
|
}
|
|
if format == commandkit.FormatText {
|
|
if !strings.Contains(output.String(), "Success with fallback:") || !strings.Contains(output.String(), "1 operations used fallback.") {
|
|
t.Fatalf("fallback text: %s", output.String())
|
|
}
|
|
} else {
|
|
report := decodeCompletionReport(t, format, output.Bytes())
|
|
if report["fallback_count"] != float64(1) || report["artifact_count"] != float64(1) || len(report) != 4 {
|
|
t.Fatalf("fallback report: %#v", report)
|
|
}
|
|
}
|
|
assertCompletedRunFiles(t, file, false)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestNativeRunQuietAndDebugUseNativeModeOrdering(t *testing.T) {
|
|
for _, test := range []struct {
|
|
flags []string
|
|
quiet bool
|
|
progress bool
|
|
}{
|
|
{[]string{"--debug", "--quiet"}, true, false},
|
|
{[]string{"--quiet", "--debug"}, false, true},
|
|
{[]string{"--debug", "--progress=false"}, false, false},
|
|
} {
|
|
probe := &runProbe{}
|
|
app, file, input, _ := runFixture(t, probe)
|
|
var output, diagnostics bytes.Buffer
|
|
args := append([]string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--format", "json"}, test.flags...)
|
|
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: &diagnostics})
|
|
if outcome.Status != 0 || test.quiet != (output.Len() == 0) || test.progress != strings.Contains(diagnostics.String(), "Running pipeline") {
|
|
t.Fatalf("native mode order: %#v output=%s stderr=%s", outcome, output.String(), diagnostics.String())
|
|
}
|
|
assertCompletedRunFiles(t, file, true)
|
|
}
|
|
}
|
|
|
|
func TestNativeRunCancellationKeepsItsCauseAndPublishesNothing(t *testing.T) {
|
|
probe := &runProbe{}
|
|
app, file, input, _ := runFixture(t, probe)
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
probe.onGenerate = cancel
|
|
var output, diagnostics bytes.Buffer
|
|
outcome := app.Execute(ctx, []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--quiet"}, commandkit.Streams{Out: &output, Err: &diagnostics})
|
|
if outcome.Status != 1 || !errors.Is(outcome.Cause, context.Canceled) || output.Len() != 0 || diagnostics.Len() == 0 {
|
|
t.Fatalf("cancellation: %#v stdout=%s stderr=%s", outcome, output.String(), diagnostics.String())
|
|
}
|
|
if _, err := os.Stat(filepath.Join(filepath.Dir(file), "output-must-not-exist")); !os.IsNotExist(err) {
|
|
t.Fatalf("cancellation published artifacts: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestNativeRunPrimaryFailureSurvivesCaptureAndDiagnosticFailures(t *testing.T) {
|
|
primary := errors.New("provider failure sentinel")
|
|
delivery := errors.New("diagnostic failure sentinel")
|
|
probe := &runProbe{providerErr: primary}
|
|
app, file, input, _ := runFixture(t, probe)
|
|
debug := filepath.Join(filepath.Dir(file), "debug-must-not-exist", "run-1")
|
|
probe.onGenerate = func() {
|
|
if err := os.Mkdir(filepath.Join(debug, "outcome.json"), 0700); err != nil {
|
|
t.Error(err)
|
|
}
|
|
}
|
|
var output bytes.Buffer
|
|
outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--llm-transient-retries=0"}, commandkit.Streams{Out: &output, Err: failingResultWriter{cause: delivery}})
|
|
var captureFailure *os.LinkError
|
|
if outcome.Status != 1 || !errors.Is(outcome.Cause, primary) || !errors.As(outcome.Cause, &captureFailure) || errors.Is(outcome.Cause, delivery) || !errors.Is(outcome.Presentation.Diagnostics.WriteError, delivery) || output.Len() != 0 || len(probe.requests) != 1 {
|
|
t.Fatalf("primary/capture/delivery failure: %#v", outcome)
|
|
}
|
|
if !strings.Contains(outcome.Explanation.Message, debug) {
|
|
t.Fatalf("failed run capture location unavailable: %#v", outcome)
|
|
}
|
|
}
|
|
|
|
func TestNativeRunReportDeliveryFailuresRetainArtifactsAndNativePresentation(t *testing.T) {
|
|
sentinel := errors.New("result delivery sentinel")
|
|
for _, partial := range []bool{false, true} {
|
|
probe := &runProbe{}
|
|
app, file, input, _ := runFixture(t, probe)
|
|
var output io.Writer = failingResultWriter{cause: sentinel}
|
|
prefix := &partialResultWriter{cause: sentinel}
|
|
if partial {
|
|
output = prefix
|
|
}
|
|
var diagnostics bytes.Buffer
|
|
outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--format", "json"}, commandkit.Streams{Out: output, Err: &diagnostics})
|
|
if outcome.Status != 1 || outcome.Cause != nil || !errors.Is(outcome.Presentation.Output.WriteError, sentinel) || outcome.Presentation.Output.State != commandkit.DeliveryIncomplete || probe.produced != 1 || partial && prefix.content.Len() == 0 {
|
|
t.Fatalf("delivery failure: %#v %s", outcome, diagnostics.String())
|
|
}
|
|
assertCompletedRunFiles(t, file, true)
|
|
for _, root := range []string{"output-must-not-exist", "debug-must-not-exist"} {
|
|
if !strings.Contains(diagnostics.String(), filepath.Join(filepath.Dir(file), root, "run-1")) {
|
|
t.Fatalf("missing available path after delivery failure: %s", diagnostics.String())
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestNativeRunDiagnosticDeliveryFailureIsNotAnOperationFailure(t *testing.T) {
|
|
sentinel := errors.New("diagnostic delivery sentinel")
|
|
probe := &runProbe{correctFirst: true}
|
|
app, file, input, _ := runFixture(t, probe)
|
|
var output bytes.Buffer
|
|
outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--set", `/pipelines/demo/mode="permissive"`, "--set", "/pipelines/demo/max_additional_turns=0", "--format", "json", "--verbose"}, commandkit.Streams{Out: &output, Err: failingResultWriter{cause: sentinel}})
|
|
if outcome.Status != 1 || outcome.Cause != nil || !errors.Is(outcome.Presentation.Diagnostics.WriteError, sentinel) || outcome.Presentation.Diagnostics.State != commandkit.DeliveryIncomplete {
|
|
t.Fatalf("diagnostic delivery: %#v", outcome)
|
|
}
|
|
if report := decodeCompletionReport(t, commandkit.FormatJSON, output.Bytes()); report["fallback_count"] != float64(1) {
|
|
t.Fatalf("completed report was replaced by a delivery envelope: %#v", report)
|
|
}
|
|
if !strings.Contains(outcome.Explanation.Message, filepath.Join(filepath.Dir(file), "output-must-not-exist", "run-1")) {
|
|
t.Fatalf("presentation failure lost available artifacts: %#v", outcome)
|
|
}
|
|
assertCompletedRunFiles(t, file, false)
|
|
}
|
|
|
|
func TestNativeRunCaptureFailureAfterPublicationDoesNotReportSuccess(t *testing.T) {
|
|
probe := &runProbe{}
|
|
app, file, input, _ := runFixture(t, probe)
|
|
var once sync.Once
|
|
debug := filepath.Join(filepath.Dir(file), "debug-must-not-exist", "run-1")
|
|
probe.onGenerate = func() {
|
|
once.Do(func() {
|
|
// The external provider boundary runs after capture allocation but
|
|
// before terminalization. Make only the terminal file unusable.
|
|
if err := os.Mkdir(filepath.Join(debug, "outcome.json"), 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
})
|
|
}
|
|
var output, diagnostics bytes.Buffer
|
|
outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--format", "json"}, commandkit.Streams{Out: &output, Err: &diagnostics})
|
|
if outcome.Status != 1 || outcome.Cause == nil || output.Len() != 0 || probe.produced != 1 {
|
|
t.Fatalf("post-publication capture failure: %#v stdout=%s stderr=%s", outcome, output.String(), diagnostics.String())
|
|
}
|
|
assertCompletedRunFiles(t, file, false)
|
|
if !strings.Contains(diagnostics.String(), debug) || !strings.Contains(diagnostics.String(), filepath.Join(filepath.Dir(file), "output-must-not-exist", "run-1")) {
|
|
t.Fatalf("post-publication locations: %s", diagnostics.String())
|
|
}
|
|
if _, err := os.Stat(filepath.Join(debug, "executions.json")); err != nil {
|
|
t.Fatalf("existing capture was removed: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestNativeRunFailureCategoriesPreserveCausesWithoutSuccessReports(t *testing.T) {
|
|
for _, test := range []struct {
|
|
name string
|
|
configure func(*testing.T, *runProbe, string, string) []string
|
|
status int
|
|
cause error
|
|
}{
|
|
{"missing configuration", func(t *testing.T, _ *runProbe, _, input string) []string {
|
|
return []string{"run", "demo", "--input-file", input, "--config", filepath.Join(t.TempDir(), "missing.yml")}
|
|
}, 1, nil},
|
|
{"invalid configuration", func(t *testing.T, _ *runProbe, file, input string) []string {
|
|
return []string{"run", "demo", "--config", file, "--input-file", input, "--set", "/version=5"}
|
|
}, 2, nil},
|
|
{"unknown pipeline", func(_ *testing.T, _ *runProbe, file, input string) []string {
|
|
return []string{"run", "missing", "--config", file, "--input-file", input}
|
|
}, 2, nil},
|
|
{"missing input", func(t *testing.T, _ *runProbe, file, _ string) []string {
|
|
return []string{"run", "demo", "--config", file, "--input-file", filepath.Join(t.TempDir(), "missing.txt")}
|
|
}, 1, os.ErrNotExist},
|
|
{"publication failure", func(t *testing.T, _ *runProbe, file, input string) []string {
|
|
path := filepath.Join(t.TempDir(), "occupied")
|
|
if err := os.WriteFile(path, []byte("keep"), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--output-dir", path}
|
|
}, 1, nil},
|
|
{"provider failure", func(_ *testing.T, probe *runProbe, file, input string) []string {
|
|
probe.providerErr = io.ErrUnexpectedEOF
|
|
return []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--llm-transient-retries=0"}
|
|
}, 1, io.ErrUnexpectedEOF},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
probe := &runProbe{}
|
|
app, file, input, _ := runFixture(t, probe)
|
|
var output, diagnostics bytes.Buffer
|
|
args := append(test.configure(t, probe, file, input), "--quiet", "--format", "json")
|
|
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: &diagnostics})
|
|
if outcome.Status != test.status || test.cause != nil && !errors.Is(outcome.Cause, test.cause) || output.Len() != 0 || diagnostics.Len() == 0 {
|
|
t.Fatalf("failure category: %#v stdout=%s stderr=%s", outcome, output.String(), diagnostics.String())
|
|
}
|
|
if test.name == "missing configuration" || test.name == "invalid configuration" {
|
|
var load *configkit.LoadError
|
|
if !errors.As(outcome.Cause, &load) {
|
|
t.Fatalf("lost native load cause: %#v", outcome)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|