package cli import ( "bytes" "context" "encoding/json" "errors" "io" "os" "path/filepath" "strings" "sync" "testing" "go.maximumdirect.net/commandkit" "go.maximumdirect.net/configkit" ) func decodeCompletionReport(t *testing.T, format commandkit.Format, content []byte) map[string]any { t.Helper() if format == commandkit.FormatYAML { value, err := commandkit.DecodeYAMLReport(bytes.NewReader(content)) if err != nil { t.Fatalf("decode native YAML report: %v\n%s", err, content) } content, err = json.Marshal(value) if err != nil { t.Fatal(err) } } var report map[string]any if err := json.Unmarshal(content, &report); err != nil { t.Fatalf("decode native report: %v\n%s", err, content) } return report } func assertCompletedRunFiles(t *testing.T, file string, captured bool) { t.Helper() run := filepath.Join(filepath.Dir(file), "output-must-not-exist", "run-1") if content, err := os.ReadFile(filepath.Join(run, "lanes", "notes.json")); err != nil || len(content) == 0 { t.Fatalf("published domain artifact: %s %v", content, err) } files, err := os.ReadDir(run) if err != nil || len(files) != 1 || files[0].Name() != "lanes" { t.Fatalf("unexpected reporting sidecars: %#v %v", files, err) } if captured { debug := filepath.Join(filepath.Dir(file), "debug-must-not-exist", "run-1") content, err := os.ReadFile(filepath.Join(debug, "outcome.json")) if err != nil { t.Fatal(err) } var terminal struct { Outcome struct { Published bool `json:"published"` } `json:"outcome"` Error string `json:"error"` CaptureError string `json:"capture_error"` } if err := json.Unmarshal(content, &terminal); err != nil || !terminal.Outcome.Published || terminal.Error != "" || terminal.CaptureError != "" { t.Fatalf("capture was not completed before reporting: %s %v", content, err) } } } type completedRunWriter struct { t *testing.T file string captured bool content bytes.Buffer } func (writer *completedRunWriter) Write(content []byte) (int, error) { assertCompletedRunFiles(writer.t, writer.file, writer.captured) return writer.content.Write(content) } func TestNativeRunCompletionFormatsFollowPublicationAndCapture(t *testing.T) { for _, format := range []commandkit.Format{commandkit.FormatText, commandkit.FormatJSON, commandkit.FormatYAML} { t.Run(string(format), func(t *testing.T) { probe := &runProbe{} app, file, input, _ := runFixture(t, probe) output := &completedRunWriter{t: t, file: file, captured: true} var diagnostics bytes.Buffer outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--format", string(format)}, commandkit.Streams{Out: output, Err: &diagnostics}) if outcome.Status != 0 || !strings.Contains(diagnostics.String(), "Running pipeline") { t.Fatalf("completion: %#v stderr=%s", outcome, diagnostics.String()) } directory := filepath.Join(filepath.Dir(file), "output-must-not-exist", "run-1") debug := filepath.Join(filepath.Dir(file), "debug-must-not-exist", "run-1") if format == commandkit.FormatText { if !strings.Contains(output.content.String(), "Success: demo produced 1 artifacts in "+directory) || !strings.Contains(output.content.String(), "Debug capture: "+debug) || strings.Contains(output.content.String(), "fallback") { t.Fatalf("native completion text: %s", output.content.String()) } } else { report := decodeCompletionReport(t, format, output.content.Bytes()) if len(report) != 4 || report["pipeline"] != "demo" || report["artifact_count"] != float64(1) || report["output_directory"] != directory || report["debug_directory"] != debug { t.Fatalf("native report: %#v", report) } } if probe.produced != 1 { t.Fatalf("reporting reran a producer: %d", probe.produced) } }) } } func TestNativeRunDisclosesFallbackIndependentlyOfDiagnosticVisibility(t *testing.T) { for _, format := range []commandkit.Format{commandkit.FormatText, commandkit.FormatJSON, commandkit.FormatYAML} { for _, verbose := range []bool{false, true} { probe := &runProbe{correctFirst: true} app, file, input, _ := runFixture(t, probe) args := []string{"pipeline", "run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--set", `/pipelines/demo/mode="permissive"`, "--set", "/pipelines/demo/max_additional_turns=0", "--format", string(format)} if verbose { args = append(args, "--verbose") } var output, diagnostics bytes.Buffer outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: &diagnostics}) if outcome.Status != 0 || probe.produced != 1 { t.Fatalf("permissive completion: %#v %s", outcome, diagnostics.String()) } if verbose && !strings.Contains(diagnostics.String(), "WARN:") || !verbose && diagnostics.Len() != 0 { t.Fatalf("native warning visibility: %s", diagnostics.String()) } if format == commandkit.FormatText { if !strings.Contains(output.String(), "Success with fallback:") || !strings.Contains(output.String(), "1 operations used fallback.") { t.Fatalf("fallback text: %s", output.String()) } } else { report := decodeCompletionReport(t, format, output.Bytes()) if report["fallback_count"] != float64(1) || report["artifact_count"] != float64(1) || len(report) != 4 { t.Fatalf("fallback report: %#v", report) } } assertCompletedRunFiles(t, file, false) } } } func TestNativeRunQuietAndDebugUseNativeModeOrdering(t *testing.T) { for _, test := range []struct { flags []string quiet bool progress bool }{ {[]string{"--debug", "--quiet"}, true, false}, {[]string{"--quiet", "--debug"}, false, true}, {[]string{"--debug", "--progress=false"}, false, false}, } { probe := &runProbe{} app, file, input, _ := runFixture(t, probe) var output, diagnostics bytes.Buffer args := append([]string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--format", "json"}, test.flags...) outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: &diagnostics}) if outcome.Status != 0 || test.quiet != (output.Len() == 0) || test.progress != strings.Contains(diagnostics.String(), "Running pipeline") { t.Fatalf("native mode order: %#v output=%s stderr=%s", outcome, output.String(), diagnostics.String()) } assertCompletedRunFiles(t, file, true) } } func TestNativeRunCancellationKeepsItsCauseAndPublishesNothing(t *testing.T) { probe := &runProbe{} app, file, input, _ := runFixture(t, probe) ctx, cancel := context.WithCancel(context.Background()) defer cancel() probe.onGenerate = cancel var output, diagnostics bytes.Buffer outcome := app.Execute(ctx, []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--quiet"}, commandkit.Streams{Out: &output, Err: &diagnostics}) if outcome.Status != 1 || !errors.Is(outcome.Cause, context.Canceled) || output.Len() != 0 || diagnostics.Len() == 0 { t.Fatalf("cancellation: %#v stdout=%s stderr=%s", outcome, output.String(), diagnostics.String()) } if _, err := os.Stat(filepath.Join(filepath.Dir(file), "output-must-not-exist")); !os.IsNotExist(err) { t.Fatalf("cancellation published artifacts: %v", err) } } func TestNativeRunPrimaryFailureSurvivesCaptureAndDiagnosticFailures(t *testing.T) { primary := errors.New("provider failure sentinel") delivery := errors.New("diagnostic failure sentinel") probe := &runProbe{providerErr: primary} app, file, input, _ := runFixture(t, probe) debug := filepath.Join(filepath.Dir(file), "debug-must-not-exist", "run-1") probe.onGenerate = func() { if err := os.Mkdir(filepath.Join(debug, "outcome.json"), 0700); err != nil { t.Error(err) } } var output bytes.Buffer outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--llm-transient-retries=0"}, commandkit.Streams{Out: &output, Err: failingResultWriter{cause: delivery}}) var captureFailure *os.LinkError if outcome.Status != 1 || !errors.Is(outcome.Cause, primary) || !errors.As(outcome.Cause, &captureFailure) || errors.Is(outcome.Cause, delivery) || !errors.Is(outcome.Presentation.Diagnostics.WriteError, delivery) || output.Len() != 0 || len(probe.requests) != 1 { t.Fatalf("primary/capture/delivery failure: %#v", outcome) } if !strings.Contains(outcome.Explanation.Message, debug) { t.Fatalf("failed run capture location unavailable: %#v", outcome) } } func TestNativeRunReportDeliveryFailuresRetainArtifactsAndNativePresentation(t *testing.T) { sentinel := errors.New("result delivery sentinel") for _, partial := range []bool{false, true} { probe := &runProbe{} app, file, input, _ := runFixture(t, probe) var output io.Writer = failingResultWriter{cause: sentinel} prefix := &partialResultWriter{cause: sentinel} if partial { output = prefix } var diagnostics bytes.Buffer outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--format", "json"}, commandkit.Streams{Out: output, Err: &diagnostics}) if outcome.Status != 1 || outcome.Cause != nil || !errors.Is(outcome.Presentation.Output.WriteError, sentinel) || outcome.Presentation.Output.State != commandkit.DeliveryIncomplete || probe.produced != 1 || partial && prefix.content.Len() == 0 { t.Fatalf("delivery failure: %#v %s", outcome, diagnostics.String()) } assertCompletedRunFiles(t, file, true) for _, root := range []string{"output-must-not-exist", "debug-must-not-exist"} { if !strings.Contains(diagnostics.String(), filepath.Join(filepath.Dir(file), root, "run-1")) { t.Fatalf("missing available path after delivery failure: %s", diagnostics.String()) } } } } func TestNativeRunDiagnosticDeliveryFailureIsNotAnOperationFailure(t *testing.T) { sentinel := errors.New("diagnostic delivery sentinel") probe := &runProbe{correctFirst: true} app, file, input, _ := runFixture(t, probe) var output bytes.Buffer outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--set", `/pipelines/demo/mode="permissive"`, "--set", "/pipelines/demo/max_additional_turns=0", "--format", "json", "--verbose"}, commandkit.Streams{Out: &output, Err: failingResultWriter{cause: sentinel}}) if outcome.Status != 1 || outcome.Cause != nil || !errors.Is(outcome.Presentation.Diagnostics.WriteError, sentinel) || outcome.Presentation.Diagnostics.State != commandkit.DeliveryIncomplete { t.Fatalf("diagnostic delivery: %#v", outcome) } if report := decodeCompletionReport(t, commandkit.FormatJSON, output.Bytes()); report["fallback_count"] != float64(1) { t.Fatalf("completed report was replaced by a delivery envelope: %#v", report) } if !strings.Contains(outcome.Explanation.Message, filepath.Join(filepath.Dir(file), "output-must-not-exist", "run-1")) { t.Fatalf("presentation failure lost available artifacts: %#v", outcome) } assertCompletedRunFiles(t, file, false) } func TestNativeRunCaptureFailureAfterPublicationDoesNotReportSuccess(t *testing.T) { probe := &runProbe{} app, file, input, _ := runFixture(t, probe) var once sync.Once debug := filepath.Join(filepath.Dir(file), "debug-must-not-exist", "run-1") probe.onGenerate = func() { once.Do(func() { // The external provider boundary runs after capture allocation but // before terminalization. Make only the terminal file unusable. if err := os.Mkdir(filepath.Join(debug, "outcome.json"), 0700); err != nil { t.Fatal(err) } }) } var output, diagnostics bytes.Buffer outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--format", "json"}, commandkit.Streams{Out: &output, Err: &diagnostics}) if outcome.Status != 1 || outcome.Cause == nil || output.Len() != 0 || probe.produced != 1 { t.Fatalf("post-publication capture failure: %#v stdout=%s stderr=%s", outcome, output.String(), diagnostics.String()) } assertCompletedRunFiles(t, file, false) if !strings.Contains(diagnostics.String(), debug) || !strings.Contains(diagnostics.String(), filepath.Join(filepath.Dir(file), "output-must-not-exist", "run-1")) { t.Fatalf("post-publication locations: %s", diagnostics.String()) } if _, err := os.Stat(filepath.Join(debug, "executions.json")); err != nil { t.Fatalf("existing capture was removed: %v", err) } } func TestNativeRunFailureCategoriesPreserveCausesWithoutSuccessReports(t *testing.T) { for _, test := range []struct { name string configure func(*testing.T, *runProbe, string, string) []string status int cause error }{ {"missing configuration", func(t *testing.T, _ *runProbe, _, input string) []string { return []string{"run", "demo", "--input-file", input, "--config", filepath.Join(t.TempDir(), "missing.yml")} }, 1, nil}, {"invalid configuration", func(t *testing.T, _ *runProbe, file, input string) []string { return []string{"run", "demo", "--config", file, "--input-file", input, "--set", "/version=5"} }, 2, nil}, {"unknown pipeline", func(_ *testing.T, _ *runProbe, file, input string) []string { return []string{"run", "missing", "--config", file, "--input-file", input} }, 2, nil}, {"missing input", func(t *testing.T, _ *runProbe, file, _ string) []string { return []string{"run", "demo", "--config", file, "--input-file", filepath.Join(t.TempDir(), "missing.txt")} }, 1, os.ErrNotExist}, {"publication failure", func(t *testing.T, _ *runProbe, file, input string) []string { path := filepath.Join(t.TempDir(), "occupied") if err := os.WriteFile(path, []byte("keep"), 0600); err != nil { t.Fatal(err) } return []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--output-dir", path} }, 1, nil}, {"provider failure", func(_ *testing.T, probe *runProbe, file, input string) []string { probe.providerErr = io.ErrUnexpectedEOF return []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--llm-transient-retries=0"} }, 1, io.ErrUnexpectedEOF}, } { t.Run(test.name, func(t *testing.T) { probe := &runProbe{} app, file, input, _ := runFixture(t, probe) var output, diagnostics bytes.Buffer args := append(test.configure(t, probe, file, input), "--quiet", "--format", "json") outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: &diagnostics}) if outcome.Status != test.status || test.cause != nil && !errors.Is(outcome.Cause, test.cause) || output.Len() != 0 || diagnostics.Len() == 0 { t.Fatalf("failure category: %#v stdout=%s stderr=%s", outcome, output.String(), diagnostics.String()) } if test.name == "missing configuration" || test.name == "invalid configuration" { var load *configkit.LoadError if !errors.As(outcome.Cause, &load) { t.Fatalf("lost native load cause: %#v", outcome) } } }) } }