Report completed runs through native presentation and preserve failure outcomes
This commit is contained in:
@@ -2,198 +2,318 @@ package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
alwaysreject "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/validate/always_reject"
|
||||
"go.maximumdirect.net/commandkit"
|
||||
"go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
func TestMaintainedMinimalInvocationEmitsRunResult(t *testing.T) {
|
||||
outputRoot := filepath.Join(t.TempDir(), "output")
|
||||
var stdout, stderr strings.Builder
|
||||
code := RunWithOptions([]string{
|
||||
"run", "dnd-session",
|
||||
"--config", repositoryPath("examples", "dnd-minimal.config.yml"),
|
||||
"--input", repositoryPath("examples", "seriatim-minimal-transcript.json"),
|
||||
"--only", "spells", "--chunk_cache", "bypass", "--output-dir", outputRoot, "--json",
|
||||
}, &stdout, &stderr, productionRunOptions(t, &productionFakeLLMClient{}))
|
||||
if code != 0 || stderr.Len() != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
|
||||
receipt := decodeRunResultDocument(t, stdout.String())
|
||||
if got := receipt["schema_version"]; got != "notarius.run-result.v2" {
|
||||
t.Fatalf("schema_version = %q", got)
|
||||
}
|
||||
if got := receipt["run_id"]; got != productionRunID {
|
||||
t.Fatalf("run_id = %q", got)
|
||||
}
|
||||
if got := receipt["pipeline_id"]; got != "dnd-session" {
|
||||
t.Fatalf("pipeline_id = %q", got)
|
||||
}
|
||||
if got := receipt["index_file"]; got != "index.json" {
|
||||
t.Fatalf("index_file = %q", got)
|
||||
}
|
||||
if got := receipt["normalized_output_count"]; got != float64(1) {
|
||||
t.Fatalf("normalized_output_count = %v", got)
|
||||
}
|
||||
if got := receipt["rejected_output_count"]; got != float64(0) {
|
||||
t.Fatalf("rejected_output_count = %v", got)
|
||||
}
|
||||
if got := receipt["warning_group_count"]; got != float64(0) || receipt["warning_occurrence_count"] != float64(0) || receipt["diagnostic_group_count"] != float64(0) || receipt["diagnostic_occurrence_count"] != float64(0) || receipt["diagnostics_truncated"] != false {
|
||||
t.Fatalf("diagnostic counts = %#v", receipt)
|
||||
}
|
||||
if got := receipt["validation_status"]; got != "approved" {
|
||||
t.Fatalf("validation_status = %q", got)
|
||||
}
|
||||
|
||||
outputDirectory, ok := receipt["output_directory"].(string)
|
||||
if !ok || !filepath.IsAbs(outputDirectory) || outputDirectory != filepath.Join(outputRoot, productionRunID) {
|
||||
t.Fatalf("output_directory = %q", receipt["output_directory"])
|
||||
}
|
||||
indexFile := receipt["index_file"].(string)
|
||||
assertFile(t, filepath.Join(outputDirectory, indexFile))
|
||||
}
|
||||
|
||||
func TestRunResultReportsWarningsAndDebugBundle(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
harness := newStateTestHarness()
|
||||
harness.chunkDiagnostics = []contracts.ProducerDiagnostic{stateTestDiagnostic("chunk", "contract-warning", "warning retained")}
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{
|
||||
"run", "sample", "--config", roots.config, "--input", roots.input,
|
||||
"--chunk_cache", "bypass", "--debug", "--json",
|
||||
}, &stdout, &stderr, harness.options())
|
||||
if code != 0 || !strings.Contains(stderr.String(), "1 warning group(s), 1 occurrence(s)") || strings.Contains(stderr.String(), "warnings.json") {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
|
||||
receipt := decodeRunResultDocument(t, stdout.String())
|
||||
if got := receipt["warning_group_count"]; got != float64(1) || receipt["warning_occurrence_count"] != float64(1) || receipt["diagnostic_group_count"] != float64(0) || receipt["diagnostic_occurrence_count"] != float64(0) || receipt["diagnostics_truncated"] != false {
|
||||
t.Fatalf("diagnostic counts = %#v", receipt)
|
||||
}
|
||||
debugDirectory, ok := receipt["debug_directory"].(string)
|
||||
if !ok || !filepath.IsAbs(debugDirectory) || debugDirectory != onlyChildDir(t, roots.debug) {
|
||||
t.Fatalf("debug_directory = %q", receipt["debug_directory"])
|
||||
}
|
||||
if strings.Contains(stdout.String(), "complete:") || strings.Contains(stdout.String(), "debug=") {
|
||||
t.Fatalf("machine stdout contains human reporting: %q", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunResultReportsSuccessfulRejection(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
configBytes, err := os.ReadFile(roots.config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
configBytes = []byte(replaceRequiredOnce(t, string(configBytes), " normalize:\n module: test/normalize\n", " normalize:\n module: test/normalize\n validators:\n - module: generic/always_reject\n validation_policy:\n semantic_rejection: reject_output\n"))
|
||||
if err := os.WriteFile(roots.config, configBytes, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
harness := newStateTestHarness()
|
||||
opts := harness.options()
|
||||
if err := alwaysreject.RegisterTyped[stateTestArtifact](opts.Registries.Validators, stateTestArtifactKind); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts.Catalog = catalogFromRegistries(opts.Registries)
|
||||
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{
|
||||
"run", "sample", "--config", roots.config, "--input", roots.input,
|
||||
"--chunk_cache", "bypass", "--json",
|
||||
}, &stdout, &stderr, opts)
|
||||
if code != 0 || stderr.Len() != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
receipt := decodeRunResultDocument(t, stdout.String())
|
||||
if got := receipt["normalized_output_count"]; got != float64(0) {
|
||||
t.Fatalf("normalized_output_count = %v", got)
|
||||
}
|
||||
if got := receipt["rejected_output_count"]; got != float64(1) {
|
||||
t.Fatalf("rejected_output_count = %v", got)
|
||||
}
|
||||
if got := receipt["validation_status"]; got != "rejected" {
|
||||
t.Fatalf("validation_status = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunResultIsAbsentForSyntaxAndRuntimeFailures(t *testing.T) {
|
||||
t.Run("syntax", func(t *testing.T) {
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "sample", "--json"}, &stdout, &stderr, newStateTestHarness().options())
|
||||
if code != 2 || stdout.Len() != 0 || stderr.Len() == 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("runtime", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
harness := newStateTestHarness()
|
||||
harness.extractErr = errors.New("injected extraction failure")
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{
|
||||
"run", "sample", "--config", roots.config, "--input", roots.input,
|
||||
"--chunk_cache", "bypass", "--json",
|
||||
}, &stdout, &stderr, harness.options())
|
||||
if code != 1 || stdout.Len() != 0 || stderr.Len() == 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestRunResultDeliveryFailureRetainsPublishedBundles(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
writerErr := errors.New("result writer sentinel")
|
||||
stdout := &resultDeliveryWriter{err: writerErr}
|
||||
var stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{
|
||||
"run", "sample", "--config", roots.config, "--input", roots.input,
|
||||
"--chunk_cache", "bypass", "--debug", "--json",
|
||||
}, stdout, &stderr, newStateTestHarness().options())
|
||||
if code != 1 || !strings.Contains(stderr.String(), "write run result") || strings.Contains(stderr.String(), writerErr.Error()) {
|
||||
t.Fatalf("code=%d stderr=%q", code, stderr.String())
|
||||
}
|
||||
if stdout.accepted.Len() != 0 {
|
||||
t.Fatalf("accepted stdout = %q", stdout.accepted.String())
|
||||
}
|
||||
assertStateTestOutput(t, roots.output)
|
||||
debugBundle := onlyChildDir(t, roots.debug)
|
||||
report := readStateTestRunReport(t, debugBundle)
|
||||
if !report.Succeeded {
|
||||
t.Fatalf("debug report = %#v, want successful persisted run", report)
|
||||
}
|
||||
if strings.Contains(readAllFiles(t, debugBundle), writerErr.Error()) {
|
||||
t.Fatalf("debug bundle contains result writer error")
|
||||
}
|
||||
}
|
||||
|
||||
func decodeRunResultDocument(t *testing.T, stdout string) map[string]any {
|
||||
func decodeCompletionReport(t *testing.T, format commandkit.Format, content []byte) map[string]any {
|
||||
t.Helper()
|
||||
if strings.Count(stdout, "\n") != 1 {
|
||||
t.Fatalf("stdout = %q, want one JSON document", stdout)
|
||||
if format == commandkit.FormatYAML {
|
||||
value, err := commandkit.DecodeYAMLReport(bytes.NewReader(content))
|
||||
if err != nil {
|
||||
t.Fatalf("decode native YAML report: %v\n%s", err, content)
|
||||
}
|
||||
content, err = json.Marshal(value)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
var receipt map[string]any
|
||||
if err := json.Unmarshal([]byte(stdout), &receipt); err != nil {
|
||||
t.Fatalf("decode run result: %v; stdout=%q", err, stdout)
|
||||
var report map[string]any
|
||||
if err := json.Unmarshal(content, &report); err != nil {
|
||||
t.Fatalf("decode native report: %v\n%s", err, content)
|
||||
}
|
||||
return receipt
|
||||
return report
|
||||
}
|
||||
|
||||
type resultDeliveryWriter struct {
|
||||
err error
|
||||
accepted bytes.Buffer
|
||||
func assertCompletedRunFiles(t *testing.T, file string, captured bool) {
|
||||
t.Helper()
|
||||
run := filepath.Join(filepath.Dir(file), "output-must-not-exist", "run-1")
|
||||
if content, err := os.ReadFile(filepath.Join(run, "lanes", "notes.json")); err != nil || len(content) == 0 {
|
||||
t.Fatalf("published domain artifact: %s %v", content, err)
|
||||
}
|
||||
files, err := os.ReadDir(run)
|
||||
if err != nil || len(files) != 1 || files[0].Name() != "lanes" {
|
||||
t.Fatalf("unexpected reporting sidecars: %#v %v", files, err)
|
||||
}
|
||||
if captured {
|
||||
debug := filepath.Join(filepath.Dir(file), "debug-must-not-exist", "run-1")
|
||||
content, err := os.ReadFile(filepath.Join(debug, "outcome.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var terminal struct {
|
||||
Outcome struct {
|
||||
Published bool `json:"published"`
|
||||
} `json:"outcome"`
|
||||
Error string `json:"error"`
|
||||
CaptureError string `json:"capture_error"`
|
||||
}
|
||||
if err := json.Unmarshal(content, &terminal); err != nil || !terminal.Outcome.Published || terminal.Error != "" || terminal.CaptureError != "" {
|
||||
t.Fatalf("capture was not completed before reporting: %s %v", content, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *resultDeliveryWriter) Write(content []byte) (int, error) {
|
||||
if w.err != nil {
|
||||
return 0, w.err
|
||||
}
|
||||
return w.accepted.Write(content)
|
||||
type completedRunWriter struct {
|
||||
t *testing.T
|
||||
file string
|
||||
captured bool
|
||||
content bytes.Buffer
|
||||
}
|
||||
|
||||
func (writer *completedRunWriter) Write(content []byte) (int, error) {
|
||||
assertCompletedRunFiles(writer.t, writer.file, writer.captured)
|
||||
return writer.content.Write(content)
|
||||
}
|
||||
|
||||
func TestNativeRunCompletionFormatsFollowPublicationAndCapture(t *testing.T) {
|
||||
for _, format := range []commandkit.Format{commandkit.FormatText, commandkit.FormatJSON, commandkit.FormatYAML} {
|
||||
t.Run(string(format), func(t *testing.T) {
|
||||
probe := &runProbe{}
|
||||
app, file, input, _ := runFixture(t, probe)
|
||||
output := &completedRunWriter{t: t, file: file, captured: true}
|
||||
var diagnostics bytes.Buffer
|
||||
outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--format", string(format)}, commandkit.Streams{Out: output, Err: &diagnostics})
|
||||
if outcome.Status != 0 || !strings.Contains(diagnostics.String(), "Running pipeline") {
|
||||
t.Fatalf("completion: %#v stderr=%s", outcome, diagnostics.String())
|
||||
}
|
||||
directory := filepath.Join(filepath.Dir(file), "output-must-not-exist", "run-1")
|
||||
debug := filepath.Join(filepath.Dir(file), "debug-must-not-exist", "run-1")
|
||||
if format == commandkit.FormatText {
|
||||
if !strings.Contains(output.content.String(), "Success: demo produced 1 artifacts in "+directory) || !strings.Contains(output.content.String(), "Debug capture: "+debug) || strings.Contains(output.content.String(), "fallback") {
|
||||
t.Fatalf("native completion text: %s", output.content.String())
|
||||
}
|
||||
} else {
|
||||
report := decodeCompletionReport(t, format, output.content.Bytes())
|
||||
if len(report) != 4 || report["pipeline"] != "demo" || report["artifact_count"] != float64(1) || report["output_directory"] != directory || report["debug_directory"] != debug {
|
||||
t.Fatalf("native report: %#v", report)
|
||||
}
|
||||
}
|
||||
if probe.produced != 1 {
|
||||
t.Fatalf("reporting reran a producer: %d", probe.produced)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeRunDisclosesFallbackIndependentlyOfDiagnosticVisibility(t *testing.T) {
|
||||
for _, format := range []commandkit.Format{commandkit.FormatText, commandkit.FormatJSON, commandkit.FormatYAML} {
|
||||
for _, verbose := range []bool{false, true} {
|
||||
probe := &runProbe{correctFirst: true}
|
||||
app, file, input, _ := runFixture(t, probe)
|
||||
args := []string{"pipeline", "run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--set", `/pipelines/demo/mode="permissive"`, "--set", "/pipelines/demo/max_additional_turns=0", "--format", string(format)}
|
||||
if verbose {
|
||||
args = append(args, "--verbose")
|
||||
}
|
||||
var output, diagnostics bytes.Buffer
|
||||
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: &diagnostics})
|
||||
if outcome.Status != 0 || probe.produced != 1 {
|
||||
t.Fatalf("permissive completion: %#v %s", outcome, diagnostics.String())
|
||||
}
|
||||
if verbose && !strings.Contains(diagnostics.String(), "WARN:") || !verbose && diagnostics.Len() != 0 {
|
||||
t.Fatalf("native warning visibility: %s", diagnostics.String())
|
||||
}
|
||||
if format == commandkit.FormatText {
|
||||
if !strings.Contains(output.String(), "Success with fallback:") || !strings.Contains(output.String(), "1 operations used fallback.") {
|
||||
t.Fatalf("fallback text: %s", output.String())
|
||||
}
|
||||
} else {
|
||||
report := decodeCompletionReport(t, format, output.Bytes())
|
||||
if report["fallback_count"] != float64(1) || report["artifact_count"] != float64(1) || len(report) != 4 {
|
||||
t.Fatalf("fallback report: %#v", report)
|
||||
}
|
||||
}
|
||||
assertCompletedRunFiles(t, file, false)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeRunQuietAndDebugUseNativeModeOrdering(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
flags []string
|
||||
quiet bool
|
||||
progress bool
|
||||
}{
|
||||
{[]string{"--debug", "--quiet"}, true, false},
|
||||
{[]string{"--quiet", "--debug"}, false, true},
|
||||
{[]string{"--debug", "--progress=false"}, false, false},
|
||||
} {
|
||||
probe := &runProbe{}
|
||||
app, file, input, _ := runFixture(t, probe)
|
||||
var output, diagnostics bytes.Buffer
|
||||
args := append([]string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--format", "json"}, test.flags...)
|
||||
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: &diagnostics})
|
||||
if outcome.Status != 0 || test.quiet != (output.Len() == 0) || test.progress != strings.Contains(diagnostics.String(), "Running pipeline") {
|
||||
t.Fatalf("native mode order: %#v output=%s stderr=%s", outcome, output.String(), diagnostics.String())
|
||||
}
|
||||
assertCompletedRunFiles(t, file, true)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeRunCancellationKeepsItsCauseAndPublishesNothing(t *testing.T) {
|
||||
probe := &runProbe{}
|
||||
app, file, input, _ := runFixture(t, probe)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
probe.onGenerate = cancel
|
||||
var output, diagnostics bytes.Buffer
|
||||
outcome := app.Execute(ctx, []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--quiet"}, commandkit.Streams{Out: &output, Err: &diagnostics})
|
||||
if outcome.Status != 1 || !errors.Is(outcome.Cause, context.Canceled) || output.Len() != 0 || diagnostics.Len() == 0 {
|
||||
t.Fatalf("cancellation: %#v stdout=%s stderr=%s", outcome, output.String(), diagnostics.String())
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(filepath.Dir(file), "output-must-not-exist")); !os.IsNotExist(err) {
|
||||
t.Fatalf("cancellation published artifacts: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeRunPrimaryFailureSurvivesCaptureAndDiagnosticFailures(t *testing.T) {
|
||||
primary := errors.New("provider failure sentinel")
|
||||
delivery := errors.New("diagnostic failure sentinel")
|
||||
probe := &runProbe{providerErr: primary}
|
||||
app, file, input, _ := runFixture(t, probe)
|
||||
debug := filepath.Join(filepath.Dir(file), "debug-must-not-exist", "run-1")
|
||||
probe.onGenerate = func() {
|
||||
if err := os.Mkdir(filepath.Join(debug, "outcome.json"), 0700); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
var output bytes.Buffer
|
||||
outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--llm-transient-retries=0"}, commandkit.Streams{Out: &output, Err: failingResultWriter{cause: delivery}})
|
||||
var captureFailure *os.LinkError
|
||||
if outcome.Status != 1 || !errors.Is(outcome.Cause, primary) || !errors.As(outcome.Cause, &captureFailure) || errors.Is(outcome.Cause, delivery) || !errors.Is(outcome.Presentation.Diagnostics.WriteError, delivery) || output.Len() != 0 || len(probe.requests) != 1 {
|
||||
t.Fatalf("primary/capture/delivery failure: %#v", outcome)
|
||||
}
|
||||
if !strings.Contains(outcome.Explanation.Message, debug) {
|
||||
t.Fatalf("failed run capture location unavailable: %#v", outcome)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeRunReportDeliveryFailuresRetainArtifactsAndNativePresentation(t *testing.T) {
|
||||
sentinel := errors.New("result delivery sentinel")
|
||||
for _, partial := range []bool{false, true} {
|
||||
probe := &runProbe{}
|
||||
app, file, input, _ := runFixture(t, probe)
|
||||
var output io.Writer = failingResultWriter{cause: sentinel}
|
||||
prefix := &partialResultWriter{cause: sentinel}
|
||||
if partial {
|
||||
output = prefix
|
||||
}
|
||||
var diagnostics bytes.Buffer
|
||||
outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--format", "json"}, commandkit.Streams{Out: output, Err: &diagnostics})
|
||||
if outcome.Status != 1 || outcome.Cause != nil || !errors.Is(outcome.Presentation.Output.WriteError, sentinel) || outcome.Presentation.Output.State != commandkit.DeliveryIncomplete || probe.produced != 1 || partial && prefix.content.Len() == 0 {
|
||||
t.Fatalf("delivery failure: %#v %s", outcome, diagnostics.String())
|
||||
}
|
||||
assertCompletedRunFiles(t, file, true)
|
||||
for _, root := range []string{"output-must-not-exist", "debug-must-not-exist"} {
|
||||
if !strings.Contains(diagnostics.String(), filepath.Join(filepath.Dir(file), root, "run-1")) {
|
||||
t.Fatalf("missing available path after delivery failure: %s", diagnostics.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeRunDiagnosticDeliveryFailureIsNotAnOperationFailure(t *testing.T) {
|
||||
sentinel := errors.New("diagnostic delivery sentinel")
|
||||
probe := &runProbe{correctFirst: true}
|
||||
app, file, input, _ := runFixture(t, probe)
|
||||
var output bytes.Buffer
|
||||
outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--set", `/pipelines/demo/mode="permissive"`, "--set", "/pipelines/demo/max_additional_turns=0", "--format", "json", "--verbose"}, commandkit.Streams{Out: &output, Err: failingResultWriter{cause: sentinel}})
|
||||
if outcome.Status != 1 || outcome.Cause != nil || !errors.Is(outcome.Presentation.Diagnostics.WriteError, sentinel) || outcome.Presentation.Diagnostics.State != commandkit.DeliveryIncomplete {
|
||||
t.Fatalf("diagnostic delivery: %#v", outcome)
|
||||
}
|
||||
if report := decodeCompletionReport(t, commandkit.FormatJSON, output.Bytes()); report["fallback_count"] != float64(1) {
|
||||
t.Fatalf("completed report was replaced by a delivery envelope: %#v", report)
|
||||
}
|
||||
if !strings.Contains(outcome.Explanation.Message, filepath.Join(filepath.Dir(file), "output-must-not-exist", "run-1")) {
|
||||
t.Fatalf("presentation failure lost available artifacts: %#v", outcome)
|
||||
}
|
||||
assertCompletedRunFiles(t, file, false)
|
||||
}
|
||||
|
||||
func TestNativeRunCaptureFailureAfterPublicationDoesNotReportSuccess(t *testing.T) {
|
||||
probe := &runProbe{}
|
||||
app, file, input, _ := runFixture(t, probe)
|
||||
var once sync.Once
|
||||
debug := filepath.Join(filepath.Dir(file), "debug-must-not-exist", "run-1")
|
||||
probe.onGenerate = func() {
|
||||
once.Do(func() {
|
||||
// The external provider boundary runs after capture allocation but
|
||||
// before terminalization. Make only the terminal file unusable.
|
||||
if err := os.Mkdir(filepath.Join(debug, "outcome.json"), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
var output, diagnostics bytes.Buffer
|
||||
outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--format", "json"}, commandkit.Streams{Out: &output, Err: &diagnostics})
|
||||
if outcome.Status != 1 || outcome.Cause == nil || output.Len() != 0 || probe.produced != 1 {
|
||||
t.Fatalf("post-publication capture failure: %#v stdout=%s stderr=%s", outcome, output.String(), diagnostics.String())
|
||||
}
|
||||
assertCompletedRunFiles(t, file, false)
|
||||
if !strings.Contains(diagnostics.String(), debug) || !strings.Contains(diagnostics.String(), filepath.Join(filepath.Dir(file), "output-must-not-exist", "run-1")) {
|
||||
t.Fatalf("post-publication locations: %s", diagnostics.String())
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(debug, "executions.json")); err != nil {
|
||||
t.Fatalf("existing capture was removed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeRunFailureCategoriesPreserveCausesWithoutSuccessReports(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
configure func(*testing.T, *runProbe, string, string) []string
|
||||
status int
|
||||
cause error
|
||||
}{
|
||||
{"missing configuration", func(t *testing.T, _ *runProbe, _, input string) []string {
|
||||
return []string{"run", "demo", "--input-file", input, "--config", filepath.Join(t.TempDir(), "missing.yml")}
|
||||
}, 1, nil},
|
||||
{"invalid configuration", func(t *testing.T, _ *runProbe, file, input string) []string {
|
||||
return []string{"run", "demo", "--config", file, "--input-file", input, "--set", "/version=5"}
|
||||
}, 2, nil},
|
||||
{"unknown pipeline", func(_ *testing.T, _ *runProbe, file, input string) []string {
|
||||
return []string{"run", "missing", "--config", file, "--input-file", input}
|
||||
}, 2, nil},
|
||||
{"missing input", func(t *testing.T, _ *runProbe, file, _ string) []string {
|
||||
return []string{"run", "demo", "--config", file, "--input-file", filepath.Join(t.TempDir(), "missing.txt")}
|
||||
}, 1, os.ErrNotExist},
|
||||
{"publication failure", func(t *testing.T, _ *runProbe, file, input string) []string {
|
||||
path := filepath.Join(t.TempDir(), "occupied")
|
||||
if err := os.WriteFile(path, []byte("keep"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--output-dir", path}
|
||||
}, 1, nil},
|
||||
{"provider failure", func(_ *testing.T, probe *runProbe, file, input string) []string {
|
||||
probe.providerErr = io.ErrUnexpectedEOF
|
||||
return []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--llm-transient-retries=0"}
|
||||
}, 1, io.ErrUnexpectedEOF},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
probe := &runProbe{}
|
||||
app, file, input, _ := runFixture(t, probe)
|
||||
var output, diagnostics bytes.Buffer
|
||||
args := append(test.configure(t, probe, file, input), "--quiet", "--format", "json")
|
||||
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: &diagnostics})
|
||||
if outcome.Status != test.status || test.cause != nil && !errors.Is(outcome.Cause, test.cause) || output.Len() != 0 || diagnostics.Len() == 0 {
|
||||
t.Fatalf("failure category: %#v stdout=%s stderr=%s", outcome, output.String(), diagnostics.String())
|
||||
}
|
||||
if test.name == "missing configuration" || test.name == "invalid configuration" {
|
||||
var load *configkit.LoadError
|
||||
if !errors.As(outcome.Cause, &load) {
|
||||
t.Fatalf("lost native load cause: %#v", outcome)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user