Report completed runs through native presentation and preserve failure outcomes

This commit is contained in:
2026-11-22 04:05:44 +00:00
parent 0cbc2b39cb
commit a8d256f676
10 changed files with 656 additions and 511 deletions

View File

@@ -2,198 +2,318 @@ package cli
import (
"bytes"
"context"
"encoding/json"
"errors"
"io"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
alwaysreject "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/validate/always_reject"
"go.maximumdirect.net/commandkit"
"go.maximumdirect.net/configkit"
)
func TestMaintainedMinimalInvocationEmitsRunResult(t *testing.T) {
outputRoot := filepath.Join(t.TempDir(), "output")
var stdout, stderr strings.Builder
code := RunWithOptions([]string{
"run", "dnd-session",
"--config", repositoryPath("examples", "dnd-minimal.config.yml"),
"--input", repositoryPath("examples", "seriatim-minimal-transcript.json"),
"--only", "spells", "--chunk_cache", "bypass", "--output-dir", outputRoot, "--json",
}, &stdout, &stderr, productionRunOptions(t, &productionFakeLLMClient{}))
if code != 0 || stderr.Len() != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
receipt := decodeRunResultDocument(t, stdout.String())
if got := receipt["schema_version"]; got != "notarius.run-result.v2" {
t.Fatalf("schema_version = %q", got)
}
if got := receipt["run_id"]; got != productionRunID {
t.Fatalf("run_id = %q", got)
}
if got := receipt["pipeline_id"]; got != "dnd-session" {
t.Fatalf("pipeline_id = %q", got)
}
if got := receipt["index_file"]; got != "index.json" {
t.Fatalf("index_file = %q", got)
}
if got := receipt["normalized_output_count"]; got != float64(1) {
t.Fatalf("normalized_output_count = %v", got)
}
if got := receipt["rejected_output_count"]; got != float64(0) {
t.Fatalf("rejected_output_count = %v", got)
}
if got := receipt["warning_group_count"]; got != float64(0) || receipt["warning_occurrence_count"] != float64(0) || receipt["diagnostic_group_count"] != float64(0) || receipt["diagnostic_occurrence_count"] != float64(0) || receipt["diagnostics_truncated"] != false {
t.Fatalf("diagnostic counts = %#v", receipt)
}
if got := receipt["validation_status"]; got != "approved" {
t.Fatalf("validation_status = %q", got)
}
outputDirectory, ok := receipt["output_directory"].(string)
if !ok || !filepath.IsAbs(outputDirectory) || outputDirectory != filepath.Join(outputRoot, productionRunID) {
t.Fatalf("output_directory = %q", receipt["output_directory"])
}
indexFile := receipt["index_file"].(string)
assertFile(t, filepath.Join(outputDirectory, indexFile))
}
func TestRunResultReportsWarningsAndDebugBundle(t *testing.T) {
roots := newStateTestRoots(t)
harness := newStateTestHarness()
harness.chunkDiagnostics = []contracts.ProducerDiagnostic{stateTestDiagnostic("chunk", "contract-warning", "warning retained")}
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{
"run", "sample", "--config", roots.config, "--input", roots.input,
"--chunk_cache", "bypass", "--debug", "--json",
}, &stdout, &stderr, harness.options())
if code != 0 || !strings.Contains(stderr.String(), "1 warning group(s), 1 occurrence(s)") || strings.Contains(stderr.String(), "warnings.json") {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
receipt := decodeRunResultDocument(t, stdout.String())
if got := receipt["warning_group_count"]; got != float64(1) || receipt["warning_occurrence_count"] != float64(1) || receipt["diagnostic_group_count"] != float64(0) || receipt["diagnostic_occurrence_count"] != float64(0) || receipt["diagnostics_truncated"] != false {
t.Fatalf("diagnostic counts = %#v", receipt)
}
debugDirectory, ok := receipt["debug_directory"].(string)
if !ok || !filepath.IsAbs(debugDirectory) || debugDirectory != onlyChildDir(t, roots.debug) {
t.Fatalf("debug_directory = %q", receipt["debug_directory"])
}
if strings.Contains(stdout.String(), "complete:") || strings.Contains(stdout.String(), "debug=") {
t.Fatalf("machine stdout contains human reporting: %q", stdout.String())
}
}
func TestRunResultReportsSuccessfulRejection(t *testing.T) {
roots := newStateTestRoots(t)
configBytes, err := os.ReadFile(roots.config)
if err != nil {
t.Fatal(err)
}
configBytes = []byte(replaceRequiredOnce(t, string(configBytes), " normalize:\n module: test/normalize\n", " normalize:\n module: test/normalize\n validators:\n - module: generic/always_reject\n validation_policy:\n semantic_rejection: reject_output\n"))
if err := os.WriteFile(roots.config, configBytes, 0o600); err != nil {
t.Fatal(err)
}
harness := newStateTestHarness()
opts := harness.options()
if err := alwaysreject.RegisterTyped[stateTestArtifact](opts.Registries.Validators, stateTestArtifactKind); err != nil {
t.Fatal(err)
}
opts.Catalog = catalogFromRegistries(opts.Registries)
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{
"run", "sample", "--config", roots.config, "--input", roots.input,
"--chunk_cache", "bypass", "--json",
}, &stdout, &stderr, opts)
if code != 0 || stderr.Len() != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
receipt := decodeRunResultDocument(t, stdout.String())
if got := receipt["normalized_output_count"]; got != float64(0) {
t.Fatalf("normalized_output_count = %v", got)
}
if got := receipt["rejected_output_count"]; got != float64(1) {
t.Fatalf("rejected_output_count = %v", got)
}
if got := receipt["validation_status"]; got != "rejected" {
t.Fatalf("validation_status = %q", got)
}
}
func TestRunResultIsAbsentForSyntaxAndRuntimeFailures(t *testing.T) {
t.Run("syntax", func(t *testing.T) {
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"run", "sample", "--json"}, &stdout, &stderr, newStateTestHarness().options())
if code != 2 || stdout.Len() != 0 || stderr.Len() == 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
})
t.Run("runtime", func(t *testing.T) {
roots := newStateTestRoots(t)
harness := newStateTestHarness()
harness.extractErr = errors.New("injected extraction failure")
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{
"run", "sample", "--config", roots.config, "--input", roots.input,
"--chunk_cache", "bypass", "--json",
}, &stdout, &stderr, harness.options())
if code != 1 || stdout.Len() != 0 || stderr.Len() == 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
})
}
func TestRunResultDeliveryFailureRetainsPublishedBundles(t *testing.T) {
roots := newStateTestRoots(t)
writerErr := errors.New("result writer sentinel")
stdout := &resultDeliveryWriter{err: writerErr}
var stderr bytes.Buffer
code := RunWithOptions([]string{
"run", "sample", "--config", roots.config, "--input", roots.input,
"--chunk_cache", "bypass", "--debug", "--json",
}, stdout, &stderr, newStateTestHarness().options())
if code != 1 || !strings.Contains(stderr.String(), "write run result") || strings.Contains(stderr.String(), writerErr.Error()) {
t.Fatalf("code=%d stderr=%q", code, stderr.String())
}
if stdout.accepted.Len() != 0 {
t.Fatalf("accepted stdout = %q", stdout.accepted.String())
}
assertStateTestOutput(t, roots.output)
debugBundle := onlyChildDir(t, roots.debug)
report := readStateTestRunReport(t, debugBundle)
if !report.Succeeded {
t.Fatalf("debug report = %#v, want successful persisted run", report)
}
if strings.Contains(readAllFiles(t, debugBundle), writerErr.Error()) {
t.Fatalf("debug bundle contains result writer error")
}
}
func decodeRunResultDocument(t *testing.T, stdout string) map[string]any {
func decodeCompletionReport(t *testing.T, format commandkit.Format, content []byte) map[string]any {
t.Helper()
if strings.Count(stdout, "\n") != 1 {
t.Fatalf("stdout = %q, want one JSON document", stdout)
if format == commandkit.FormatYAML {
value, err := commandkit.DecodeYAMLReport(bytes.NewReader(content))
if err != nil {
t.Fatalf("decode native YAML report: %v\n%s", err, content)
}
content, err = json.Marshal(value)
if err != nil {
t.Fatal(err)
}
}
var receipt map[string]any
if err := json.Unmarshal([]byte(stdout), &receipt); err != nil {
t.Fatalf("decode run result: %v; stdout=%q", err, stdout)
var report map[string]any
if err := json.Unmarshal(content, &report); err != nil {
t.Fatalf("decode native report: %v\n%s", err, content)
}
return receipt
return report
}
type resultDeliveryWriter struct {
err error
accepted bytes.Buffer
func assertCompletedRunFiles(t *testing.T, file string, captured bool) {
t.Helper()
run := filepath.Join(filepath.Dir(file), "output-must-not-exist", "run-1")
if content, err := os.ReadFile(filepath.Join(run, "lanes", "notes.json")); err != nil || len(content) == 0 {
t.Fatalf("published domain artifact: %s %v", content, err)
}
files, err := os.ReadDir(run)
if err != nil || len(files) != 1 || files[0].Name() != "lanes" {
t.Fatalf("unexpected reporting sidecars: %#v %v", files, err)
}
if captured {
debug := filepath.Join(filepath.Dir(file), "debug-must-not-exist", "run-1")
content, err := os.ReadFile(filepath.Join(debug, "outcome.json"))
if err != nil {
t.Fatal(err)
}
var terminal struct {
Outcome struct {
Published bool `json:"published"`
} `json:"outcome"`
Error string `json:"error"`
CaptureError string `json:"capture_error"`
}
if err := json.Unmarshal(content, &terminal); err != nil || !terminal.Outcome.Published || terminal.Error != "" || terminal.CaptureError != "" {
t.Fatalf("capture was not completed before reporting: %s %v", content, err)
}
}
}
func (w *resultDeliveryWriter) Write(content []byte) (int, error) {
if w.err != nil {
return 0, w.err
}
return w.accepted.Write(content)
type completedRunWriter struct {
t *testing.T
file string
captured bool
content bytes.Buffer
}
func (writer *completedRunWriter) Write(content []byte) (int, error) {
assertCompletedRunFiles(writer.t, writer.file, writer.captured)
return writer.content.Write(content)
}
func TestNativeRunCompletionFormatsFollowPublicationAndCapture(t *testing.T) {
for _, format := range []commandkit.Format{commandkit.FormatText, commandkit.FormatJSON, commandkit.FormatYAML} {
t.Run(string(format), func(t *testing.T) {
probe := &runProbe{}
app, file, input, _ := runFixture(t, probe)
output := &completedRunWriter{t: t, file: file, captured: true}
var diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--format", string(format)}, commandkit.Streams{Out: output, Err: &diagnostics})
if outcome.Status != 0 || !strings.Contains(diagnostics.String(), "Running pipeline") {
t.Fatalf("completion: %#v stderr=%s", outcome, diagnostics.String())
}
directory := filepath.Join(filepath.Dir(file), "output-must-not-exist", "run-1")
debug := filepath.Join(filepath.Dir(file), "debug-must-not-exist", "run-1")
if format == commandkit.FormatText {
if !strings.Contains(output.content.String(), "Success: demo produced 1 artifacts in "+directory) || !strings.Contains(output.content.String(), "Debug capture: "+debug) || strings.Contains(output.content.String(), "fallback") {
t.Fatalf("native completion text: %s", output.content.String())
}
} else {
report := decodeCompletionReport(t, format, output.content.Bytes())
if len(report) != 4 || report["pipeline"] != "demo" || report["artifact_count"] != float64(1) || report["output_directory"] != directory || report["debug_directory"] != debug {
t.Fatalf("native report: %#v", report)
}
}
if probe.produced != 1 {
t.Fatalf("reporting reran a producer: %d", probe.produced)
}
})
}
}
func TestNativeRunDisclosesFallbackIndependentlyOfDiagnosticVisibility(t *testing.T) {
for _, format := range []commandkit.Format{commandkit.FormatText, commandkit.FormatJSON, commandkit.FormatYAML} {
for _, verbose := range []bool{false, true} {
probe := &runProbe{correctFirst: true}
app, file, input, _ := runFixture(t, probe)
args := []string{"pipeline", "run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--set", `/pipelines/demo/mode="permissive"`, "--set", "/pipelines/demo/max_additional_turns=0", "--format", string(format)}
if verbose {
args = append(args, "--verbose")
}
var output, diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: &diagnostics})
if outcome.Status != 0 || probe.produced != 1 {
t.Fatalf("permissive completion: %#v %s", outcome, diagnostics.String())
}
if verbose && !strings.Contains(diagnostics.String(), "WARN:") || !verbose && diagnostics.Len() != 0 {
t.Fatalf("native warning visibility: %s", diagnostics.String())
}
if format == commandkit.FormatText {
if !strings.Contains(output.String(), "Success with fallback:") || !strings.Contains(output.String(), "1 operations used fallback.") {
t.Fatalf("fallback text: %s", output.String())
}
} else {
report := decodeCompletionReport(t, format, output.Bytes())
if report["fallback_count"] != float64(1) || report["artifact_count"] != float64(1) || len(report) != 4 {
t.Fatalf("fallback report: %#v", report)
}
}
assertCompletedRunFiles(t, file, false)
}
}
}
func TestNativeRunQuietAndDebugUseNativeModeOrdering(t *testing.T) {
for _, test := range []struct {
flags []string
quiet bool
progress bool
}{
{[]string{"--debug", "--quiet"}, true, false},
{[]string{"--quiet", "--debug"}, false, true},
{[]string{"--debug", "--progress=false"}, false, false},
} {
probe := &runProbe{}
app, file, input, _ := runFixture(t, probe)
var output, diagnostics bytes.Buffer
args := append([]string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--format", "json"}, test.flags...)
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: &diagnostics})
if outcome.Status != 0 || test.quiet != (output.Len() == 0) || test.progress != strings.Contains(diagnostics.String(), "Running pipeline") {
t.Fatalf("native mode order: %#v output=%s stderr=%s", outcome, output.String(), diagnostics.String())
}
assertCompletedRunFiles(t, file, true)
}
}
func TestNativeRunCancellationKeepsItsCauseAndPublishesNothing(t *testing.T) {
probe := &runProbe{}
app, file, input, _ := runFixture(t, probe)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
probe.onGenerate = cancel
var output, diagnostics bytes.Buffer
outcome := app.Execute(ctx, []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--quiet"}, commandkit.Streams{Out: &output, Err: &diagnostics})
if outcome.Status != 1 || !errors.Is(outcome.Cause, context.Canceled) || output.Len() != 0 || diagnostics.Len() == 0 {
t.Fatalf("cancellation: %#v stdout=%s stderr=%s", outcome, output.String(), diagnostics.String())
}
if _, err := os.Stat(filepath.Join(filepath.Dir(file), "output-must-not-exist")); !os.IsNotExist(err) {
t.Fatalf("cancellation published artifacts: %v", err)
}
}
func TestNativeRunPrimaryFailureSurvivesCaptureAndDiagnosticFailures(t *testing.T) {
primary := errors.New("provider failure sentinel")
delivery := errors.New("diagnostic failure sentinel")
probe := &runProbe{providerErr: primary}
app, file, input, _ := runFixture(t, probe)
debug := filepath.Join(filepath.Dir(file), "debug-must-not-exist", "run-1")
probe.onGenerate = func() {
if err := os.Mkdir(filepath.Join(debug, "outcome.json"), 0700); err != nil {
t.Error(err)
}
}
var output bytes.Buffer
outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--llm-transient-retries=0"}, commandkit.Streams{Out: &output, Err: failingResultWriter{cause: delivery}})
var captureFailure *os.LinkError
if outcome.Status != 1 || !errors.Is(outcome.Cause, primary) || !errors.As(outcome.Cause, &captureFailure) || errors.Is(outcome.Cause, delivery) || !errors.Is(outcome.Presentation.Diagnostics.WriteError, delivery) || output.Len() != 0 || len(probe.requests) != 1 {
t.Fatalf("primary/capture/delivery failure: %#v", outcome)
}
if !strings.Contains(outcome.Explanation.Message, debug) {
t.Fatalf("failed run capture location unavailable: %#v", outcome)
}
}
func TestNativeRunReportDeliveryFailuresRetainArtifactsAndNativePresentation(t *testing.T) {
sentinel := errors.New("result delivery sentinel")
for _, partial := range []bool{false, true} {
probe := &runProbe{}
app, file, input, _ := runFixture(t, probe)
var output io.Writer = failingResultWriter{cause: sentinel}
prefix := &partialResultWriter{cause: sentinel}
if partial {
output = prefix
}
var diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--format", "json"}, commandkit.Streams{Out: output, Err: &diagnostics})
if outcome.Status != 1 || outcome.Cause != nil || !errors.Is(outcome.Presentation.Output.WriteError, sentinel) || outcome.Presentation.Output.State != commandkit.DeliveryIncomplete || probe.produced != 1 || partial && prefix.content.Len() == 0 {
t.Fatalf("delivery failure: %#v %s", outcome, diagnostics.String())
}
assertCompletedRunFiles(t, file, true)
for _, root := range []string{"output-must-not-exist", "debug-must-not-exist"} {
if !strings.Contains(diagnostics.String(), filepath.Join(filepath.Dir(file), root, "run-1")) {
t.Fatalf("missing available path after delivery failure: %s", diagnostics.String())
}
}
}
}
func TestNativeRunDiagnosticDeliveryFailureIsNotAnOperationFailure(t *testing.T) {
sentinel := errors.New("diagnostic delivery sentinel")
probe := &runProbe{correctFirst: true}
app, file, input, _ := runFixture(t, probe)
var output bytes.Buffer
outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--set", `/pipelines/demo/mode="permissive"`, "--set", "/pipelines/demo/max_additional_turns=0", "--format", "json", "--verbose"}, commandkit.Streams{Out: &output, Err: failingResultWriter{cause: sentinel}})
if outcome.Status != 1 || outcome.Cause != nil || !errors.Is(outcome.Presentation.Diagnostics.WriteError, sentinel) || outcome.Presentation.Diagnostics.State != commandkit.DeliveryIncomplete {
t.Fatalf("diagnostic delivery: %#v", outcome)
}
if report := decodeCompletionReport(t, commandkit.FormatJSON, output.Bytes()); report["fallback_count"] != float64(1) {
t.Fatalf("completed report was replaced by a delivery envelope: %#v", report)
}
if !strings.Contains(outcome.Explanation.Message, filepath.Join(filepath.Dir(file), "output-must-not-exist", "run-1")) {
t.Fatalf("presentation failure lost available artifacts: %#v", outcome)
}
assertCompletedRunFiles(t, file, false)
}
func TestNativeRunCaptureFailureAfterPublicationDoesNotReportSuccess(t *testing.T) {
probe := &runProbe{}
app, file, input, _ := runFixture(t, probe)
var once sync.Once
debug := filepath.Join(filepath.Dir(file), "debug-must-not-exist", "run-1")
probe.onGenerate = func() {
once.Do(func() {
// The external provider boundary runs after capture allocation but
// before terminalization. Make only the terminal file unusable.
if err := os.Mkdir(filepath.Join(debug, "outcome.json"), 0700); err != nil {
t.Fatal(err)
}
})
}
var output, diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--format", "json"}, commandkit.Streams{Out: &output, Err: &diagnostics})
if outcome.Status != 1 || outcome.Cause == nil || output.Len() != 0 || probe.produced != 1 {
t.Fatalf("post-publication capture failure: %#v stdout=%s stderr=%s", outcome, output.String(), diagnostics.String())
}
assertCompletedRunFiles(t, file, false)
if !strings.Contains(diagnostics.String(), debug) || !strings.Contains(diagnostics.String(), filepath.Join(filepath.Dir(file), "output-must-not-exist", "run-1")) {
t.Fatalf("post-publication locations: %s", diagnostics.String())
}
if _, err := os.Stat(filepath.Join(debug, "executions.json")); err != nil {
t.Fatalf("existing capture was removed: %v", err)
}
}
func TestNativeRunFailureCategoriesPreserveCausesWithoutSuccessReports(t *testing.T) {
for _, test := range []struct {
name string
configure func(*testing.T, *runProbe, string, string) []string
status int
cause error
}{
{"missing configuration", func(t *testing.T, _ *runProbe, _, input string) []string {
return []string{"run", "demo", "--input-file", input, "--config", filepath.Join(t.TempDir(), "missing.yml")}
}, 1, nil},
{"invalid configuration", func(t *testing.T, _ *runProbe, file, input string) []string {
return []string{"run", "demo", "--config", file, "--input-file", input, "--set", "/version=5"}
}, 2, nil},
{"unknown pipeline", func(_ *testing.T, _ *runProbe, file, input string) []string {
return []string{"run", "missing", "--config", file, "--input-file", input}
}, 2, nil},
{"missing input", func(t *testing.T, _ *runProbe, file, _ string) []string {
return []string{"run", "demo", "--config", file, "--input-file", filepath.Join(t.TempDir(), "missing.txt")}
}, 1, os.ErrNotExist},
{"publication failure", func(t *testing.T, _ *runProbe, file, input string) []string {
path := filepath.Join(t.TempDir(), "occupied")
if err := os.WriteFile(path, []byte("keep"), 0600); err != nil {
t.Fatal(err)
}
return []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--output-dir", path}
}, 1, nil},
{"provider failure", func(_ *testing.T, probe *runProbe, file, input string) []string {
probe.providerErr = io.ErrUnexpectedEOF
return []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--llm-transient-retries=0"}
}, 1, io.ErrUnexpectedEOF},
} {
t.Run(test.name, func(t *testing.T) {
probe := &runProbe{}
app, file, input, _ := runFixture(t, probe)
var output, diagnostics bytes.Buffer
args := append(test.configure(t, probe, file, input), "--quiet", "--format", "json")
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: &diagnostics})
if outcome.Status != test.status || test.cause != nil && !errors.Is(outcome.Cause, test.cause) || output.Len() != 0 || diagnostics.Len() == 0 {
t.Fatalf("failure category: %#v stdout=%s stderr=%s", outcome, output.String(), diagnostics.String())
}
if test.name == "missing configuration" || test.name == "invalid configuration" {
var load *configkit.LoadError
if !errors.As(outcome.Cause, &load) {
t.Fatalf("lost native load cause: %#v", outcome)
}
}
})
}
}