Report completed runs through native presentation and preserve failure outcomes
This commit is contained in:
@@ -2,229 +2,136 @@ package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"text/template"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"go.maximumdirect.net/commandkit"
|
||||
)
|
||||
|
||||
func TestRunResultEncodesRequiredFieldsAndCounts(t *testing.T) {
|
||||
result, err := newRunResult(testResolvedPipeline(pipeline.DefaultOutputModule), testRunOutput(), "relative-output", "relative-debug")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
encoded, err := encodeRunResult(result)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if encoded[len(encoded)-1] != '\n' || bytes.Count(encoded, []byte{'\n'}) != 1 {
|
||||
t.Fatalf("encoded result is not one newline-terminated object: %q", encoded)
|
||||
}
|
||||
|
||||
var decoded map[string]any
|
||||
if err := json.Unmarshal(encoded, &decoded); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := decoded["schema_version"]; got != runResultSchemaVersion {
|
||||
t.Fatalf("schema_version = %q", got)
|
||||
}
|
||||
if got := decoded["run_id"]; got != "run-123" {
|
||||
t.Fatalf("run_id = %q", got)
|
||||
}
|
||||
if got := decoded["pipeline_id"]; got != "sample" {
|
||||
t.Fatalf("pipeline_id = %q", got)
|
||||
}
|
||||
if got := decoded["validation_status"]; got != "rejected" {
|
||||
t.Fatalf("validation_status = %q", got)
|
||||
}
|
||||
if got := decoded["index_file"]; got != "index.json" {
|
||||
t.Fatalf("index_file = %q", got)
|
||||
}
|
||||
if got := decoded["normalized_output_count"]; got != float64(2) {
|
||||
t.Fatalf("normalized_output_count = %v", got)
|
||||
}
|
||||
if got := decoded["rejected_output_count"]; got != float64(1) {
|
||||
t.Fatalf("rejected_output_count = %v", got)
|
||||
}
|
||||
if got := decoded["warning_group_count"]; got != float64(1) || decoded["warning_occurrence_count"] != float64(1) || decoded["diagnostic_group_count"] != float64(0) || decoded["diagnostic_occurrence_count"] != float64(0) || decoded["diagnostics_truncated"] != false {
|
||||
t.Fatalf("diagnostic counts = %#v", decoded)
|
||||
}
|
||||
if got := decoded["validation_summaries"]; got != nil {
|
||||
t.Fatalf("validation_summaries = %#v, want omitted when empty", got)
|
||||
}
|
||||
if got := decoded["output_directory"]; got != filepath.Join(mustWorkingDirectory(t), "relative-output") {
|
||||
t.Fatalf("output_directory = %q", got)
|
||||
}
|
||||
if got := decoded["debug_directory"]; got != filepath.Join(mustWorkingDirectory(t), "relative-debug") {
|
||||
t.Fatalf("debug_directory = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunResultRejectsInvalidRequiredValues(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
resolved pipeline.ResolvedPipeline
|
||||
output pipeline.RunOutput
|
||||
directory string
|
||||
}{
|
||||
{name: "blank run ID", resolved: testResolvedPipeline(pipeline.DefaultOutputModule), output: testRunOutputWithout(func(output *pipeline.RunOutput) { output.Manifest.RunID = " " }), directory: "output"},
|
||||
{name: "blank resolved pipeline ID", resolved: pipeline.ResolvedPipeline{Output: pipeline.ModuleBinding{Module: pipeline.DefaultOutputModule}}, output: testRunOutput(), directory: "output"},
|
||||
{name: "blank manifest pipeline ID", resolved: testResolvedPipeline(pipeline.DefaultOutputModule), output: testRunOutputWithout(func(output *pipeline.RunOutput) { output.Manifest.PipelineID = "" }), directory: "output"},
|
||||
{name: "mismatched pipeline IDs", resolved: testResolvedPipeline(pipeline.DefaultOutputModule), output: testRunOutputWithout(func(output *pipeline.RunOutput) { output.Manifest.PipelineID = "other" }), directory: "output"},
|
||||
{name: "blank validation status", resolved: testResolvedPipeline(pipeline.DefaultOutputModule), output: testRunOutputWithout(func(output *pipeline.RunOutput) { output.Manifest.ValidationStatus = " " }), directory: "output"},
|
||||
{name: "blank output directory", resolved: testResolvedPipeline(pipeline.DefaultOutputModule), output: testRunOutput(), directory: " "},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if _, err := newRunResult(tt.resolved, tt.output, tt.directory, ""); err == nil {
|
||||
t.Fatal("newRunResult() succeeded")
|
||||
func TestRunResultUsesOneNativeValueAcrossFormats(t *testing.T) {
|
||||
value := runResult{Pipeline: "demo", ArtifactCount: 3, OutputDirectory: "/artifacts/run-one", DebugDirectory: "/debug/run-one", FallbackCount: 2}
|
||||
for _, format := range []commandkit.Format{commandkit.FormatText, commandkit.FormatJSON, commandkit.FormatYAML} {
|
||||
var output bytes.Buffer
|
||||
if err := commandkit.RenderReport(&output, commandkit.Report{Value: value, TextTemplate: runResultTemplate}, format); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if format == commandkit.FormatText {
|
||||
if !strings.Contains(output.String(), "Success with fallback: demo produced 3 artifacts in /artifacts/run-one") || !strings.Contains(output.String(), "2 operations used fallback.") || !strings.Contains(output.String(), "Debug capture: /debug/run-one") {
|
||||
t.Fatalf("fallback text: %s", output.String())
|
||||
}
|
||||
})
|
||||
continue
|
||||
}
|
||||
report := decodeCompletionReport(t, format, output.Bytes())
|
||||
if report["pipeline"] != "demo" || report["output_directory"] != value.OutputDirectory || report["debug_directory"] != value.DebugDirectory || report["fallback_count"] != float64(2) || report["artifact_count"] != float64(3) || len(report) != 5 {
|
||||
t.Fatalf("completion value: %#v", report)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunResultOmitsIndexFileForOtherOutputModules(t *testing.T) {
|
||||
result, err := newRunResult(testResolvedPipeline("test/output"), testRunOutputWithout(func(output *pipeline.RunOutput) {
|
||||
output.OutputFiles = nil
|
||||
}), "output", "")
|
||||
func TestNativeCompletionDiagnosticsRetainDomainClassification(t *testing.T) {
|
||||
collection := contracts.DiagnosticCollection{}
|
||||
for _, disposition := range []contracts.DiagnosticDisposition{contracts.DiagnosticDispositionWarning, contracts.DiagnosticDispositionAdvisory, contracts.DiagnosticDispositionObservation} {
|
||||
category := contracts.DiagnosticCategoryDataQuality
|
||||
if disposition == contracts.DiagnosticDispositionWarning {
|
||||
category = contracts.DiagnosticCategoryFallback
|
||||
} else if disposition == contracts.DiagnosticDispositionObservation {
|
||||
category = contracts.DiagnosticCategoryNormalization
|
||||
}
|
||||
collection.Groups = append(collection.Groups, contracts.DiagnosticGroup{Disposition: disposition, Category: category, ReasonCode: string(disposition), Origin: contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract, StepID: "build", LaneID: "notes", ModuleKey: "producer"}, OccurrenceCount: 1, Samples: []contracts.DiagnosticSample{{Scope: "current source", Message: string(disposition) + " finding"}}})
|
||||
}
|
||||
if err := collection.Validate(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var output, diagnostics bytes.Buffer
|
||||
app, err := commandkit.New(commandkit.Identity{Name: "fixture"}, commandkit.NewCommand(commandkit.CommandSpec[struct{}]{Name: "report", Run: func(invocation commandkit.Invocation, _ struct{}) commandkit.Outcome {
|
||||
reportRunDiagnostics(invocation.Reporter(), collection)
|
||||
return reportRunResult(invocation, runResult{Pipeline: "demo", ArtifactCount: 1, OutputDirectory: "/artifacts/run-one"})
|
||||
}}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.IndexFile != "" {
|
||||
t.Fatalf("index_file = %q", result.IndexFile)
|
||||
outcome := app.Execute(context.Background(), []string{"report", "--verbose"}, commandkit.Streams{Out: &output, Err: &diagnostics})
|
||||
if outcome.Status != 0 || strings.Contains(output.String(), "finding") || strings.Count(diagnostics.String(), "WARN:") != 1 {
|
||||
t.Fatalf("diagnostic classification: %#v stdout=%s stderr=%s", outcome, output.String(), diagnostics.String())
|
||||
}
|
||||
encoded, err := encodeRunResult(result)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var decoded map[string]any
|
||||
if err := json.Unmarshal(encoded, &decoded); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := decoded["index_file"]; ok {
|
||||
t.Fatalf("encoded non-JSON result contains index_file: %s", encoded)
|
||||
}
|
||||
if _, ok := decoded["debug_directory"]; ok {
|
||||
t.Fatalf("encoded result without debug capture contains debug_directory: %s", encoded)
|
||||
for _, group := range collection.Groups {
|
||||
if !strings.Contains(diagnostics.String(), string(group.Disposition)+" finding") {
|
||||
t.Fatalf("missing diagnostic: %s", diagnostics.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunResultProjectsOwnedValidationSummaries(t *testing.T) {
|
||||
output := testRunOutput()
|
||||
output.Manifest.ValidationSummaries = []artifacts.ValidationSummary{{Status: "incomplete", IncompleteValidators: []string{"validator"}, ProducerAttemptCount: 1, TerminalAction: "warn_continue"}}
|
||||
result, err := newRunResult(testResolvedPipeline(pipeline.DefaultOutputModule), output, "output", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
output.Manifest.ValidationSummaries[0].IncompleteValidators[0] = "caller mutation"
|
||||
if got := result.ValidationSummaries[0].IncompleteValidators; len(got) != 1 || got[0] != "validator" {
|
||||
t.Fatalf("result validation summaries = %#v", result.ValidationSummaries)
|
||||
}
|
||||
encoded, err := encodeRunResult(result)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Contains(encoded, []byte(`"validation_summaries":[{"status":"incomplete","incomplete_validators":["validator"],"producer_attempt_count":1,"terminal_action":"warn_continue"}]`)) {
|
||||
t.Fatalf("encoded result = %s", encoded)
|
||||
func TestCompletedRunRenderingFailureRetainsPublicationEvenWhenQuiet(t *testing.T) {
|
||||
previous := runResultTemplate
|
||||
t.Cleanup(func() { runResultTemplate = previous })
|
||||
// Simulate a broken application-owned text template, not a new renderer seam.
|
||||
runResultTemplate = template.Must(commandkit.ParseTextTemplate("invalid", "{{.MissingField}}", nil))
|
||||
for _, quiet := range []bool{false, true} {
|
||||
probe := &runProbe{}
|
||||
app, file, input, _ := runFixture(t, probe)
|
||||
args := []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug"}
|
||||
if quiet {
|
||||
args = append(args, "--quiet")
|
||||
}
|
||||
var output, diagnostics bytes.Buffer
|
||||
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: &diagnostics})
|
||||
if outcome.Status != 1 || outcome.Cause != nil || outcome.Presentation.Output.RenderError == nil || outcome.Presentation.Output.State != commandkit.DeliveryIncomplete || probe.produced != 1 {
|
||||
t.Fatalf("render failure: %#v %s", outcome, diagnostics.String())
|
||||
}
|
||||
assertCompletedRunFiles(t, file, true)
|
||||
artifacts := filepath.Join(filepath.Dir(file), "output-must-not-exist", "run-1")
|
||||
if !strings.Contains(diagnostics.String(), artifacts) || (quiet && output.Len() != 0) {
|
||||
t.Fatalf("render failure locations/quiet: %s", diagnostics.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunResultRequiresOneProductionIndexFile(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
files []contracts.OutputFile
|
||||
}{
|
||||
{name: "missing", files: nil},
|
||||
{name: "duplicate", files: []contracts.OutputFile{{Name: "index.json"}, {Name: "index.json"}}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
output := testRunOutput()
|
||||
output.OutputFiles = tt.files
|
||||
if _, err := newRunResult(testResolvedPipeline(pipeline.DefaultOutputModule), output, "output", ""); err == nil {
|
||||
t.Fatal("newRunResult() succeeded")
|
||||
}
|
||||
})
|
||||
}
|
||||
type failingResultWriter struct {
|
||||
cause error
|
||||
}
|
||||
|
||||
func TestWriteRunResultCompletesAndReportsWriterFailure(t *testing.T) {
|
||||
content := []byte("result\n")
|
||||
var target bytes.Buffer
|
||||
if err := writeRunResult(partialResultWriter{writer: &target, limit: 2}, content); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := target.String(); got != string(content) {
|
||||
t.Fatalf("written result = %q", got)
|
||||
}
|
||||
|
||||
writerErr := errors.New("result writer failed")
|
||||
if err := writeRunResult(failingResultWriter{err: writerErr}, content); !errors.Is(err, writerErr) {
|
||||
t.Fatalf("writeRunResult() error = %v", err)
|
||||
}
|
||||
if err := writeRunResult(zeroResultWriter{}, content); !errors.Is(err, io.ErrShortWrite) {
|
||||
t.Fatalf("zero-progress error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func testResolvedPipeline(outputModule string) pipeline.ResolvedPipeline {
|
||||
return pipeline.ResolvedPipeline{ID: "sample", Output: pipeline.ModuleBinding{Module: outputModule}}
|
||||
}
|
||||
|
||||
func testRunOutput() pipeline.RunOutput {
|
||||
return pipeline.RunOutput{
|
||||
Manifest: artifacts.RunManifest{RunID: "run-123", PipelineID: "sample", ValidationStatus: "rejected"},
|
||||
NormalizeOutputs: []contracts.SerializedOutput{{}, {}},
|
||||
Rejected: []contracts.RejectedOutput{{}},
|
||||
Diagnostics: contracts.DiagnosticCollection{Groups: []contracts.DiagnosticGroup{{
|
||||
Disposition: contracts.DiagnosticDispositionWarning,
|
||||
Category: contracts.DiagnosticCategoryFallback,
|
||||
ReasonCode: "fallback",
|
||||
Origin: contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageNormalize, StepID: "step", LaneID: "lane", ModuleKey: "module"},
|
||||
OccurrenceCount: 1,
|
||||
Samples: []contracts.DiagnosticSample{{Scope: "scope", Message: "message"}},
|
||||
}}},
|
||||
OutputFiles: []contracts.OutputFile{{Name: "index.json"}},
|
||||
}
|
||||
}
|
||||
|
||||
func testRunOutputWithout(change func(*pipeline.RunOutput)) pipeline.RunOutput {
|
||||
output := testRunOutput()
|
||||
change(&output)
|
||||
return output
|
||||
}
|
||||
|
||||
func mustWorkingDirectory(t *testing.T) string {
|
||||
t.Helper()
|
||||
workingDirectory, err := filepath.Abs(".")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return workingDirectory
|
||||
}
|
||||
func (writer failingResultWriter) Write([]byte) (int, error) { return 0, writer.cause }
|
||||
|
||||
type partialResultWriter struct {
|
||||
writer io.Writer
|
||||
limit int
|
||||
content bytes.Buffer
|
||||
cause error
|
||||
}
|
||||
|
||||
func (w partialResultWriter) Write(content []byte) (int, error) {
|
||||
if len(content) > w.limit {
|
||||
content = content[:w.limit]
|
||||
func (writer *partialResultWriter) Write(content []byte) (int, error) {
|
||||
if len(content) > 8 {
|
||||
content = content[:8]
|
||||
}
|
||||
return w.writer.Write(content)
|
||||
n, _ := writer.content.Write(content)
|
||||
return n, writer.cause
|
||||
}
|
||||
|
||||
type failingResultWriter struct{ err error }
|
||||
var _ io.Writer = (*partialResultWriter)(nil)
|
||||
|
||||
func (w failingResultWriter) Write([]byte) (int, error) { return 0, w.err }
|
||||
|
||||
type zeroResultWriter struct{}
|
||||
|
||||
func (zeroResultWriter) Write([]byte) (int, error) { return 0, nil }
|
||||
func TestPipelinePreflightClassifiesOnlyIdentifiableInvalidSelections(t *testing.T) {
|
||||
// Real native missing-resource errors retain their category separately from
|
||||
// private lookup causes. A joined operational branch must still win.
|
||||
probe := &runProbe{}
|
||||
app, file, input, _ := runFixture(t, probe)
|
||||
var diagnostics bytes.Buffer
|
||||
missing := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--llm-profile", "absent"}, commandkit.Streams{Out: io.Discard, Err: &diagnostics}).Cause
|
||||
for _, test := range []struct {
|
||||
cause error
|
||||
status int
|
||||
}{
|
||||
{missing, 2},
|
||||
{errors.Join(missing, context.Canceled), 1},
|
||||
{errors.Join(missing, errors.New("unavailable inspection")), 1},
|
||||
{errors.New("unavailable inspection"), 1},
|
||||
} {
|
||||
outcome := pipelineResourceFailure(test.cause)
|
||||
if outcome.Status != test.status || !errors.Is(outcome.Cause, test.cause) {
|
||||
t.Fatalf("preflight classification: %#v", outcome)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user