Report completed runs through native presentation and preserve failure outcomes

This commit is contained in:
2026-11-22 04:05:44 +00:00
parent 0cbc2b39cb
commit a8d256f676
10 changed files with 656 additions and 511 deletions

View File

@@ -2,21 +2,34 @@ package cli
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"testing"
"go.maximumdirect.net/commandkit"
)
func TestExecutableRunRoutesDeclareTheSameRequiredInputs(t *testing.T) {
func buildNativeExecutable(t *testing.T) string {
t.Helper()
binary := filepath.Join(t.TempDir(), "notarius")
build := exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"), "build", "-o", binary, "../../cmd/notarius")
build.Env = append(os.Environ(), "GOWORK=off")
if output, err := build.CombinedOutput(); err != nil {
t.Fatalf("build executable: %v\n%s", err, output)
}
return binary
}
func TestExecutableRunRoutesDeclareTheSameRequiredInputs(t *testing.T) {
binary := buildNativeExecutable(t)
for _, route := range [][]string{{"run"}, {"pipeline", "run"}} {
help := exec.Command(binary, append(append([]string(nil), route...), "--help")...)
output, err := help.CombinedOutput()
@@ -31,3 +44,68 @@ func TestExecutableRunRoutesDeclareTheSameRequiredInputs(t *testing.T) {
}
}
}
func TestExecutableReportsDomainArtifactsAndQualifiedFallbacks(t *testing.T) {
binary := buildNativeExecutable(t)
for _, fallback := range []bool{false, true} {
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
if fallback {
writer.WriteHeader(http.StatusServiceUnavailable)
return
}
writer.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(writer).Encode(map[string]any{"id": "local-fake", "object": "chat.completion", "choices": []any{map[string]any{"index": 0, "message": map[string]any{"role": "assistant", "content": `{"spell_casts":[]}`}, "finish_reason": "stop"}}})
}))
t.Cleanup(server.Close)
for _, format := range []commandkit.Format{commandkit.FormatText, commandkit.FormatJSON, commandkit.FormatYAML} {
t.Run(fmt.Sprintf("%s/fallback=%t", format, fallback), func(t *testing.T) {
root := t.TempDir()
configuration := fmt.Sprintf(`version: 6
output: {directory: output}
pipelines:
demo:
input: {module: seriatim}
chunk: {module: generic}
llm_profile: local-fake
mode: permissive
max_additional_turns: 0
artifacts:
spells:
extract: {module: dnd/spells}
promptkit:
profiles:
- id: local-fake
endpoint: %q
model: fake-model
api_key_required: false
`, server.URL+"/v1")
file := filepath.Join(root, "config.yml")
input := filepath.Join(root, "input.json")
for path, content := range map[string]string{file: configuration, input: `{"metadata":{"id":"report-source"},"segments":[{"id":1,"start":0,"end":1,"speaker":"GM","text":"The party rests."}]}`} {
if err := os.WriteFile(path, []byte(content), 0600); err != nil {
t.Fatal(err)
}
}
command := exec.Command(binary, "run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--llm-transient-retries=0", "--format", string(format), "--verbose")
var output, diagnostics bytes.Buffer
command.Stdout, command.Stderr = &output, &diagnostics
if err := command.Run(); err != nil {
t.Fatalf("subprocess: %v stdout=%s stderr=%s", err, output.String(), diagnostics.String())
}
if format == commandkit.FormatText {
if !strings.Contains(output.String(), "produced 1 artifacts") || fallback != strings.Contains(output.String(), "Success with fallback:") {
t.Fatalf("completion text: %s", output.String())
}
} else {
report := decodeCompletionReport(t, format, output.Bytes())
if report["artifact_count"] != float64(1) || report["pipeline"] != "demo" || fallback && report["fallback_count"] != float64(1) || !fallback && report["fallback_count"] != nil {
t.Fatalf("completion report: %#v", report)
}
if _, err := os.Stat(filepath.Join(report["output_directory"].(string), "lanes", "spells.json")); err != nil {
t.Fatal(err)
}
}
})
}
}
}

View File

@@ -107,7 +107,7 @@ func pipelineCommands(newState func() *nativeLoaders, catalog func() (pipeline.M
}
}
if err := preflight.Validate(invocation.Context, effective.ResolvedPipeline, modules, engine); err != nil {
return pipelineInvalidConfiguration(err)
return pipelineResourceFailure(err)
}
return reportPipelineInspection(invocation, commandkit.Report{Value: struct {
Pipeline string `json:"pipeline"`
@@ -124,16 +124,56 @@ var pipelineValidationTemplate = template.Must(commandkit.ParseTextTemplate("pip
func reportPipelineInspection(invocation commandkit.Invocation, report commandkit.Report) commandkit.Outcome {
if err := invocation.Reporter().Report(report); err != nil {
return commandkit.Failure("Could not report pipeline inspection.").WithCause(err)
return commandkit.Failure("Could not report pipeline inspection.")
}
return commandkit.Success()
}
func pipelineConfigurationFailure(err error) commandkit.Outcome {
outcome := commandkit.Failure(err.Error()).WithCause(err)
if ckconfig.ClassifyConfigurationError(err) == commandkit.OutcomeUsage {
outcome = commandkit.Usage(err.Error()).WithCause(err)
}
var load *configkit.LoadError
if errors.As(err, &load) {
_, explanation := ckconfig.AdaptDiagnostics(err.Error()+"\n", load.Sources(), load.Diagnostics())
outcome = outcome.WithExplanation(explanation.Message, explanation.Details...)
}
return outcome
}
func pipelineResourceFailure(err error) commandkit.Outcome {
if preflightInputError(err) {
return commandkit.Usage(err.Error()).WithCause(err)
}
return commandkit.Failure(err.Error()).WithCause(err)
return pipelineConfigurationFailure(err)
}
// Only identifiable missing native selections are invalid preflight input.
// Unknown failures and any operational branch of a joined error remain failures.
func preflightInputError(err error) bool {
if err == nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return false
}
if _, ok := err.(*preflight.SelectionError); ok {
return true
}
if joined, ok := err.(interface{ Unwrap() []error }); ok {
found := false
for _, child := range joined.Unwrap() {
if child != nil {
found = true
if !preflightInputError(child) {
return false
}
}
}
return found
}
if wrapped, ok := err.(interface{ Unwrap() error }); ok {
return preflightInputError(wrapped.Unwrap())
}
return false
}
func pipelineInvalidConfiguration(err error) commandkit.Outcome {

View File

@@ -262,7 +262,7 @@ func TestPipelineInspectionReportFailureRemainsOperational(t *testing.T) {
primary := errors.New("inspection stdout delivery failed")
var diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), []string{"pipeline", "list", "--config", file}, commandkit.Streams{Out: inspectionFailWriter{primary}, Err: &diagnostics})
if outcome.Status != 1 || !errors.Is(outcome.Cause, primary) {
if outcome.Status != 1 || outcome.Cause != nil || !errors.Is(outcome.Presentation.Output.WriteError, primary) {
t.Fatalf("delivery failure classified as invalid configuration: %#v %s", outcome, diagnostics.String())
}
}

View File

@@ -11,6 +11,7 @@ import (
frameworkdebug "gitea.maximumdirect.net/eric/notarius/internal/framework/debug"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/preflight"
"go.maximumdirect.net/ckconfig"
"go.maximumdirect.net/commandkit"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
@@ -31,6 +32,14 @@ func pipelineRunCommand(version string, newState func() *nativeLoaders, modules
if err != nil {
return pipelineConfigurationFailure(err)
}
diagnostics, _ := ckconfig.AdaptDiagnostics("Configuration\n", result.Sources(), result.Diagnostics())
for _, diagnostic := range diagnostics {
if diagnostic.Severity == commandkit.SeverityWarning {
_ = invocation.Reporter().AtEnd().Warn("", diagnostic.Write)
} else {
_ = invocation.Reporter().AtEnd().Info("", diagnostic.Write)
}
}
loaded, err := config.FromResult(result)
if err != nil {
return pipelineInvalidConfiguration(err)
@@ -53,20 +62,20 @@ func pipelineRunCommand(version string, newState func() *nativeLoaders, modules
}
finish := func(outcome commandkit.Outcome) commandkit.Outcome {
if err := capture.Finalize(captureOutcome(), outcome.Cause); err != nil {
message := err.Error()
if published {
message += fmt.Sprintf("\nArtifacts: %s", outputDirectory)
}
if capture.Path() != "" {
message += fmt.Sprintf("\nDebug capture: %s", capture.Path())
}
if outcome.Status == 0 {
outcome = commandkit.Failure(message)
outcome = commandkit.Failure(err.Error())
} else {
outcome = outcome.WithExplanation(message)
outcome = outcome.WithExplanation(err.Error(), outcome.Explanation.Details...)
}
outcome = outcome.WithCause(err)
}
if outcome.Status != 0 {
availableOutput := ""
if published {
availableOutput = outputDirectory
}
outcome = outcome.WithExplanation(outcome.Explanation.Message+runLocations(availableOutput, capture.Path()), outcome.Explanation.Details...)
}
return outcome
}
capture, err = startRunDebug(input.Debug, cfg.Debug.Directory, runID, startedAt)
@@ -107,13 +116,13 @@ func pipelineRunCommand(version string, newState func() *nativeLoaders, modules
}
}
if err := preflight.Validate(invocation.Context, effective.ResolvedPipeline, catalog, engine); err != nil {
return finish(pipelineInvalidConfiguration(err))
return finish(pipelineResourceFailure(err))
}
workingDirectory, err := os.Getwd()
if err != nil {
return finish(commandkit.Failure(err.Error()).WithCause(err))
}
materialized, diagnostics, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{ConfigPath: loaded.MainSourcePath(), WorkingDir: workingDirectory})
materialized, referenceDiagnostics, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{ConfigPath: loaded.MainSourcePath(), WorkingDir: workingDirectory})
if err != nil {
return finish(commandkit.Failure(err.Error()).WithCause(err))
}
@@ -138,7 +147,9 @@ func pipelineRunCommand(version string, newState func() *nativeLoaders, modules
recorder = newRunEvidenceRecorder(capture, result)
}
group := &promptkit.ExecutionGroup{}
output, err = pipeline.New().Run(invocation.Context, pipeline.RunInput{Prepared: prepared, Path: input.InputFile.Value, RawInput: raw, SessionID: sessionID, RunID: runID, StartedAt: startedAt, Diagnostics: diagnostics, Cache: cache, Debug: recorder, Workers: cfg.Concurrency.Workers, Executions: group, Execution: contracts.ExecutionOptions{Execution: input.Overrides.Execution.Execution, TransientRetries: input.Overrides.Execution.TransientRetries}})
_ = invocation.Reporter().Progress(fmt.Sprintf("Running pipeline %q.", effective.ResolvedPipeline.ID))
output, err = pipeline.New().Run(invocation.Context, pipeline.RunInput{Prepared: prepared, Path: input.InputFile.Value, RawInput: raw, SessionID: sessionID, RunID: runID, StartedAt: startedAt, Diagnostics: referenceDiagnostics, Cache: cache, Debug: recorder, Workers: cfg.Concurrency.Workers, Executions: group, Execution: contracts.ExecutionOptions{Execution: input.Overrides.Execution.Execution, TransientRetries: input.Overrides.Execution.TransientRetries}})
reportRunDiagnostics(invocation.Reporter(), output.Diagnostics)
if err != nil {
return finish(commandkit.Failure(err.Error()).WithCause(err))
}
@@ -146,7 +157,11 @@ func pipelineRunCommand(version string, newState func() *nativeLoaders, modules
return finish(commandkit.Failure(err.Error()).WithCause(err))
}
published = true
return finish(commandkit.Success())
outcome := finish(commandkit.Success())
if outcome.Status != 0 {
return outcome
}
return reportRunResult(invocation, runResult{Pipeline: effective.ResolvedPipeline.ID, ArtifactCount: output.ArtifactCount(), OutputDirectory: outputDirectory, DebugDirectory: capture.Path(), FallbackCount: output.FallbackCount})
},
})
}

View File

@@ -31,12 +31,20 @@ type runProbe struct {
parsed int
produced int
correctFirst bool
providerErr error
onGenerate func()
}
func (probe *runProbe) Generate(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
probe.mu.Lock()
defer probe.mu.Unlock()
probe.requests = append(probe.requests, request)
if probe.onGenerate != nil {
probe.onGenerate()
}
if probe.providerErr != nil {
return nil, probe.providerErr
}
value := "accepted"
if request.Prompt.Messages[len(request.Prompt.Messages)-1].Content != "Assess." {
probe.produced++
@@ -98,7 +106,9 @@ func (producer *runProducer) Extract(ctx context.Context, request contracts.Type
return &value, nil, err
}, func(context.Context, []string) (promptkit.RunRequest, error) {
return promptkit.RunRequest{PromptID: "inspect-correction", ProfileID: request.LLMProfile, SessionID: request.SessionID, OnPrepared: producer.probe.onPrepared}, nil
}, nil)
}, func(context.Context, *inspectionArtifact) (inspectionArtifact, error) {
return inspectionArtifact{Value: "fallback"}, nil
})
return contracts.TypedExtractionResult[inspectionArtifact]{Value: value, Outcome: outcome}, err
}
func (producer *runProducer) Merge(_ context.Context, request contracts.TypedMergeRequest[inspectionArtifact]) (contracts.TypedMergeResult[inspectionArtifact], error) {

View File

@@ -1,142 +1,78 @@
package cli
import (
"encoding/json"
"fmt"
"io"
"path/filepath"
"strings"
"text/template"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"go.maximumdirect.net/commandkit"
)
const runResultSchemaVersion = "notarius.run-result.v2"
// runResult is the small completion value. Domain files, not a saved report or
// execution ledger, are the durable deliverables.
type runResult struct {
SchemaVersion string `json:"schema_version"`
RunID string `json:"run_id"`
PipelineID string `json:"pipeline_id"`
OutputDirectory string `json:"output_directory"`
IndexFile string `json:"index_file,omitempty"`
NormalizedOutputCount int `json:"normalized_output_count"`
RejectedOutputCount int `json:"rejected_output_count"`
WarningGroupCount int `json:"warning_group_count"`
WarningOccurrenceCount int `json:"warning_occurrence_count"`
DiagnosticGroupCount int `json:"diagnostic_group_count"`
DiagnosticOccurrenceCount int `json:"diagnostic_occurrence_count"`
DiagnosticsTruncated bool `json:"diagnostics_truncated"`
ValidationStatus string `json:"validation_status"`
ValidationSummaries []artifacts.ValidationSummary `json:"validation_summaries,omitempty"`
DebugDirectory string `json:"debug_directory,omitempty"`
Pipeline string `json:"pipeline"`
ArtifactCount int `json:"artifact_count"`
OutputDirectory string `json:"output_directory"`
DebugDirectory string `json:"debug_directory,omitempty"`
FallbackCount int `json:"fallback_count,omitempty"`
}
func newRunResult(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput, outputDirectory, debugDirectory string) (runResult, error) {
diagnosticProjection, err := contracts.ProjectDiagnosticCollection(output.Diagnostics)
if err != nil {
return runResult{}, fmt.Errorf("summarize run diagnostics: %w", err)
var runResultTemplate = template.Must(commandkit.ParseTextTemplate("run-completion", `{{if .FallbackCount}}Success with fallback: {{else}}Success: {{end}}{{.Pipeline}} produced {{.ArtifactCount}} artifacts in {{.OutputDirectory}}
{{if .FallbackCount}}{{.FallbackCount}} operations used fallback.
{{end}}{{if .DebugDirectory}}Debug capture: {{.DebugDirectory}}
{{end}}`, nil))
func reportRunResult(invocation commandkit.Invocation, result runResult) commandkit.Outcome {
message := "Could not present the completed run." + runLocations(result.OutputDirectory, result.DebugDirectory)
if err := invocation.Reporter().Report(commandkit.Report{Value: result, TextTemplate: runResultTemplate}); err != nil {
// The reporter retains render/write failures in Presentation. They are
// not a replacement operation Cause, and capture is already finalized.
return commandkit.Failure(message)
}
return newRunResultWithDiagnostics(resolved, output, outputDirectory, debugDirectory, diagnosticProjection)
// Deferred diagnostic delivery can still fail during native finalization.
// Supply useful locations without presenting a failure on successful runs.
return commandkit.Success().WithExplanation(message)
}
func newRunResultWithDiagnostics(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput, outputDirectory, debugDirectory string, diagnosticProjection contracts.DiagnosticProjection) (runResult, error) {
if strings.TrimSpace(output.Manifest.RunID) == "" {
return runResult{}, fmt.Errorf("run result requires a run ID")
func runLocations(outputDirectory, debugDirectory string) string {
var locations string
if outputDirectory != "" {
locations += "\nArtifacts: " + outputDirectory
}
if strings.TrimSpace(resolved.ID) == "" {
return runResult{}, fmt.Errorf("run result requires a resolved pipeline ID")
}
if strings.TrimSpace(output.Manifest.PipelineID) == "" {
return runResult{}, fmt.Errorf("run result requires a manifest pipeline ID")
}
if output.Manifest.PipelineID != resolved.ID {
return runResult{}, fmt.Errorf("run result pipeline ID does not match resolved pipeline")
}
if strings.TrimSpace(output.Manifest.ValidationStatus) == "" {
return runResult{}, fmt.Errorf("run result requires a validation status")
}
if strings.TrimSpace(outputDirectory) == "" {
return runResult{}, fmt.Errorf("run result requires an output directory")
if debugDirectory != "" {
locations += "\nDebug capture: " + debugDirectory
}
return locations
}
absOutputDirectory, err := filepath.Abs(outputDirectory)
if err != nil {
return runResult{}, fmt.Errorf("make output directory absolute: %w", err)
}
result := runResult{
SchemaVersion: runResultSchemaVersion,
RunID: output.Manifest.RunID,
PipelineID: output.Manifest.PipelineID,
OutputDirectory: absOutputDirectory,
NormalizedOutputCount: len(output.NormalizeOutputs),
RejectedOutputCount: len(output.Rejected),
WarningGroupCount: len(diagnosticProjection.Warnings),
WarningOccurrenceCount: diagnosticProjection.WarningOccurrenceCount,
DiagnosticGroupCount: len(diagnosticProjection.Diagnostics),
DiagnosticOccurrenceCount: diagnosticProjection.DiagnosticOccurrenceCount,
DiagnosticsTruncated: output.Diagnostics.Truncated,
ValidationStatus: output.Manifest.ValidationStatus,
ValidationSummaries: cloneValidationSummaries(output.Manifest.ValidationSummaries),
}
if strings.TrimSpace(debugDirectory) != "" {
absDebugDirectory, err := filepath.Abs(debugDirectory)
if err != nil {
return runResult{}, fmt.Errorf("make debug directory absolute: %w", err)
}
result.DebugDirectory = absDebugDirectory
}
if resolved.Output.Module == pipeline.DefaultOutputModule {
indexCount := 0
for _, file := range output.OutputFiles {
if file.Name == "index.json" {
indexCount++
func reportRunDiagnostics(reporter *commandkit.Reporter, collection contracts.DiagnosticCollection) {
for _, group := range collection.Groups {
origin := group.Origin
scope := []string{string(origin.Stage)}
for _, value := range []string{origin.StepID, origin.LaneID, origin.ModuleKey, origin.ValidatorKey} {
if value != "" {
scope = append(scope, value)
}
}
if indexCount != 1 {
return runResult{}, fmt.Errorf("production JSON output must contain exactly one index.json file")
heading := strings.Join(scope, " / ") + ": " + group.ReasonCode
detail := func(writer io.Writer) error {
for _, sample := range group.Samples {
if _, err := fmt.Fprintf(writer, "%s: %s\n", sample.Scope, sample.Message); err != nil {
return err
}
}
return nil
}
result.IndexFile = "index.json"
}
return result, nil
}
func cloneValidationSummaries(summaries []artifacts.ValidationSummary) []artifacts.ValidationSummary {
if len(summaries) == 0 {
return nil
}
cloned := make([]artifacts.ValidationSummary, len(summaries))
for index, summary := range summaries {
cloned[index] = artifacts.CloneValidationSummary(summary)
}
return cloned
}
func encodeRunResult(result runResult) ([]byte, error) {
encoded, err := json.Marshal(result)
if err != nil {
return nil, fmt.Errorf("encode run result: %w", err)
}
return append(encoded, '\n'), nil
}
func writeRunResult(writer io.Writer, content []byte) error {
for len(content) > 0 {
written, err := writer.Write(content)
if written < 0 || written > len(content) {
return io.ErrShortWrite
}
content = content[written:]
if err != nil {
return err
}
if written == 0 {
return io.ErrShortWrite
if group.Disposition == contracts.DiagnosticDispositionWarning {
_ = reporter.AtEnd().Warn(heading, detail)
} else {
_ = reporter.AtEnd().Info(heading, detail)
}
}
return nil
if collection.Truncated {
_ = reporter.AtEnd().Info("Additional pipeline diagnostics were omitted.")
}
}

View File

@@ -2,198 +2,318 @@ package cli
import (
"bytes"
"context"
"encoding/json"
"errors"
"io"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
alwaysreject "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/validate/always_reject"
"go.maximumdirect.net/commandkit"
"go.maximumdirect.net/configkit"
)
func TestMaintainedMinimalInvocationEmitsRunResult(t *testing.T) {
outputRoot := filepath.Join(t.TempDir(), "output")
var stdout, stderr strings.Builder
code := RunWithOptions([]string{
"run", "dnd-session",
"--config", repositoryPath("examples", "dnd-minimal.config.yml"),
"--input", repositoryPath("examples", "seriatim-minimal-transcript.json"),
"--only", "spells", "--chunk_cache", "bypass", "--output-dir", outputRoot, "--json",
}, &stdout, &stderr, productionRunOptions(t, &productionFakeLLMClient{}))
if code != 0 || stderr.Len() != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
receipt := decodeRunResultDocument(t, stdout.String())
if got := receipt["schema_version"]; got != "notarius.run-result.v2" {
t.Fatalf("schema_version = %q", got)
}
if got := receipt["run_id"]; got != productionRunID {
t.Fatalf("run_id = %q", got)
}
if got := receipt["pipeline_id"]; got != "dnd-session" {
t.Fatalf("pipeline_id = %q", got)
}
if got := receipt["index_file"]; got != "index.json" {
t.Fatalf("index_file = %q", got)
}
if got := receipt["normalized_output_count"]; got != float64(1) {
t.Fatalf("normalized_output_count = %v", got)
}
if got := receipt["rejected_output_count"]; got != float64(0) {
t.Fatalf("rejected_output_count = %v", got)
}
if got := receipt["warning_group_count"]; got != float64(0) || receipt["warning_occurrence_count"] != float64(0) || receipt["diagnostic_group_count"] != float64(0) || receipt["diagnostic_occurrence_count"] != float64(0) || receipt["diagnostics_truncated"] != false {
t.Fatalf("diagnostic counts = %#v", receipt)
}
if got := receipt["validation_status"]; got != "approved" {
t.Fatalf("validation_status = %q", got)
}
outputDirectory, ok := receipt["output_directory"].(string)
if !ok || !filepath.IsAbs(outputDirectory) || outputDirectory != filepath.Join(outputRoot, productionRunID) {
t.Fatalf("output_directory = %q", receipt["output_directory"])
}
indexFile := receipt["index_file"].(string)
assertFile(t, filepath.Join(outputDirectory, indexFile))
}
func TestRunResultReportsWarningsAndDebugBundle(t *testing.T) {
roots := newStateTestRoots(t)
harness := newStateTestHarness()
harness.chunkDiagnostics = []contracts.ProducerDiagnostic{stateTestDiagnostic("chunk", "contract-warning", "warning retained")}
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{
"run", "sample", "--config", roots.config, "--input", roots.input,
"--chunk_cache", "bypass", "--debug", "--json",
}, &stdout, &stderr, harness.options())
if code != 0 || !strings.Contains(stderr.String(), "1 warning group(s), 1 occurrence(s)") || strings.Contains(stderr.String(), "warnings.json") {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
receipt := decodeRunResultDocument(t, stdout.String())
if got := receipt["warning_group_count"]; got != float64(1) || receipt["warning_occurrence_count"] != float64(1) || receipt["diagnostic_group_count"] != float64(0) || receipt["diagnostic_occurrence_count"] != float64(0) || receipt["diagnostics_truncated"] != false {
t.Fatalf("diagnostic counts = %#v", receipt)
}
debugDirectory, ok := receipt["debug_directory"].(string)
if !ok || !filepath.IsAbs(debugDirectory) || debugDirectory != onlyChildDir(t, roots.debug) {
t.Fatalf("debug_directory = %q", receipt["debug_directory"])
}
if strings.Contains(stdout.String(), "complete:") || strings.Contains(stdout.String(), "debug=") {
t.Fatalf("machine stdout contains human reporting: %q", stdout.String())
}
}
func TestRunResultReportsSuccessfulRejection(t *testing.T) {
roots := newStateTestRoots(t)
configBytes, err := os.ReadFile(roots.config)
if err != nil {
t.Fatal(err)
}
configBytes = []byte(replaceRequiredOnce(t, string(configBytes), " normalize:\n module: test/normalize\n", " normalize:\n module: test/normalize\n validators:\n - module: generic/always_reject\n validation_policy:\n semantic_rejection: reject_output\n"))
if err := os.WriteFile(roots.config, configBytes, 0o600); err != nil {
t.Fatal(err)
}
harness := newStateTestHarness()
opts := harness.options()
if err := alwaysreject.RegisterTyped[stateTestArtifact](opts.Registries.Validators, stateTestArtifactKind); err != nil {
t.Fatal(err)
}
opts.Catalog = catalogFromRegistries(opts.Registries)
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{
"run", "sample", "--config", roots.config, "--input", roots.input,
"--chunk_cache", "bypass", "--json",
}, &stdout, &stderr, opts)
if code != 0 || stderr.Len() != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
receipt := decodeRunResultDocument(t, stdout.String())
if got := receipt["normalized_output_count"]; got != float64(0) {
t.Fatalf("normalized_output_count = %v", got)
}
if got := receipt["rejected_output_count"]; got != float64(1) {
t.Fatalf("rejected_output_count = %v", got)
}
if got := receipt["validation_status"]; got != "rejected" {
t.Fatalf("validation_status = %q", got)
}
}
func TestRunResultIsAbsentForSyntaxAndRuntimeFailures(t *testing.T) {
t.Run("syntax", func(t *testing.T) {
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"run", "sample", "--json"}, &stdout, &stderr, newStateTestHarness().options())
if code != 2 || stdout.Len() != 0 || stderr.Len() == 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
})
t.Run("runtime", func(t *testing.T) {
roots := newStateTestRoots(t)
harness := newStateTestHarness()
harness.extractErr = errors.New("injected extraction failure")
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{
"run", "sample", "--config", roots.config, "--input", roots.input,
"--chunk_cache", "bypass", "--json",
}, &stdout, &stderr, harness.options())
if code != 1 || stdout.Len() != 0 || stderr.Len() == 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
})
}
func TestRunResultDeliveryFailureRetainsPublishedBundles(t *testing.T) {
roots := newStateTestRoots(t)
writerErr := errors.New("result writer sentinel")
stdout := &resultDeliveryWriter{err: writerErr}
var stderr bytes.Buffer
code := RunWithOptions([]string{
"run", "sample", "--config", roots.config, "--input", roots.input,
"--chunk_cache", "bypass", "--debug", "--json",
}, stdout, &stderr, newStateTestHarness().options())
if code != 1 || !strings.Contains(stderr.String(), "write run result") || strings.Contains(stderr.String(), writerErr.Error()) {
t.Fatalf("code=%d stderr=%q", code, stderr.String())
}
if stdout.accepted.Len() != 0 {
t.Fatalf("accepted stdout = %q", stdout.accepted.String())
}
assertStateTestOutput(t, roots.output)
debugBundle := onlyChildDir(t, roots.debug)
report := readStateTestRunReport(t, debugBundle)
if !report.Succeeded {
t.Fatalf("debug report = %#v, want successful persisted run", report)
}
if strings.Contains(readAllFiles(t, debugBundle), writerErr.Error()) {
t.Fatalf("debug bundle contains result writer error")
}
}
func decodeRunResultDocument(t *testing.T, stdout string) map[string]any {
func decodeCompletionReport(t *testing.T, format commandkit.Format, content []byte) map[string]any {
t.Helper()
if strings.Count(stdout, "\n") != 1 {
t.Fatalf("stdout = %q, want one JSON document", stdout)
if format == commandkit.FormatYAML {
value, err := commandkit.DecodeYAMLReport(bytes.NewReader(content))
if err != nil {
t.Fatalf("decode native YAML report: %v\n%s", err, content)
}
content, err = json.Marshal(value)
if err != nil {
t.Fatal(err)
}
}
var receipt map[string]any
if err := json.Unmarshal([]byte(stdout), &receipt); err != nil {
t.Fatalf("decode run result: %v; stdout=%q", err, stdout)
var report map[string]any
if err := json.Unmarshal(content, &report); err != nil {
t.Fatalf("decode native report: %v\n%s", err, content)
}
return receipt
return report
}
type resultDeliveryWriter struct {
err error
accepted bytes.Buffer
func assertCompletedRunFiles(t *testing.T, file string, captured bool) {
t.Helper()
run := filepath.Join(filepath.Dir(file), "output-must-not-exist", "run-1")
if content, err := os.ReadFile(filepath.Join(run, "lanes", "notes.json")); err != nil || len(content) == 0 {
t.Fatalf("published domain artifact: %s %v", content, err)
}
files, err := os.ReadDir(run)
if err != nil || len(files) != 1 || files[0].Name() != "lanes" {
t.Fatalf("unexpected reporting sidecars: %#v %v", files, err)
}
if captured {
debug := filepath.Join(filepath.Dir(file), "debug-must-not-exist", "run-1")
content, err := os.ReadFile(filepath.Join(debug, "outcome.json"))
if err != nil {
t.Fatal(err)
}
var terminal struct {
Outcome struct {
Published bool `json:"published"`
} `json:"outcome"`
Error string `json:"error"`
CaptureError string `json:"capture_error"`
}
if err := json.Unmarshal(content, &terminal); err != nil || !terminal.Outcome.Published || terminal.Error != "" || terminal.CaptureError != "" {
t.Fatalf("capture was not completed before reporting: %s %v", content, err)
}
}
}
func (w *resultDeliveryWriter) Write(content []byte) (int, error) {
if w.err != nil {
return 0, w.err
}
return w.accepted.Write(content)
type completedRunWriter struct {
t *testing.T
file string
captured bool
content bytes.Buffer
}
func (writer *completedRunWriter) Write(content []byte) (int, error) {
assertCompletedRunFiles(writer.t, writer.file, writer.captured)
return writer.content.Write(content)
}
func TestNativeRunCompletionFormatsFollowPublicationAndCapture(t *testing.T) {
for _, format := range []commandkit.Format{commandkit.FormatText, commandkit.FormatJSON, commandkit.FormatYAML} {
t.Run(string(format), func(t *testing.T) {
probe := &runProbe{}
app, file, input, _ := runFixture(t, probe)
output := &completedRunWriter{t: t, file: file, captured: true}
var diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--format", string(format)}, commandkit.Streams{Out: output, Err: &diagnostics})
if outcome.Status != 0 || !strings.Contains(diagnostics.String(), "Running pipeline") {
t.Fatalf("completion: %#v stderr=%s", outcome, diagnostics.String())
}
directory := filepath.Join(filepath.Dir(file), "output-must-not-exist", "run-1")
debug := filepath.Join(filepath.Dir(file), "debug-must-not-exist", "run-1")
if format == commandkit.FormatText {
if !strings.Contains(output.content.String(), "Success: demo produced 1 artifacts in "+directory) || !strings.Contains(output.content.String(), "Debug capture: "+debug) || strings.Contains(output.content.String(), "fallback") {
t.Fatalf("native completion text: %s", output.content.String())
}
} else {
report := decodeCompletionReport(t, format, output.content.Bytes())
if len(report) != 4 || report["pipeline"] != "demo" || report["artifact_count"] != float64(1) || report["output_directory"] != directory || report["debug_directory"] != debug {
t.Fatalf("native report: %#v", report)
}
}
if probe.produced != 1 {
t.Fatalf("reporting reran a producer: %d", probe.produced)
}
})
}
}
func TestNativeRunDisclosesFallbackIndependentlyOfDiagnosticVisibility(t *testing.T) {
for _, format := range []commandkit.Format{commandkit.FormatText, commandkit.FormatJSON, commandkit.FormatYAML} {
for _, verbose := range []bool{false, true} {
probe := &runProbe{correctFirst: true}
app, file, input, _ := runFixture(t, probe)
args := []string{"pipeline", "run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--set", `/pipelines/demo/mode="permissive"`, "--set", "/pipelines/demo/max_additional_turns=0", "--format", string(format)}
if verbose {
args = append(args, "--verbose")
}
var output, diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: &diagnostics})
if outcome.Status != 0 || probe.produced != 1 {
t.Fatalf("permissive completion: %#v %s", outcome, diagnostics.String())
}
if verbose && !strings.Contains(diagnostics.String(), "WARN:") || !verbose && diagnostics.Len() != 0 {
t.Fatalf("native warning visibility: %s", diagnostics.String())
}
if format == commandkit.FormatText {
if !strings.Contains(output.String(), "Success with fallback:") || !strings.Contains(output.String(), "1 operations used fallback.") {
t.Fatalf("fallback text: %s", output.String())
}
} else {
report := decodeCompletionReport(t, format, output.Bytes())
if report["fallback_count"] != float64(1) || report["artifact_count"] != float64(1) || len(report) != 4 {
t.Fatalf("fallback report: %#v", report)
}
}
assertCompletedRunFiles(t, file, false)
}
}
}
func TestNativeRunQuietAndDebugUseNativeModeOrdering(t *testing.T) {
for _, test := range []struct {
flags []string
quiet bool
progress bool
}{
{[]string{"--debug", "--quiet"}, true, false},
{[]string{"--quiet", "--debug"}, false, true},
{[]string{"--debug", "--progress=false"}, false, false},
} {
probe := &runProbe{}
app, file, input, _ := runFixture(t, probe)
var output, diagnostics bytes.Buffer
args := append([]string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--format", "json"}, test.flags...)
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: &diagnostics})
if outcome.Status != 0 || test.quiet != (output.Len() == 0) || test.progress != strings.Contains(diagnostics.String(), "Running pipeline") {
t.Fatalf("native mode order: %#v output=%s stderr=%s", outcome, output.String(), diagnostics.String())
}
assertCompletedRunFiles(t, file, true)
}
}
func TestNativeRunCancellationKeepsItsCauseAndPublishesNothing(t *testing.T) {
probe := &runProbe{}
app, file, input, _ := runFixture(t, probe)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
probe.onGenerate = cancel
var output, diagnostics bytes.Buffer
outcome := app.Execute(ctx, []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--quiet"}, commandkit.Streams{Out: &output, Err: &diagnostics})
if outcome.Status != 1 || !errors.Is(outcome.Cause, context.Canceled) || output.Len() != 0 || diagnostics.Len() == 0 {
t.Fatalf("cancellation: %#v stdout=%s stderr=%s", outcome, output.String(), diagnostics.String())
}
if _, err := os.Stat(filepath.Join(filepath.Dir(file), "output-must-not-exist")); !os.IsNotExist(err) {
t.Fatalf("cancellation published artifacts: %v", err)
}
}
func TestNativeRunPrimaryFailureSurvivesCaptureAndDiagnosticFailures(t *testing.T) {
primary := errors.New("provider failure sentinel")
delivery := errors.New("diagnostic failure sentinel")
probe := &runProbe{providerErr: primary}
app, file, input, _ := runFixture(t, probe)
debug := filepath.Join(filepath.Dir(file), "debug-must-not-exist", "run-1")
probe.onGenerate = func() {
if err := os.Mkdir(filepath.Join(debug, "outcome.json"), 0700); err != nil {
t.Error(err)
}
}
var output bytes.Buffer
outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--llm-transient-retries=0"}, commandkit.Streams{Out: &output, Err: failingResultWriter{cause: delivery}})
var captureFailure *os.LinkError
if outcome.Status != 1 || !errors.Is(outcome.Cause, primary) || !errors.As(outcome.Cause, &captureFailure) || errors.Is(outcome.Cause, delivery) || !errors.Is(outcome.Presentation.Diagnostics.WriteError, delivery) || output.Len() != 0 || len(probe.requests) != 1 {
t.Fatalf("primary/capture/delivery failure: %#v", outcome)
}
if !strings.Contains(outcome.Explanation.Message, debug) {
t.Fatalf("failed run capture location unavailable: %#v", outcome)
}
}
func TestNativeRunReportDeliveryFailuresRetainArtifactsAndNativePresentation(t *testing.T) {
sentinel := errors.New("result delivery sentinel")
for _, partial := range []bool{false, true} {
probe := &runProbe{}
app, file, input, _ := runFixture(t, probe)
var output io.Writer = failingResultWriter{cause: sentinel}
prefix := &partialResultWriter{cause: sentinel}
if partial {
output = prefix
}
var diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--format", "json"}, commandkit.Streams{Out: output, Err: &diagnostics})
if outcome.Status != 1 || outcome.Cause != nil || !errors.Is(outcome.Presentation.Output.WriteError, sentinel) || outcome.Presentation.Output.State != commandkit.DeliveryIncomplete || probe.produced != 1 || partial && prefix.content.Len() == 0 {
t.Fatalf("delivery failure: %#v %s", outcome, diagnostics.String())
}
assertCompletedRunFiles(t, file, true)
for _, root := range []string{"output-must-not-exist", "debug-must-not-exist"} {
if !strings.Contains(diagnostics.String(), filepath.Join(filepath.Dir(file), root, "run-1")) {
t.Fatalf("missing available path after delivery failure: %s", diagnostics.String())
}
}
}
}
func TestNativeRunDiagnosticDeliveryFailureIsNotAnOperationFailure(t *testing.T) {
sentinel := errors.New("diagnostic delivery sentinel")
probe := &runProbe{correctFirst: true}
app, file, input, _ := runFixture(t, probe)
var output bytes.Buffer
outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--set", `/pipelines/demo/mode="permissive"`, "--set", "/pipelines/demo/max_additional_turns=0", "--format", "json", "--verbose"}, commandkit.Streams{Out: &output, Err: failingResultWriter{cause: sentinel}})
if outcome.Status != 1 || outcome.Cause != nil || !errors.Is(outcome.Presentation.Diagnostics.WriteError, sentinel) || outcome.Presentation.Diagnostics.State != commandkit.DeliveryIncomplete {
t.Fatalf("diagnostic delivery: %#v", outcome)
}
if report := decodeCompletionReport(t, commandkit.FormatJSON, output.Bytes()); report["fallback_count"] != float64(1) {
t.Fatalf("completed report was replaced by a delivery envelope: %#v", report)
}
if !strings.Contains(outcome.Explanation.Message, filepath.Join(filepath.Dir(file), "output-must-not-exist", "run-1")) {
t.Fatalf("presentation failure lost available artifacts: %#v", outcome)
}
assertCompletedRunFiles(t, file, false)
}
func TestNativeRunCaptureFailureAfterPublicationDoesNotReportSuccess(t *testing.T) {
probe := &runProbe{}
app, file, input, _ := runFixture(t, probe)
var once sync.Once
debug := filepath.Join(filepath.Dir(file), "debug-must-not-exist", "run-1")
probe.onGenerate = func() {
once.Do(func() {
// The external provider boundary runs after capture allocation but
// before terminalization. Make only the terminal file unusable.
if err := os.Mkdir(filepath.Join(debug, "outcome.json"), 0700); err != nil {
t.Fatal(err)
}
})
}
var output, diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--format", "json"}, commandkit.Streams{Out: &output, Err: &diagnostics})
if outcome.Status != 1 || outcome.Cause == nil || output.Len() != 0 || probe.produced != 1 {
t.Fatalf("post-publication capture failure: %#v stdout=%s stderr=%s", outcome, output.String(), diagnostics.String())
}
assertCompletedRunFiles(t, file, false)
if !strings.Contains(diagnostics.String(), debug) || !strings.Contains(diagnostics.String(), filepath.Join(filepath.Dir(file), "output-must-not-exist", "run-1")) {
t.Fatalf("post-publication locations: %s", diagnostics.String())
}
if _, err := os.Stat(filepath.Join(debug, "executions.json")); err != nil {
t.Fatalf("existing capture was removed: %v", err)
}
}
func TestNativeRunFailureCategoriesPreserveCausesWithoutSuccessReports(t *testing.T) {
for _, test := range []struct {
name string
configure func(*testing.T, *runProbe, string, string) []string
status int
cause error
}{
{"missing configuration", func(t *testing.T, _ *runProbe, _, input string) []string {
return []string{"run", "demo", "--input-file", input, "--config", filepath.Join(t.TempDir(), "missing.yml")}
}, 1, nil},
{"invalid configuration", func(t *testing.T, _ *runProbe, file, input string) []string {
return []string{"run", "demo", "--config", file, "--input-file", input, "--set", "/version=5"}
}, 2, nil},
{"unknown pipeline", func(_ *testing.T, _ *runProbe, file, input string) []string {
return []string{"run", "missing", "--config", file, "--input-file", input}
}, 2, nil},
{"missing input", func(t *testing.T, _ *runProbe, file, _ string) []string {
return []string{"run", "demo", "--config", file, "--input-file", filepath.Join(t.TempDir(), "missing.txt")}
}, 1, os.ErrNotExist},
{"publication failure", func(t *testing.T, _ *runProbe, file, input string) []string {
path := filepath.Join(t.TempDir(), "occupied")
if err := os.WriteFile(path, []byte("keep"), 0600); err != nil {
t.Fatal(err)
}
return []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--output-dir", path}
}, 1, nil},
{"provider failure", func(_ *testing.T, probe *runProbe, file, input string) []string {
probe.providerErr = io.ErrUnexpectedEOF
return []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--llm-transient-retries=0"}
}, 1, io.ErrUnexpectedEOF},
} {
t.Run(test.name, func(t *testing.T) {
probe := &runProbe{}
app, file, input, _ := runFixture(t, probe)
var output, diagnostics bytes.Buffer
args := append(test.configure(t, probe, file, input), "--quiet", "--format", "json")
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: &diagnostics})
if outcome.Status != test.status || test.cause != nil && !errors.Is(outcome.Cause, test.cause) || output.Len() != 0 || diagnostics.Len() == 0 {
t.Fatalf("failure category: %#v stdout=%s stderr=%s", outcome, output.String(), diagnostics.String())
}
if test.name == "missing configuration" || test.name == "invalid configuration" {
var load *configkit.LoadError
if !errors.As(outcome.Cause, &load) {
t.Fatalf("lost native load cause: %#v", outcome)
}
}
})
}
}

View File

@@ -2,229 +2,136 @@ package cli
import (
"bytes"
"encoding/json"
"context"
"errors"
"io"
"path/filepath"
"strings"
"testing"
"text/template"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"go.maximumdirect.net/commandkit"
)
func TestRunResultEncodesRequiredFieldsAndCounts(t *testing.T) {
result, err := newRunResult(testResolvedPipeline(pipeline.DefaultOutputModule), testRunOutput(), "relative-output", "relative-debug")
if err != nil {
t.Fatal(err)
}
encoded, err := encodeRunResult(result)
if err != nil {
t.Fatal(err)
}
if encoded[len(encoded)-1] != '\n' || bytes.Count(encoded, []byte{'\n'}) != 1 {
t.Fatalf("encoded result is not one newline-terminated object: %q", encoded)
}
var decoded map[string]any
if err := json.Unmarshal(encoded, &decoded); err != nil {
t.Fatal(err)
}
if got := decoded["schema_version"]; got != runResultSchemaVersion {
t.Fatalf("schema_version = %q", got)
}
if got := decoded["run_id"]; got != "run-123" {
t.Fatalf("run_id = %q", got)
}
if got := decoded["pipeline_id"]; got != "sample" {
t.Fatalf("pipeline_id = %q", got)
}
if got := decoded["validation_status"]; got != "rejected" {
t.Fatalf("validation_status = %q", got)
}
if got := decoded["index_file"]; got != "index.json" {
t.Fatalf("index_file = %q", got)
}
if got := decoded["normalized_output_count"]; got != float64(2) {
t.Fatalf("normalized_output_count = %v", got)
}
if got := decoded["rejected_output_count"]; got != float64(1) {
t.Fatalf("rejected_output_count = %v", got)
}
if got := decoded["warning_group_count"]; got != float64(1) || decoded["warning_occurrence_count"] != float64(1) || decoded["diagnostic_group_count"] != float64(0) || decoded["diagnostic_occurrence_count"] != float64(0) || decoded["diagnostics_truncated"] != false {
t.Fatalf("diagnostic counts = %#v", decoded)
}
if got := decoded["validation_summaries"]; got != nil {
t.Fatalf("validation_summaries = %#v, want omitted when empty", got)
}
if got := decoded["output_directory"]; got != filepath.Join(mustWorkingDirectory(t), "relative-output") {
t.Fatalf("output_directory = %q", got)
}
if got := decoded["debug_directory"]; got != filepath.Join(mustWorkingDirectory(t), "relative-debug") {
t.Fatalf("debug_directory = %q", got)
}
}
func TestRunResultRejectsInvalidRequiredValues(t *testing.T) {
tests := []struct {
name string
resolved pipeline.ResolvedPipeline
output pipeline.RunOutput
directory string
}{
{name: "blank run ID", resolved: testResolvedPipeline(pipeline.DefaultOutputModule), output: testRunOutputWithout(func(output *pipeline.RunOutput) { output.Manifest.RunID = " " }), directory: "output"},
{name: "blank resolved pipeline ID", resolved: pipeline.ResolvedPipeline{Output: pipeline.ModuleBinding{Module: pipeline.DefaultOutputModule}}, output: testRunOutput(), directory: "output"},
{name: "blank manifest pipeline ID", resolved: testResolvedPipeline(pipeline.DefaultOutputModule), output: testRunOutputWithout(func(output *pipeline.RunOutput) { output.Manifest.PipelineID = "" }), directory: "output"},
{name: "mismatched pipeline IDs", resolved: testResolvedPipeline(pipeline.DefaultOutputModule), output: testRunOutputWithout(func(output *pipeline.RunOutput) { output.Manifest.PipelineID = "other" }), directory: "output"},
{name: "blank validation status", resolved: testResolvedPipeline(pipeline.DefaultOutputModule), output: testRunOutputWithout(func(output *pipeline.RunOutput) { output.Manifest.ValidationStatus = " " }), directory: "output"},
{name: "blank output directory", resolved: testResolvedPipeline(pipeline.DefaultOutputModule), output: testRunOutput(), directory: " "},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if _, err := newRunResult(tt.resolved, tt.output, tt.directory, ""); err == nil {
t.Fatal("newRunResult() succeeded")
func TestRunResultUsesOneNativeValueAcrossFormats(t *testing.T) {
value := runResult{Pipeline: "demo", ArtifactCount: 3, OutputDirectory: "/artifacts/run-one", DebugDirectory: "/debug/run-one", FallbackCount: 2}
for _, format := range []commandkit.Format{commandkit.FormatText, commandkit.FormatJSON, commandkit.FormatYAML} {
var output bytes.Buffer
if err := commandkit.RenderReport(&output, commandkit.Report{Value: value, TextTemplate: runResultTemplate}, format); err != nil {
t.Fatal(err)
}
if format == commandkit.FormatText {
if !strings.Contains(output.String(), "Success with fallback: demo produced 3 artifacts in /artifacts/run-one") || !strings.Contains(output.String(), "2 operations used fallback.") || !strings.Contains(output.String(), "Debug capture: /debug/run-one") {
t.Fatalf("fallback text: %s", output.String())
}
})
continue
}
report := decodeCompletionReport(t, format, output.Bytes())
if report["pipeline"] != "demo" || report["output_directory"] != value.OutputDirectory || report["debug_directory"] != value.DebugDirectory || report["fallback_count"] != float64(2) || report["artifact_count"] != float64(3) || len(report) != 5 {
t.Fatalf("completion value: %#v", report)
}
}
}
func TestRunResultOmitsIndexFileForOtherOutputModules(t *testing.T) {
result, err := newRunResult(testResolvedPipeline("test/output"), testRunOutputWithout(func(output *pipeline.RunOutput) {
output.OutputFiles = nil
}), "output", "")
func TestNativeCompletionDiagnosticsRetainDomainClassification(t *testing.T) {
collection := contracts.DiagnosticCollection{}
for _, disposition := range []contracts.DiagnosticDisposition{contracts.DiagnosticDispositionWarning, contracts.DiagnosticDispositionAdvisory, contracts.DiagnosticDispositionObservation} {
category := contracts.DiagnosticCategoryDataQuality
if disposition == contracts.DiagnosticDispositionWarning {
category = contracts.DiagnosticCategoryFallback
} else if disposition == contracts.DiagnosticDispositionObservation {
category = contracts.DiagnosticCategoryNormalization
}
collection.Groups = append(collection.Groups, contracts.DiagnosticGroup{Disposition: disposition, Category: category, ReasonCode: string(disposition), Origin: contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract, StepID: "build", LaneID: "notes", ModuleKey: "producer"}, OccurrenceCount: 1, Samples: []contracts.DiagnosticSample{{Scope: "current source", Message: string(disposition) + " finding"}}})
}
if err := collection.Validate(); err != nil {
t.Fatal(err)
}
var output, diagnostics bytes.Buffer
app, err := commandkit.New(commandkit.Identity{Name: "fixture"}, commandkit.NewCommand(commandkit.CommandSpec[struct{}]{Name: "report", Run: func(invocation commandkit.Invocation, _ struct{}) commandkit.Outcome {
reportRunDiagnostics(invocation.Reporter(), collection)
return reportRunResult(invocation, runResult{Pipeline: "demo", ArtifactCount: 1, OutputDirectory: "/artifacts/run-one"})
}}))
if err != nil {
t.Fatal(err)
}
if result.IndexFile != "" {
t.Fatalf("index_file = %q", result.IndexFile)
outcome := app.Execute(context.Background(), []string{"report", "--verbose"}, commandkit.Streams{Out: &output, Err: &diagnostics})
if outcome.Status != 0 || strings.Contains(output.String(), "finding") || strings.Count(diagnostics.String(), "WARN:") != 1 {
t.Fatalf("diagnostic classification: %#v stdout=%s stderr=%s", outcome, output.String(), diagnostics.String())
}
encoded, err := encodeRunResult(result)
if err != nil {
t.Fatal(err)
}
var decoded map[string]any
if err := json.Unmarshal(encoded, &decoded); err != nil {
t.Fatal(err)
}
if _, ok := decoded["index_file"]; ok {
t.Fatalf("encoded non-JSON result contains index_file: %s", encoded)
}
if _, ok := decoded["debug_directory"]; ok {
t.Fatalf("encoded result without debug capture contains debug_directory: %s", encoded)
for _, group := range collection.Groups {
if !strings.Contains(diagnostics.String(), string(group.Disposition)+" finding") {
t.Fatalf("missing diagnostic: %s", diagnostics.String())
}
}
}
func TestRunResultProjectsOwnedValidationSummaries(t *testing.T) {
output := testRunOutput()
output.Manifest.ValidationSummaries = []artifacts.ValidationSummary{{Status: "incomplete", IncompleteValidators: []string{"validator"}, ProducerAttemptCount: 1, TerminalAction: "warn_continue"}}
result, err := newRunResult(testResolvedPipeline(pipeline.DefaultOutputModule), output, "output", "")
if err != nil {
t.Fatal(err)
}
output.Manifest.ValidationSummaries[0].IncompleteValidators[0] = "caller mutation"
if got := result.ValidationSummaries[0].IncompleteValidators; len(got) != 1 || got[0] != "validator" {
t.Fatalf("result validation summaries = %#v", result.ValidationSummaries)
}
encoded, err := encodeRunResult(result)
if err != nil {
t.Fatal(err)
}
if !bytes.Contains(encoded, []byte(`"validation_summaries":[{"status":"incomplete","incomplete_validators":["validator"],"producer_attempt_count":1,"terminal_action":"warn_continue"}]`)) {
t.Fatalf("encoded result = %s", encoded)
func TestCompletedRunRenderingFailureRetainsPublicationEvenWhenQuiet(t *testing.T) {
previous := runResultTemplate
t.Cleanup(func() { runResultTemplate = previous })
// Simulate a broken application-owned text template, not a new renderer seam.
runResultTemplate = template.Must(commandkit.ParseTextTemplate("invalid", "{{.MissingField}}", nil))
for _, quiet := range []bool{false, true} {
probe := &runProbe{}
app, file, input, _ := runFixture(t, probe)
args := []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug"}
if quiet {
args = append(args, "--quiet")
}
var output, diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: &diagnostics})
if outcome.Status != 1 || outcome.Cause != nil || outcome.Presentation.Output.RenderError == nil || outcome.Presentation.Output.State != commandkit.DeliveryIncomplete || probe.produced != 1 {
t.Fatalf("render failure: %#v %s", outcome, diagnostics.String())
}
assertCompletedRunFiles(t, file, true)
artifacts := filepath.Join(filepath.Dir(file), "output-must-not-exist", "run-1")
if !strings.Contains(diagnostics.String(), artifacts) || (quiet && output.Len() != 0) {
t.Fatalf("render failure locations/quiet: %s", diagnostics.String())
}
}
}
func TestRunResultRequiresOneProductionIndexFile(t *testing.T) {
tests := []struct {
name string
files []contracts.OutputFile
}{
{name: "missing", files: nil},
{name: "duplicate", files: []contracts.OutputFile{{Name: "index.json"}, {Name: "index.json"}}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
output := testRunOutput()
output.OutputFiles = tt.files
if _, err := newRunResult(testResolvedPipeline(pipeline.DefaultOutputModule), output, "output", ""); err == nil {
t.Fatal("newRunResult() succeeded")
}
})
}
type failingResultWriter struct {
cause error
}
func TestWriteRunResultCompletesAndReportsWriterFailure(t *testing.T) {
content := []byte("result\n")
var target bytes.Buffer
if err := writeRunResult(partialResultWriter{writer: &target, limit: 2}, content); err != nil {
t.Fatal(err)
}
if got := target.String(); got != string(content) {
t.Fatalf("written result = %q", got)
}
writerErr := errors.New("result writer failed")
if err := writeRunResult(failingResultWriter{err: writerErr}, content); !errors.Is(err, writerErr) {
t.Fatalf("writeRunResult() error = %v", err)
}
if err := writeRunResult(zeroResultWriter{}, content); !errors.Is(err, io.ErrShortWrite) {
t.Fatalf("zero-progress error = %v", err)
}
}
func testResolvedPipeline(outputModule string) pipeline.ResolvedPipeline {
return pipeline.ResolvedPipeline{ID: "sample", Output: pipeline.ModuleBinding{Module: outputModule}}
}
func testRunOutput() pipeline.RunOutput {
return pipeline.RunOutput{
Manifest: artifacts.RunManifest{RunID: "run-123", PipelineID: "sample", ValidationStatus: "rejected"},
NormalizeOutputs: []contracts.SerializedOutput{{}, {}},
Rejected: []contracts.RejectedOutput{{}},
Diagnostics: contracts.DiagnosticCollection{Groups: []contracts.DiagnosticGroup{{
Disposition: contracts.DiagnosticDispositionWarning,
Category: contracts.DiagnosticCategoryFallback,
ReasonCode: "fallback",
Origin: contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageNormalize, StepID: "step", LaneID: "lane", ModuleKey: "module"},
OccurrenceCount: 1,
Samples: []contracts.DiagnosticSample{{Scope: "scope", Message: "message"}},
}}},
OutputFiles: []contracts.OutputFile{{Name: "index.json"}},
}
}
func testRunOutputWithout(change func(*pipeline.RunOutput)) pipeline.RunOutput {
output := testRunOutput()
change(&output)
return output
}
func mustWorkingDirectory(t *testing.T) string {
t.Helper()
workingDirectory, err := filepath.Abs(".")
if err != nil {
t.Fatal(err)
}
return workingDirectory
}
func (writer failingResultWriter) Write([]byte) (int, error) { return 0, writer.cause }
type partialResultWriter struct {
writer io.Writer
limit int
content bytes.Buffer
cause error
}
func (w partialResultWriter) Write(content []byte) (int, error) {
if len(content) > w.limit {
content = content[:w.limit]
func (writer *partialResultWriter) Write(content []byte) (int, error) {
if len(content) > 8 {
content = content[:8]
}
return w.writer.Write(content)
n, _ := writer.content.Write(content)
return n, writer.cause
}
type failingResultWriter struct{ err error }
var _ io.Writer = (*partialResultWriter)(nil)
func (w failingResultWriter) Write([]byte) (int, error) { return 0, w.err }
type zeroResultWriter struct{}
func (zeroResultWriter) Write([]byte) (int, error) { return 0, nil }
func TestPipelinePreflightClassifiesOnlyIdentifiableInvalidSelections(t *testing.T) {
// Real native missing-resource errors retain their category separately from
// private lookup causes. A joined operational branch must still win.
probe := &runProbe{}
app, file, input, _ := runFixture(t, probe)
var diagnostics bytes.Buffer
missing := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--llm-profile", "absent"}, commandkit.Streams{Out: io.Discard, Err: &diagnostics}).Cause
for _, test := range []struct {
cause error
status int
}{
{missing, 2},
{errors.Join(missing, context.Canceled), 1},
{errors.Join(missing, errors.New("unavailable inspection")), 1},
{errors.New("unavailable inspection"), 1},
} {
outcome := pipelineResourceFailure(test.cause)
if outcome.Status != test.status || !errors.Is(outcome.Cause, test.cause) {
t.Fatalf("preflight classification: %#v", outcome)
}
}
}

View File

@@ -12,6 +12,21 @@ import (
"go.maximumdirect.net/promptkit"
)
// SelectionError identifies an absent configured native resource. Callers can
// distinguish this invalid selection from unavailable inspection or cancellation
// without inspecting diagnostic text or the kit's private lookup causes.
type SelectionError struct{ Cause error }
func (err *SelectionError) Error() string { return err.Cause.Error() }
func (err *SelectionError) Unwrap() error { return err.Cause }
func resourceFinding(err error) error {
if !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) && (errors.Is(err, promptkit.ErrPromptNotFound) || errors.Is(err, promptkit.ErrProfileNotFound)) {
return &SelectionError{Cause: err}
}
return err
}
// RequiresEngine reports selected model work, including nested validators.
// Deterministic pipelines need no engine merely to validate their composition.
func RequiresEngine(resolved pipeline.ResolvedPipeline) bool {
@@ -69,7 +84,7 @@ func Validate(ctx context.Context, resolved pipeline.ResolvedPipeline, catalog p
}
if profile != "" {
if _, err := engine.InspectProfile(ctx, profile); err != nil {
findings = append(findings, fmt.Errorf("%s profile %q: %w", scope, profile, err))
findings = append(findings, resourceFinding(fmt.Errorf("%s profile %q: %w", scope, profile, err)))
}
}
for _, id := range prompts {
@@ -78,7 +93,7 @@ func Validate(ctx context.Context, resolved pipeline.ResolvedPipeline, catalog p
}
prompt, err := engine.InspectPrompt(ctx, id)
if err != nil {
findings = append(findings, fmt.Errorf("%s prompt %q: %w", scope, id, err))
findings = append(findings, resourceFinding(fmt.Errorf("%s prompt %q: %w", scope, id, err)))
continue
}
if profile != "" {
@@ -90,7 +105,7 @@ func Validate(ctx context.Context, resolved pipeline.ResolvedPipeline, catalog p
continue
}
if _, err := engine.InspectProfile(ctx, *prompt.DefaultProfileID); err != nil {
findings = append(findings, fmt.Errorf("%s prompt %q default profile %q: %w", scope, id, *prompt.DefaultProfileID, err))
findings = append(findings, resourceFinding(fmt.Errorf("%s prompt %q default profile %q: %w", scope, id, *prompt.DefaultProfileID, err)))
}
}
}