Report completed runs through native presentation and preserve failure outcomes

This commit is contained in:
2026-11-22 04:05:44 +00:00
parent 0cbc2b39cb
commit a8d256f676
10 changed files with 656 additions and 511 deletions

View File

@@ -1606,6 +1606,30 @@ input-hash session stability across changed reference bundles.
## Stage 34: Finish Publication, Native Completion Reports, And Failure Outcomes ## Stage 34: Finish Publication, Native Completion Reports, And Failure Outcomes
**Status:** Complete. Replaced the receipt/encoder/index and validation-ledger
reporting model with one native completion value and text template. Publication
and requested capture finish before reporting; JSON/YAML share the same value,
omit zero fallback counts and disabled debug paths, and never create reporting
sidecars. Qualified fallback summaries remain visible independently of warning
verbosity. Native progress, configuration diagnostics, and process findings use
stderr without promoting domain advisories to warnings. Operation causes remain
separate from native rendering/output/diagnostic delivery evidence. Failed capture
or presentation retains published files and available paths without model retries
or repeat terminalization. Identifiable preflight selections now retain a typed
classification alongside native causes; cancellation and unknown/mixed failures
remain operational. Replaced obsolete receipt tests with real-kit/fake-provider
coverage for formats, counts, native modes/progress, quiet rendering failure,
partial/failed stdout, failed diagnostics, cancellation, publication/capture
ordering, and primary/capture/delivery cause preservation.
Focused CLI/pipeline tests, supporting preflight/debug/cache/contracts/assets/
generic/Seriatim tests, focused vet/race checks, and diff checks pass. Added real
executable format/fallback tests using a local fake HTTP provider; their setup and
the full CLI/executable checks remain blocked by legacy `framework/llm` importing
the removed reporting package and old ConfigKit's removed YAML `plugin/limit`
dependency. Those subprocess results are not claimed passing. Stage 35 owns
remaining legacy code/dependency/fixture removal and whole-tree compilation;
assembled and final verification must execute the pending checks.
**Goal:** subprocesses receive artifacts plus the selected small native report. **Goal:** subprocesses receive artifacts plus the selected small native report.
**Inspect/change:** `run_result.go`, `run_terminal.go`, physical output placement, **Inspect/change:** `run_result.go`, `run_terminal.go`, physical output placement,

View File

@@ -2,21 +2,34 @@ package cli
import ( import (
"bytes" "bytes"
"encoding/json"
"errors" "errors"
"fmt"
"net/http"
"net/http/httptest"
"os" "os"
"os/exec" "os/exec"
"path/filepath" "path/filepath"
"runtime" "runtime"
"strings"
"testing" "testing"
"go.maximumdirect.net/commandkit"
) )
func TestExecutableRunRoutesDeclareTheSameRequiredInputs(t *testing.T) { func buildNativeExecutable(t *testing.T) string {
t.Helper()
binary := filepath.Join(t.TempDir(), "notarius") binary := filepath.Join(t.TempDir(), "notarius")
build := exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"), "build", "-o", binary, "../../cmd/notarius") build := exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"), "build", "-o", binary, "../../cmd/notarius")
build.Env = append(os.Environ(), "GOWORK=off") build.Env = append(os.Environ(), "GOWORK=off")
if output, err := build.CombinedOutput(); err != nil { if output, err := build.CombinedOutput(); err != nil {
t.Fatalf("build executable: %v\n%s", err, output) t.Fatalf("build executable: %v\n%s", err, output)
} }
return binary
}
func TestExecutableRunRoutesDeclareTheSameRequiredInputs(t *testing.T) {
binary := buildNativeExecutable(t)
for _, route := range [][]string{{"run"}, {"pipeline", "run"}} { for _, route := range [][]string{{"run"}, {"pipeline", "run"}} {
help := exec.Command(binary, append(append([]string(nil), route...), "--help")...) help := exec.Command(binary, append(append([]string(nil), route...), "--help")...)
output, err := help.CombinedOutput() output, err := help.CombinedOutput()
@@ -31,3 +44,68 @@ func TestExecutableRunRoutesDeclareTheSameRequiredInputs(t *testing.T) {
} }
} }
} }
func TestExecutableReportsDomainArtifactsAndQualifiedFallbacks(t *testing.T) {
binary := buildNativeExecutable(t)
for _, fallback := range []bool{false, true} {
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
if fallback {
writer.WriteHeader(http.StatusServiceUnavailable)
return
}
writer.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(writer).Encode(map[string]any{"id": "local-fake", "object": "chat.completion", "choices": []any{map[string]any{"index": 0, "message": map[string]any{"role": "assistant", "content": `{"spell_casts":[]}`}, "finish_reason": "stop"}}})
}))
t.Cleanup(server.Close)
for _, format := range []commandkit.Format{commandkit.FormatText, commandkit.FormatJSON, commandkit.FormatYAML} {
t.Run(fmt.Sprintf("%s/fallback=%t", format, fallback), func(t *testing.T) {
root := t.TempDir()
configuration := fmt.Sprintf(`version: 6
output: {directory: output}
pipelines:
demo:
input: {module: seriatim}
chunk: {module: generic}
llm_profile: local-fake
mode: permissive
max_additional_turns: 0
artifacts:
spells:
extract: {module: dnd/spells}
promptkit:
profiles:
- id: local-fake
endpoint: %q
model: fake-model
api_key_required: false
`, server.URL+"/v1")
file := filepath.Join(root, "config.yml")
input := filepath.Join(root, "input.json")
for path, content := range map[string]string{file: configuration, input: `{"metadata":{"id":"report-source"},"segments":[{"id":1,"start":0,"end":1,"speaker":"GM","text":"The party rests."}]}`} {
if err := os.WriteFile(path, []byte(content), 0600); err != nil {
t.Fatal(err)
}
}
command := exec.Command(binary, "run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--llm-transient-retries=0", "--format", string(format), "--verbose")
var output, diagnostics bytes.Buffer
command.Stdout, command.Stderr = &output, &diagnostics
if err := command.Run(); err != nil {
t.Fatalf("subprocess: %v stdout=%s stderr=%s", err, output.String(), diagnostics.String())
}
if format == commandkit.FormatText {
if !strings.Contains(output.String(), "produced 1 artifacts") || fallback != strings.Contains(output.String(), "Success with fallback:") {
t.Fatalf("completion text: %s", output.String())
}
} else {
report := decodeCompletionReport(t, format, output.Bytes())
if report["artifact_count"] != float64(1) || report["pipeline"] != "demo" || fallback && report["fallback_count"] != float64(1) || !fallback && report["fallback_count"] != nil {
t.Fatalf("completion report: %#v", report)
}
if _, err := os.Stat(filepath.Join(report["output_directory"].(string), "lanes", "spells.json")); err != nil {
t.Fatal(err)
}
}
})
}
}
}

View File

@@ -107,7 +107,7 @@ func pipelineCommands(newState func() *nativeLoaders, catalog func() (pipeline.M
} }
} }
if err := preflight.Validate(invocation.Context, effective.ResolvedPipeline, modules, engine); err != nil { if err := preflight.Validate(invocation.Context, effective.ResolvedPipeline, modules, engine); err != nil {
return pipelineInvalidConfiguration(err) return pipelineResourceFailure(err)
} }
return reportPipelineInspection(invocation, commandkit.Report{Value: struct { return reportPipelineInspection(invocation, commandkit.Report{Value: struct {
Pipeline string `json:"pipeline"` Pipeline string `json:"pipeline"`
@@ -124,16 +124,56 @@ var pipelineValidationTemplate = template.Must(commandkit.ParseTextTemplate("pip
func reportPipelineInspection(invocation commandkit.Invocation, report commandkit.Report) commandkit.Outcome { func reportPipelineInspection(invocation commandkit.Invocation, report commandkit.Report) commandkit.Outcome {
if err := invocation.Reporter().Report(report); err != nil { if err := invocation.Reporter().Report(report); err != nil {
return commandkit.Failure("Could not report pipeline inspection.").WithCause(err) return commandkit.Failure("Could not report pipeline inspection.")
} }
return commandkit.Success() return commandkit.Success()
} }
func pipelineConfigurationFailure(err error) commandkit.Outcome { func pipelineConfigurationFailure(err error) commandkit.Outcome {
outcome := commandkit.Failure(err.Error()).WithCause(err)
if ckconfig.ClassifyConfigurationError(err) == commandkit.OutcomeUsage { if ckconfig.ClassifyConfigurationError(err) == commandkit.OutcomeUsage {
outcome = commandkit.Usage(err.Error()).WithCause(err)
}
var load *configkit.LoadError
if errors.As(err, &load) {
_, explanation := ckconfig.AdaptDiagnostics(err.Error()+"\n", load.Sources(), load.Diagnostics())
outcome = outcome.WithExplanation(explanation.Message, explanation.Details...)
}
return outcome
}
func pipelineResourceFailure(err error) commandkit.Outcome {
if preflightInputError(err) {
return commandkit.Usage(err.Error()).WithCause(err) return commandkit.Usage(err.Error()).WithCause(err)
} }
return commandkit.Failure(err.Error()).WithCause(err) return pipelineConfigurationFailure(err)
}
// Only identifiable missing native selections are invalid preflight input.
// Unknown failures and any operational branch of a joined error remain failures.
func preflightInputError(err error) bool {
if err == nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return false
}
if _, ok := err.(*preflight.SelectionError); ok {
return true
}
if joined, ok := err.(interface{ Unwrap() []error }); ok {
found := false
for _, child := range joined.Unwrap() {
if child != nil {
found = true
if !preflightInputError(child) {
return false
}
}
}
return found
}
if wrapped, ok := err.(interface{ Unwrap() error }); ok {
return preflightInputError(wrapped.Unwrap())
}
return false
} }
func pipelineInvalidConfiguration(err error) commandkit.Outcome { func pipelineInvalidConfiguration(err error) commandkit.Outcome {

View File

@@ -262,7 +262,7 @@ func TestPipelineInspectionReportFailureRemainsOperational(t *testing.T) {
primary := errors.New("inspection stdout delivery failed") primary := errors.New("inspection stdout delivery failed")
var diagnostics bytes.Buffer var diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), []string{"pipeline", "list", "--config", file}, commandkit.Streams{Out: inspectionFailWriter{primary}, Err: &diagnostics}) outcome := app.Execute(context.Background(), []string{"pipeline", "list", "--config", file}, commandkit.Streams{Out: inspectionFailWriter{primary}, Err: &diagnostics})
if outcome.Status != 1 || !errors.Is(outcome.Cause, primary) { if outcome.Status != 1 || outcome.Cause != nil || !errors.Is(outcome.Presentation.Output.WriteError, primary) {
t.Fatalf("delivery failure classified as invalid configuration: %#v %s", outcome, diagnostics.String()) t.Fatalf("delivery failure classified as invalid configuration: %#v %s", outcome, diagnostics.String())
} }
} }

View File

@@ -11,6 +11,7 @@ import (
frameworkdebug "gitea.maximumdirect.net/eric/notarius/internal/framework/debug" frameworkdebug "gitea.maximumdirect.net/eric/notarius/internal/framework/debug"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/preflight" "gitea.maximumdirect.net/eric/notarius/internal/preflight"
"go.maximumdirect.net/ckconfig"
"go.maximumdirect.net/commandkit" "go.maximumdirect.net/commandkit"
"go.maximumdirect.net/configkit" "go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit" "go.maximumdirect.net/promptkit"
@@ -31,6 +32,14 @@ func pipelineRunCommand(version string, newState func() *nativeLoaders, modules
if err != nil { if err != nil {
return pipelineConfigurationFailure(err) return pipelineConfigurationFailure(err)
} }
diagnostics, _ := ckconfig.AdaptDiagnostics("Configuration\n", result.Sources(), result.Diagnostics())
for _, diagnostic := range diagnostics {
if diagnostic.Severity == commandkit.SeverityWarning {
_ = invocation.Reporter().AtEnd().Warn("", diagnostic.Write)
} else {
_ = invocation.Reporter().AtEnd().Info("", diagnostic.Write)
}
}
loaded, err := config.FromResult(result) loaded, err := config.FromResult(result)
if err != nil { if err != nil {
return pipelineInvalidConfiguration(err) return pipelineInvalidConfiguration(err)
@@ -53,20 +62,20 @@ func pipelineRunCommand(version string, newState func() *nativeLoaders, modules
} }
finish := func(outcome commandkit.Outcome) commandkit.Outcome { finish := func(outcome commandkit.Outcome) commandkit.Outcome {
if err := capture.Finalize(captureOutcome(), outcome.Cause); err != nil { if err := capture.Finalize(captureOutcome(), outcome.Cause); err != nil {
message := err.Error()
if published {
message += fmt.Sprintf("\nArtifacts: %s", outputDirectory)
}
if capture.Path() != "" {
message += fmt.Sprintf("\nDebug capture: %s", capture.Path())
}
if outcome.Status == 0 { if outcome.Status == 0 {
outcome = commandkit.Failure(message) outcome = commandkit.Failure(err.Error())
} else { } else {
outcome = outcome.WithExplanation(message) outcome = outcome.WithExplanation(err.Error(), outcome.Explanation.Details...)
} }
outcome = outcome.WithCause(err) outcome = outcome.WithCause(err)
} }
if outcome.Status != 0 {
availableOutput := ""
if published {
availableOutput = outputDirectory
}
outcome = outcome.WithExplanation(outcome.Explanation.Message+runLocations(availableOutput, capture.Path()), outcome.Explanation.Details...)
}
return outcome return outcome
} }
capture, err = startRunDebug(input.Debug, cfg.Debug.Directory, runID, startedAt) capture, err = startRunDebug(input.Debug, cfg.Debug.Directory, runID, startedAt)
@@ -107,13 +116,13 @@ func pipelineRunCommand(version string, newState func() *nativeLoaders, modules
} }
} }
if err := preflight.Validate(invocation.Context, effective.ResolvedPipeline, catalog, engine); err != nil { if err := preflight.Validate(invocation.Context, effective.ResolvedPipeline, catalog, engine); err != nil {
return finish(pipelineInvalidConfiguration(err)) return finish(pipelineResourceFailure(err))
} }
workingDirectory, err := os.Getwd() workingDirectory, err := os.Getwd()
if err != nil { if err != nil {
return finish(commandkit.Failure(err.Error()).WithCause(err)) return finish(commandkit.Failure(err.Error()).WithCause(err))
} }
materialized, diagnostics, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{ConfigPath: loaded.MainSourcePath(), WorkingDir: workingDirectory}) materialized, referenceDiagnostics, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{ConfigPath: loaded.MainSourcePath(), WorkingDir: workingDirectory})
if err != nil { if err != nil {
return finish(commandkit.Failure(err.Error()).WithCause(err)) return finish(commandkit.Failure(err.Error()).WithCause(err))
} }
@@ -138,7 +147,9 @@ func pipelineRunCommand(version string, newState func() *nativeLoaders, modules
recorder = newRunEvidenceRecorder(capture, result) recorder = newRunEvidenceRecorder(capture, result)
} }
group := &promptkit.ExecutionGroup{} group := &promptkit.ExecutionGroup{}
output, err = pipeline.New().Run(invocation.Context, pipeline.RunInput{Prepared: prepared, Path: input.InputFile.Value, RawInput: raw, SessionID: sessionID, RunID: runID, StartedAt: startedAt, Diagnostics: diagnostics, Cache: cache, Debug: recorder, Workers: cfg.Concurrency.Workers, Executions: group, Execution: contracts.ExecutionOptions{Execution: input.Overrides.Execution.Execution, TransientRetries: input.Overrides.Execution.TransientRetries}}) _ = invocation.Reporter().Progress(fmt.Sprintf("Running pipeline %q.", effective.ResolvedPipeline.ID))
output, err = pipeline.New().Run(invocation.Context, pipeline.RunInput{Prepared: prepared, Path: input.InputFile.Value, RawInput: raw, SessionID: sessionID, RunID: runID, StartedAt: startedAt, Diagnostics: referenceDiagnostics, Cache: cache, Debug: recorder, Workers: cfg.Concurrency.Workers, Executions: group, Execution: contracts.ExecutionOptions{Execution: input.Overrides.Execution.Execution, TransientRetries: input.Overrides.Execution.TransientRetries}})
reportRunDiagnostics(invocation.Reporter(), output.Diagnostics)
if err != nil { if err != nil {
return finish(commandkit.Failure(err.Error()).WithCause(err)) return finish(commandkit.Failure(err.Error()).WithCause(err))
} }
@@ -146,7 +157,11 @@ func pipelineRunCommand(version string, newState func() *nativeLoaders, modules
return finish(commandkit.Failure(err.Error()).WithCause(err)) return finish(commandkit.Failure(err.Error()).WithCause(err))
} }
published = true published = true
return finish(commandkit.Success()) outcome := finish(commandkit.Success())
if outcome.Status != 0 {
return outcome
}
return reportRunResult(invocation, runResult{Pipeline: effective.ResolvedPipeline.ID, ArtifactCount: output.ArtifactCount(), OutputDirectory: outputDirectory, DebugDirectory: capture.Path(), FallbackCount: output.FallbackCount})
}, },
}) })
} }

View File

@@ -31,12 +31,20 @@ type runProbe struct {
parsed int parsed int
produced int produced int
correctFirst bool correctFirst bool
providerErr error
onGenerate func()
} }
func (probe *runProbe) Generate(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { func (probe *runProbe) Generate(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
probe.mu.Lock() probe.mu.Lock()
defer probe.mu.Unlock() defer probe.mu.Unlock()
probe.requests = append(probe.requests, request) probe.requests = append(probe.requests, request)
if probe.onGenerate != nil {
probe.onGenerate()
}
if probe.providerErr != nil {
return nil, probe.providerErr
}
value := "accepted" value := "accepted"
if request.Prompt.Messages[len(request.Prompt.Messages)-1].Content != "Assess." { if request.Prompt.Messages[len(request.Prompt.Messages)-1].Content != "Assess." {
probe.produced++ probe.produced++
@@ -98,7 +106,9 @@ func (producer *runProducer) Extract(ctx context.Context, request contracts.Type
return &value, nil, err return &value, nil, err
}, func(context.Context, []string) (promptkit.RunRequest, error) { }, func(context.Context, []string) (promptkit.RunRequest, error) {
return promptkit.RunRequest{PromptID: "inspect-correction", ProfileID: request.LLMProfile, SessionID: request.SessionID, OnPrepared: producer.probe.onPrepared}, nil return promptkit.RunRequest{PromptID: "inspect-correction", ProfileID: request.LLMProfile, SessionID: request.SessionID, OnPrepared: producer.probe.onPrepared}, nil
}, nil) }, func(context.Context, *inspectionArtifact) (inspectionArtifact, error) {
return inspectionArtifact{Value: "fallback"}, nil
})
return contracts.TypedExtractionResult[inspectionArtifact]{Value: value, Outcome: outcome}, err return contracts.TypedExtractionResult[inspectionArtifact]{Value: value, Outcome: outcome}, err
} }
func (producer *runProducer) Merge(_ context.Context, request contracts.TypedMergeRequest[inspectionArtifact]) (contracts.TypedMergeResult[inspectionArtifact], error) { func (producer *runProducer) Merge(_ context.Context, request contracts.TypedMergeRequest[inspectionArtifact]) (contracts.TypedMergeResult[inspectionArtifact], error) {

View File

@@ -1,142 +1,78 @@
package cli package cli
import ( import (
"encoding/json"
"fmt" "fmt"
"io" "io"
"path/filepath"
"strings" "strings"
"text/template"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "go.maximumdirect.net/commandkit"
) )
const runResultSchemaVersion = "notarius.run-result.v2" // runResult is the small completion value. Domain files, not a saved report or
// execution ledger, are the durable deliverables.
type runResult struct { type runResult struct {
SchemaVersion string `json:"schema_version"` Pipeline string `json:"pipeline"`
RunID string `json:"run_id"` ArtifactCount int `json:"artifact_count"`
PipelineID string `json:"pipeline_id"` OutputDirectory string `json:"output_directory"`
OutputDirectory string `json:"output_directory"` DebugDirectory string `json:"debug_directory,omitempty"`
IndexFile string `json:"index_file,omitempty"` FallbackCount int `json:"fallback_count,omitempty"`
NormalizedOutputCount int `json:"normalized_output_count"`
RejectedOutputCount int `json:"rejected_output_count"`
WarningGroupCount int `json:"warning_group_count"`
WarningOccurrenceCount int `json:"warning_occurrence_count"`
DiagnosticGroupCount int `json:"diagnostic_group_count"`
DiagnosticOccurrenceCount int `json:"diagnostic_occurrence_count"`
DiagnosticsTruncated bool `json:"diagnostics_truncated"`
ValidationStatus string `json:"validation_status"`
ValidationSummaries []artifacts.ValidationSummary `json:"validation_summaries,omitempty"`
DebugDirectory string `json:"debug_directory,omitempty"`
} }
func newRunResult(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput, outputDirectory, debugDirectory string) (runResult, error) { var runResultTemplate = template.Must(commandkit.ParseTextTemplate("run-completion", `{{if .FallbackCount}}Success with fallback: {{else}}Success: {{end}}{{.Pipeline}} produced {{.ArtifactCount}} artifacts in {{.OutputDirectory}}
diagnosticProjection, err := contracts.ProjectDiagnosticCollection(output.Diagnostics) {{if .FallbackCount}}{{.FallbackCount}} operations used fallback.
if err != nil { {{end}}{{if .DebugDirectory}}Debug capture: {{.DebugDirectory}}
return runResult{}, fmt.Errorf("summarize run diagnostics: %w", err) {{end}}`, nil))
func reportRunResult(invocation commandkit.Invocation, result runResult) commandkit.Outcome {
message := "Could not present the completed run." + runLocations(result.OutputDirectory, result.DebugDirectory)
if err := invocation.Reporter().Report(commandkit.Report{Value: result, TextTemplate: runResultTemplate}); err != nil {
// The reporter retains render/write failures in Presentation. They are
// not a replacement operation Cause, and capture is already finalized.
return commandkit.Failure(message)
} }
return newRunResultWithDiagnostics(resolved, output, outputDirectory, debugDirectory, diagnosticProjection) // Deferred diagnostic delivery can still fail during native finalization.
// Supply useful locations without presenting a failure on successful runs.
return commandkit.Success().WithExplanation(message)
} }
func newRunResultWithDiagnostics(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput, outputDirectory, debugDirectory string, diagnosticProjection contracts.DiagnosticProjection) (runResult, error) { func runLocations(outputDirectory, debugDirectory string) string {
if strings.TrimSpace(output.Manifest.RunID) == "" { var locations string
return runResult{}, fmt.Errorf("run result requires a run ID") if outputDirectory != "" {
locations += "\nArtifacts: " + outputDirectory
} }
if strings.TrimSpace(resolved.ID) == "" { if debugDirectory != "" {
return runResult{}, fmt.Errorf("run result requires a resolved pipeline ID") locations += "\nDebug capture: " + debugDirectory
}
if strings.TrimSpace(output.Manifest.PipelineID) == "" {
return runResult{}, fmt.Errorf("run result requires a manifest pipeline ID")
}
if output.Manifest.PipelineID != resolved.ID {
return runResult{}, fmt.Errorf("run result pipeline ID does not match resolved pipeline")
}
if strings.TrimSpace(output.Manifest.ValidationStatus) == "" {
return runResult{}, fmt.Errorf("run result requires a validation status")
}
if strings.TrimSpace(outputDirectory) == "" {
return runResult{}, fmt.Errorf("run result requires an output directory")
} }
return locations
}
absOutputDirectory, err := filepath.Abs(outputDirectory) func reportRunDiagnostics(reporter *commandkit.Reporter, collection contracts.DiagnosticCollection) {
if err != nil { for _, group := range collection.Groups {
return runResult{}, fmt.Errorf("make output directory absolute: %w", err) origin := group.Origin
} scope := []string{string(origin.Stage)}
for _, value := range []string{origin.StepID, origin.LaneID, origin.ModuleKey, origin.ValidatorKey} {
result := runResult{ if value != "" {
SchemaVersion: runResultSchemaVersion, scope = append(scope, value)
RunID: output.Manifest.RunID,
PipelineID: output.Manifest.PipelineID,
OutputDirectory: absOutputDirectory,
NormalizedOutputCount: len(output.NormalizeOutputs),
RejectedOutputCount: len(output.Rejected),
WarningGroupCount: len(diagnosticProjection.Warnings),
WarningOccurrenceCount: diagnosticProjection.WarningOccurrenceCount,
DiagnosticGroupCount: len(diagnosticProjection.Diagnostics),
DiagnosticOccurrenceCount: diagnosticProjection.DiagnosticOccurrenceCount,
DiagnosticsTruncated: output.Diagnostics.Truncated,
ValidationStatus: output.Manifest.ValidationStatus,
ValidationSummaries: cloneValidationSummaries(output.Manifest.ValidationSummaries),
}
if strings.TrimSpace(debugDirectory) != "" {
absDebugDirectory, err := filepath.Abs(debugDirectory)
if err != nil {
return runResult{}, fmt.Errorf("make debug directory absolute: %w", err)
}
result.DebugDirectory = absDebugDirectory
}
if resolved.Output.Module == pipeline.DefaultOutputModule {
indexCount := 0
for _, file := range output.OutputFiles {
if file.Name == "index.json" {
indexCount++
} }
} }
if indexCount != 1 { heading := strings.Join(scope, " / ") + ": " + group.ReasonCode
return runResult{}, fmt.Errorf("production JSON output must contain exactly one index.json file") detail := func(writer io.Writer) error {
for _, sample := range group.Samples {
if _, err := fmt.Fprintf(writer, "%s: %s\n", sample.Scope, sample.Message); err != nil {
return err
}
}
return nil
} }
result.IndexFile = "index.json" if group.Disposition == contracts.DiagnosticDispositionWarning {
} _ = reporter.AtEnd().Warn(heading, detail)
} else {
return result, nil _ = reporter.AtEnd().Info(heading, detail)
}
func cloneValidationSummaries(summaries []artifacts.ValidationSummary) []artifacts.ValidationSummary {
if len(summaries) == 0 {
return nil
}
cloned := make([]artifacts.ValidationSummary, len(summaries))
for index, summary := range summaries {
cloned[index] = artifacts.CloneValidationSummary(summary)
}
return cloned
}
func encodeRunResult(result runResult) ([]byte, error) {
encoded, err := json.Marshal(result)
if err != nil {
return nil, fmt.Errorf("encode run result: %w", err)
}
return append(encoded, '\n'), nil
}
func writeRunResult(writer io.Writer, content []byte) error {
for len(content) > 0 {
written, err := writer.Write(content)
if written < 0 || written > len(content) {
return io.ErrShortWrite
}
content = content[written:]
if err != nil {
return err
}
if written == 0 {
return io.ErrShortWrite
} }
} }
return nil if collection.Truncated {
_ = reporter.AtEnd().Info("Additional pipeline diagnostics were omitted.")
}
} }

View File

@@ -2,198 +2,318 @@ package cli
import ( import (
"bytes" "bytes"
"context"
"encoding/json" "encoding/json"
"errors" "errors"
"io"
"os" "os"
"path/filepath" "path/filepath"
"strings" "strings"
"sync"
"testing" "testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "go.maximumdirect.net/commandkit"
alwaysreject "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/validate/always_reject" "go.maximumdirect.net/configkit"
) )
func TestMaintainedMinimalInvocationEmitsRunResult(t *testing.T) { func decodeCompletionReport(t *testing.T, format commandkit.Format, content []byte) map[string]any {
outputRoot := filepath.Join(t.TempDir(), "output")
var stdout, stderr strings.Builder
code := RunWithOptions([]string{
"run", "dnd-session",
"--config", repositoryPath("examples", "dnd-minimal.config.yml"),
"--input", repositoryPath("examples", "seriatim-minimal-transcript.json"),
"--only", "spells", "--chunk_cache", "bypass", "--output-dir", outputRoot, "--json",
}, &stdout, &stderr, productionRunOptions(t, &productionFakeLLMClient{}))
if code != 0 || stderr.Len() != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
receipt := decodeRunResultDocument(t, stdout.String())
if got := receipt["schema_version"]; got != "notarius.run-result.v2" {
t.Fatalf("schema_version = %q", got)
}
if got := receipt["run_id"]; got != productionRunID {
t.Fatalf("run_id = %q", got)
}
if got := receipt["pipeline_id"]; got != "dnd-session" {
t.Fatalf("pipeline_id = %q", got)
}
if got := receipt["index_file"]; got != "index.json" {
t.Fatalf("index_file = %q", got)
}
if got := receipt["normalized_output_count"]; got != float64(1) {
t.Fatalf("normalized_output_count = %v", got)
}
if got := receipt["rejected_output_count"]; got != float64(0) {
t.Fatalf("rejected_output_count = %v", got)
}
if got := receipt["warning_group_count"]; got != float64(0) || receipt["warning_occurrence_count"] != float64(0) || receipt["diagnostic_group_count"] != float64(0) || receipt["diagnostic_occurrence_count"] != float64(0) || receipt["diagnostics_truncated"] != false {
t.Fatalf("diagnostic counts = %#v", receipt)
}
if got := receipt["validation_status"]; got != "approved" {
t.Fatalf("validation_status = %q", got)
}
outputDirectory, ok := receipt["output_directory"].(string)
if !ok || !filepath.IsAbs(outputDirectory) || outputDirectory != filepath.Join(outputRoot, productionRunID) {
t.Fatalf("output_directory = %q", receipt["output_directory"])
}
indexFile := receipt["index_file"].(string)
assertFile(t, filepath.Join(outputDirectory, indexFile))
}
func TestRunResultReportsWarningsAndDebugBundle(t *testing.T) {
roots := newStateTestRoots(t)
harness := newStateTestHarness()
harness.chunkDiagnostics = []contracts.ProducerDiagnostic{stateTestDiagnostic("chunk", "contract-warning", "warning retained")}
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{
"run", "sample", "--config", roots.config, "--input", roots.input,
"--chunk_cache", "bypass", "--debug", "--json",
}, &stdout, &stderr, harness.options())
if code != 0 || !strings.Contains(stderr.String(), "1 warning group(s), 1 occurrence(s)") || strings.Contains(stderr.String(), "warnings.json") {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
receipt := decodeRunResultDocument(t, stdout.String())
if got := receipt["warning_group_count"]; got != float64(1) || receipt["warning_occurrence_count"] != float64(1) || receipt["diagnostic_group_count"] != float64(0) || receipt["diagnostic_occurrence_count"] != float64(0) || receipt["diagnostics_truncated"] != false {
t.Fatalf("diagnostic counts = %#v", receipt)
}
debugDirectory, ok := receipt["debug_directory"].(string)
if !ok || !filepath.IsAbs(debugDirectory) || debugDirectory != onlyChildDir(t, roots.debug) {
t.Fatalf("debug_directory = %q", receipt["debug_directory"])
}
if strings.Contains(stdout.String(), "complete:") || strings.Contains(stdout.String(), "debug=") {
t.Fatalf("machine stdout contains human reporting: %q", stdout.String())
}
}
func TestRunResultReportsSuccessfulRejection(t *testing.T) {
roots := newStateTestRoots(t)
configBytes, err := os.ReadFile(roots.config)
if err != nil {
t.Fatal(err)
}
configBytes = []byte(replaceRequiredOnce(t, string(configBytes), " normalize:\n module: test/normalize\n", " normalize:\n module: test/normalize\n validators:\n - module: generic/always_reject\n validation_policy:\n semantic_rejection: reject_output\n"))
if err := os.WriteFile(roots.config, configBytes, 0o600); err != nil {
t.Fatal(err)
}
harness := newStateTestHarness()
opts := harness.options()
if err := alwaysreject.RegisterTyped[stateTestArtifact](opts.Registries.Validators, stateTestArtifactKind); err != nil {
t.Fatal(err)
}
opts.Catalog = catalogFromRegistries(opts.Registries)
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{
"run", "sample", "--config", roots.config, "--input", roots.input,
"--chunk_cache", "bypass", "--json",
}, &stdout, &stderr, opts)
if code != 0 || stderr.Len() != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
receipt := decodeRunResultDocument(t, stdout.String())
if got := receipt["normalized_output_count"]; got != float64(0) {
t.Fatalf("normalized_output_count = %v", got)
}
if got := receipt["rejected_output_count"]; got != float64(1) {
t.Fatalf("rejected_output_count = %v", got)
}
if got := receipt["validation_status"]; got != "rejected" {
t.Fatalf("validation_status = %q", got)
}
}
func TestRunResultIsAbsentForSyntaxAndRuntimeFailures(t *testing.T) {
t.Run("syntax", func(t *testing.T) {
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"run", "sample", "--json"}, &stdout, &stderr, newStateTestHarness().options())
if code != 2 || stdout.Len() != 0 || stderr.Len() == 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
})
t.Run("runtime", func(t *testing.T) {
roots := newStateTestRoots(t)
harness := newStateTestHarness()
harness.extractErr = errors.New("injected extraction failure")
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{
"run", "sample", "--config", roots.config, "--input", roots.input,
"--chunk_cache", "bypass", "--json",
}, &stdout, &stderr, harness.options())
if code != 1 || stdout.Len() != 0 || stderr.Len() == 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
})
}
func TestRunResultDeliveryFailureRetainsPublishedBundles(t *testing.T) {
roots := newStateTestRoots(t)
writerErr := errors.New("result writer sentinel")
stdout := &resultDeliveryWriter{err: writerErr}
var stderr bytes.Buffer
code := RunWithOptions([]string{
"run", "sample", "--config", roots.config, "--input", roots.input,
"--chunk_cache", "bypass", "--debug", "--json",
}, stdout, &stderr, newStateTestHarness().options())
if code != 1 || !strings.Contains(stderr.String(), "write run result") || strings.Contains(stderr.String(), writerErr.Error()) {
t.Fatalf("code=%d stderr=%q", code, stderr.String())
}
if stdout.accepted.Len() != 0 {
t.Fatalf("accepted stdout = %q", stdout.accepted.String())
}
assertStateTestOutput(t, roots.output)
debugBundle := onlyChildDir(t, roots.debug)
report := readStateTestRunReport(t, debugBundle)
if !report.Succeeded {
t.Fatalf("debug report = %#v, want successful persisted run", report)
}
if strings.Contains(readAllFiles(t, debugBundle), writerErr.Error()) {
t.Fatalf("debug bundle contains result writer error")
}
}
func decodeRunResultDocument(t *testing.T, stdout string) map[string]any {
t.Helper() t.Helper()
if strings.Count(stdout, "\n") != 1 { if format == commandkit.FormatYAML {
t.Fatalf("stdout = %q, want one JSON document", stdout) value, err := commandkit.DecodeYAMLReport(bytes.NewReader(content))
if err != nil {
t.Fatalf("decode native YAML report: %v\n%s", err, content)
}
content, err = json.Marshal(value)
if err != nil {
t.Fatal(err)
}
} }
var receipt map[string]any var report map[string]any
if err := json.Unmarshal([]byte(stdout), &receipt); err != nil { if err := json.Unmarshal(content, &report); err != nil {
t.Fatalf("decode run result: %v; stdout=%q", err, stdout) t.Fatalf("decode native report: %v\n%s", err, content)
} }
return receipt return report
} }
type resultDeliveryWriter struct { func assertCompletedRunFiles(t *testing.T, file string, captured bool) {
err error t.Helper()
accepted bytes.Buffer run := filepath.Join(filepath.Dir(file), "output-must-not-exist", "run-1")
if content, err := os.ReadFile(filepath.Join(run, "lanes", "notes.json")); err != nil || len(content) == 0 {
t.Fatalf("published domain artifact: %s %v", content, err)
}
files, err := os.ReadDir(run)
if err != nil || len(files) != 1 || files[0].Name() != "lanes" {
t.Fatalf("unexpected reporting sidecars: %#v %v", files, err)
}
if captured {
debug := filepath.Join(filepath.Dir(file), "debug-must-not-exist", "run-1")
content, err := os.ReadFile(filepath.Join(debug, "outcome.json"))
if err != nil {
t.Fatal(err)
}
var terminal struct {
Outcome struct {
Published bool `json:"published"`
} `json:"outcome"`
Error string `json:"error"`
CaptureError string `json:"capture_error"`
}
if err := json.Unmarshal(content, &terminal); err != nil || !terminal.Outcome.Published || terminal.Error != "" || terminal.CaptureError != "" {
t.Fatalf("capture was not completed before reporting: %s %v", content, err)
}
}
} }
func (w *resultDeliveryWriter) Write(content []byte) (int, error) { type completedRunWriter struct {
if w.err != nil { t *testing.T
return 0, w.err file string
} captured bool
return w.accepted.Write(content) content bytes.Buffer
}
func (writer *completedRunWriter) Write(content []byte) (int, error) {
assertCompletedRunFiles(writer.t, writer.file, writer.captured)
return writer.content.Write(content)
}
func TestNativeRunCompletionFormatsFollowPublicationAndCapture(t *testing.T) {
for _, format := range []commandkit.Format{commandkit.FormatText, commandkit.FormatJSON, commandkit.FormatYAML} {
t.Run(string(format), func(t *testing.T) {
probe := &runProbe{}
app, file, input, _ := runFixture(t, probe)
output := &completedRunWriter{t: t, file: file, captured: true}
var diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--format", string(format)}, commandkit.Streams{Out: output, Err: &diagnostics})
if outcome.Status != 0 || !strings.Contains(diagnostics.String(), "Running pipeline") {
t.Fatalf("completion: %#v stderr=%s", outcome, diagnostics.String())
}
directory := filepath.Join(filepath.Dir(file), "output-must-not-exist", "run-1")
debug := filepath.Join(filepath.Dir(file), "debug-must-not-exist", "run-1")
if format == commandkit.FormatText {
if !strings.Contains(output.content.String(), "Success: demo produced 1 artifacts in "+directory) || !strings.Contains(output.content.String(), "Debug capture: "+debug) || strings.Contains(output.content.String(), "fallback") {
t.Fatalf("native completion text: %s", output.content.String())
}
} else {
report := decodeCompletionReport(t, format, output.content.Bytes())
if len(report) != 4 || report["pipeline"] != "demo" || report["artifact_count"] != float64(1) || report["output_directory"] != directory || report["debug_directory"] != debug {
t.Fatalf("native report: %#v", report)
}
}
if probe.produced != 1 {
t.Fatalf("reporting reran a producer: %d", probe.produced)
}
})
}
}
func TestNativeRunDisclosesFallbackIndependentlyOfDiagnosticVisibility(t *testing.T) {
for _, format := range []commandkit.Format{commandkit.FormatText, commandkit.FormatJSON, commandkit.FormatYAML} {
for _, verbose := range []bool{false, true} {
probe := &runProbe{correctFirst: true}
app, file, input, _ := runFixture(t, probe)
args := []string{"pipeline", "run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--set", `/pipelines/demo/mode="permissive"`, "--set", "/pipelines/demo/max_additional_turns=0", "--format", string(format)}
if verbose {
args = append(args, "--verbose")
}
var output, diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: &diagnostics})
if outcome.Status != 0 || probe.produced != 1 {
t.Fatalf("permissive completion: %#v %s", outcome, diagnostics.String())
}
if verbose && !strings.Contains(diagnostics.String(), "WARN:") || !verbose && diagnostics.Len() != 0 {
t.Fatalf("native warning visibility: %s", diagnostics.String())
}
if format == commandkit.FormatText {
if !strings.Contains(output.String(), "Success with fallback:") || !strings.Contains(output.String(), "1 operations used fallback.") {
t.Fatalf("fallback text: %s", output.String())
}
} else {
report := decodeCompletionReport(t, format, output.Bytes())
if report["fallback_count"] != float64(1) || report["artifact_count"] != float64(1) || len(report) != 4 {
t.Fatalf("fallback report: %#v", report)
}
}
assertCompletedRunFiles(t, file, false)
}
}
}
func TestNativeRunQuietAndDebugUseNativeModeOrdering(t *testing.T) {
for _, test := range []struct {
flags []string
quiet bool
progress bool
}{
{[]string{"--debug", "--quiet"}, true, false},
{[]string{"--quiet", "--debug"}, false, true},
{[]string{"--debug", "--progress=false"}, false, false},
} {
probe := &runProbe{}
app, file, input, _ := runFixture(t, probe)
var output, diagnostics bytes.Buffer
args := append([]string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--format", "json"}, test.flags...)
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: &diagnostics})
if outcome.Status != 0 || test.quiet != (output.Len() == 0) || test.progress != strings.Contains(diagnostics.String(), "Running pipeline") {
t.Fatalf("native mode order: %#v output=%s stderr=%s", outcome, output.String(), diagnostics.String())
}
assertCompletedRunFiles(t, file, true)
}
}
func TestNativeRunCancellationKeepsItsCauseAndPublishesNothing(t *testing.T) {
probe := &runProbe{}
app, file, input, _ := runFixture(t, probe)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
probe.onGenerate = cancel
var output, diagnostics bytes.Buffer
outcome := app.Execute(ctx, []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--quiet"}, commandkit.Streams{Out: &output, Err: &diagnostics})
if outcome.Status != 1 || !errors.Is(outcome.Cause, context.Canceled) || output.Len() != 0 || diagnostics.Len() == 0 {
t.Fatalf("cancellation: %#v stdout=%s stderr=%s", outcome, output.String(), diagnostics.String())
}
if _, err := os.Stat(filepath.Join(filepath.Dir(file), "output-must-not-exist")); !os.IsNotExist(err) {
t.Fatalf("cancellation published artifacts: %v", err)
}
}
func TestNativeRunPrimaryFailureSurvivesCaptureAndDiagnosticFailures(t *testing.T) {
primary := errors.New("provider failure sentinel")
delivery := errors.New("diagnostic failure sentinel")
probe := &runProbe{providerErr: primary}
app, file, input, _ := runFixture(t, probe)
debug := filepath.Join(filepath.Dir(file), "debug-must-not-exist", "run-1")
probe.onGenerate = func() {
if err := os.Mkdir(filepath.Join(debug, "outcome.json"), 0700); err != nil {
t.Error(err)
}
}
var output bytes.Buffer
outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--llm-transient-retries=0"}, commandkit.Streams{Out: &output, Err: failingResultWriter{cause: delivery}})
var captureFailure *os.LinkError
if outcome.Status != 1 || !errors.Is(outcome.Cause, primary) || !errors.As(outcome.Cause, &captureFailure) || errors.Is(outcome.Cause, delivery) || !errors.Is(outcome.Presentation.Diagnostics.WriteError, delivery) || output.Len() != 0 || len(probe.requests) != 1 {
t.Fatalf("primary/capture/delivery failure: %#v", outcome)
}
if !strings.Contains(outcome.Explanation.Message, debug) {
t.Fatalf("failed run capture location unavailable: %#v", outcome)
}
}
func TestNativeRunReportDeliveryFailuresRetainArtifactsAndNativePresentation(t *testing.T) {
sentinel := errors.New("result delivery sentinel")
for _, partial := range []bool{false, true} {
probe := &runProbe{}
app, file, input, _ := runFixture(t, probe)
var output io.Writer = failingResultWriter{cause: sentinel}
prefix := &partialResultWriter{cause: sentinel}
if partial {
output = prefix
}
var diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--format", "json"}, commandkit.Streams{Out: output, Err: &diagnostics})
if outcome.Status != 1 || outcome.Cause != nil || !errors.Is(outcome.Presentation.Output.WriteError, sentinel) || outcome.Presentation.Output.State != commandkit.DeliveryIncomplete || probe.produced != 1 || partial && prefix.content.Len() == 0 {
t.Fatalf("delivery failure: %#v %s", outcome, diagnostics.String())
}
assertCompletedRunFiles(t, file, true)
for _, root := range []string{"output-must-not-exist", "debug-must-not-exist"} {
if !strings.Contains(diagnostics.String(), filepath.Join(filepath.Dir(file), root, "run-1")) {
t.Fatalf("missing available path after delivery failure: %s", diagnostics.String())
}
}
}
}
func TestNativeRunDiagnosticDeliveryFailureIsNotAnOperationFailure(t *testing.T) {
sentinel := errors.New("diagnostic delivery sentinel")
probe := &runProbe{correctFirst: true}
app, file, input, _ := runFixture(t, probe)
var output bytes.Buffer
outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--set", `/pipelines/demo/mode="permissive"`, "--set", "/pipelines/demo/max_additional_turns=0", "--format", "json", "--verbose"}, commandkit.Streams{Out: &output, Err: failingResultWriter{cause: sentinel}})
if outcome.Status != 1 || outcome.Cause != nil || !errors.Is(outcome.Presentation.Diagnostics.WriteError, sentinel) || outcome.Presentation.Diagnostics.State != commandkit.DeliveryIncomplete {
t.Fatalf("diagnostic delivery: %#v", outcome)
}
if report := decodeCompletionReport(t, commandkit.FormatJSON, output.Bytes()); report["fallback_count"] != float64(1) {
t.Fatalf("completed report was replaced by a delivery envelope: %#v", report)
}
if !strings.Contains(outcome.Explanation.Message, filepath.Join(filepath.Dir(file), "output-must-not-exist", "run-1")) {
t.Fatalf("presentation failure lost available artifacts: %#v", outcome)
}
assertCompletedRunFiles(t, file, false)
}
func TestNativeRunCaptureFailureAfterPublicationDoesNotReportSuccess(t *testing.T) {
probe := &runProbe{}
app, file, input, _ := runFixture(t, probe)
var once sync.Once
debug := filepath.Join(filepath.Dir(file), "debug-must-not-exist", "run-1")
probe.onGenerate = func() {
once.Do(func() {
// The external provider boundary runs after capture allocation but
// before terminalization. Make only the terminal file unusable.
if err := os.Mkdir(filepath.Join(debug, "outcome.json"), 0700); err != nil {
t.Fatal(err)
}
})
}
var output, diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--format", "json"}, commandkit.Streams{Out: &output, Err: &diagnostics})
if outcome.Status != 1 || outcome.Cause == nil || output.Len() != 0 || probe.produced != 1 {
t.Fatalf("post-publication capture failure: %#v stdout=%s stderr=%s", outcome, output.String(), diagnostics.String())
}
assertCompletedRunFiles(t, file, false)
if !strings.Contains(diagnostics.String(), debug) || !strings.Contains(diagnostics.String(), filepath.Join(filepath.Dir(file), "output-must-not-exist", "run-1")) {
t.Fatalf("post-publication locations: %s", diagnostics.String())
}
if _, err := os.Stat(filepath.Join(debug, "executions.json")); err != nil {
t.Fatalf("existing capture was removed: %v", err)
}
}
func TestNativeRunFailureCategoriesPreserveCausesWithoutSuccessReports(t *testing.T) {
for _, test := range []struct {
name string
configure func(*testing.T, *runProbe, string, string) []string
status int
cause error
}{
{"missing configuration", func(t *testing.T, _ *runProbe, _, input string) []string {
return []string{"run", "demo", "--input-file", input, "--config", filepath.Join(t.TempDir(), "missing.yml")}
}, 1, nil},
{"invalid configuration", func(t *testing.T, _ *runProbe, file, input string) []string {
return []string{"run", "demo", "--config", file, "--input-file", input, "--set", "/version=5"}
}, 2, nil},
{"unknown pipeline", func(_ *testing.T, _ *runProbe, file, input string) []string {
return []string{"run", "missing", "--config", file, "--input-file", input}
}, 2, nil},
{"missing input", func(t *testing.T, _ *runProbe, file, _ string) []string {
return []string{"run", "demo", "--config", file, "--input-file", filepath.Join(t.TempDir(), "missing.txt")}
}, 1, os.ErrNotExist},
{"publication failure", func(t *testing.T, _ *runProbe, file, input string) []string {
path := filepath.Join(t.TempDir(), "occupied")
if err := os.WriteFile(path, []byte("keep"), 0600); err != nil {
t.Fatal(err)
}
return []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--output-dir", path}
}, 1, nil},
{"provider failure", func(_ *testing.T, probe *runProbe, file, input string) []string {
probe.providerErr = io.ErrUnexpectedEOF
return []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--llm-transient-retries=0"}
}, 1, io.ErrUnexpectedEOF},
} {
t.Run(test.name, func(t *testing.T) {
probe := &runProbe{}
app, file, input, _ := runFixture(t, probe)
var output, diagnostics bytes.Buffer
args := append(test.configure(t, probe, file, input), "--quiet", "--format", "json")
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: &diagnostics})
if outcome.Status != test.status || test.cause != nil && !errors.Is(outcome.Cause, test.cause) || output.Len() != 0 || diagnostics.Len() == 0 {
t.Fatalf("failure category: %#v stdout=%s stderr=%s", outcome, output.String(), diagnostics.String())
}
if test.name == "missing configuration" || test.name == "invalid configuration" {
var load *configkit.LoadError
if !errors.As(outcome.Cause, &load) {
t.Fatalf("lost native load cause: %#v", outcome)
}
}
})
}
} }

View File

@@ -2,229 +2,136 @@ package cli
import ( import (
"bytes" "bytes"
"encoding/json" "context"
"errors" "errors"
"io" "io"
"path/filepath" "path/filepath"
"strings"
"testing" "testing"
"text/template"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "go.maximumdirect.net/commandkit"
) )
func TestRunResultEncodesRequiredFieldsAndCounts(t *testing.T) { func TestRunResultUsesOneNativeValueAcrossFormats(t *testing.T) {
result, err := newRunResult(testResolvedPipeline(pipeline.DefaultOutputModule), testRunOutput(), "relative-output", "relative-debug") value := runResult{Pipeline: "demo", ArtifactCount: 3, OutputDirectory: "/artifacts/run-one", DebugDirectory: "/debug/run-one", FallbackCount: 2}
if err != nil { for _, format := range []commandkit.Format{commandkit.FormatText, commandkit.FormatJSON, commandkit.FormatYAML} {
t.Fatal(err) var output bytes.Buffer
} if err := commandkit.RenderReport(&output, commandkit.Report{Value: value, TextTemplate: runResultTemplate}, format); err != nil {
t.Fatal(err)
encoded, err := encodeRunResult(result) }
if err != nil { if format == commandkit.FormatText {
t.Fatal(err) if !strings.Contains(output.String(), "Success with fallback: demo produced 3 artifacts in /artifacts/run-one") || !strings.Contains(output.String(), "2 operations used fallback.") || !strings.Contains(output.String(), "Debug capture: /debug/run-one") {
} t.Fatalf("fallback text: %s", output.String())
if encoded[len(encoded)-1] != '\n' || bytes.Count(encoded, []byte{'\n'}) != 1 {
t.Fatalf("encoded result is not one newline-terminated object: %q", encoded)
}
var decoded map[string]any
if err := json.Unmarshal(encoded, &decoded); err != nil {
t.Fatal(err)
}
if got := decoded["schema_version"]; got != runResultSchemaVersion {
t.Fatalf("schema_version = %q", got)
}
if got := decoded["run_id"]; got != "run-123" {
t.Fatalf("run_id = %q", got)
}
if got := decoded["pipeline_id"]; got != "sample" {
t.Fatalf("pipeline_id = %q", got)
}
if got := decoded["validation_status"]; got != "rejected" {
t.Fatalf("validation_status = %q", got)
}
if got := decoded["index_file"]; got != "index.json" {
t.Fatalf("index_file = %q", got)
}
if got := decoded["normalized_output_count"]; got != float64(2) {
t.Fatalf("normalized_output_count = %v", got)
}
if got := decoded["rejected_output_count"]; got != float64(1) {
t.Fatalf("rejected_output_count = %v", got)
}
if got := decoded["warning_group_count"]; got != float64(1) || decoded["warning_occurrence_count"] != float64(1) || decoded["diagnostic_group_count"] != float64(0) || decoded["diagnostic_occurrence_count"] != float64(0) || decoded["diagnostics_truncated"] != false {
t.Fatalf("diagnostic counts = %#v", decoded)
}
if got := decoded["validation_summaries"]; got != nil {
t.Fatalf("validation_summaries = %#v, want omitted when empty", got)
}
if got := decoded["output_directory"]; got != filepath.Join(mustWorkingDirectory(t), "relative-output") {
t.Fatalf("output_directory = %q", got)
}
if got := decoded["debug_directory"]; got != filepath.Join(mustWorkingDirectory(t), "relative-debug") {
t.Fatalf("debug_directory = %q", got)
}
}
func TestRunResultRejectsInvalidRequiredValues(t *testing.T) {
tests := []struct {
name string
resolved pipeline.ResolvedPipeline
output pipeline.RunOutput
directory string
}{
{name: "blank run ID", resolved: testResolvedPipeline(pipeline.DefaultOutputModule), output: testRunOutputWithout(func(output *pipeline.RunOutput) { output.Manifest.RunID = " " }), directory: "output"},
{name: "blank resolved pipeline ID", resolved: pipeline.ResolvedPipeline{Output: pipeline.ModuleBinding{Module: pipeline.DefaultOutputModule}}, output: testRunOutput(), directory: "output"},
{name: "blank manifest pipeline ID", resolved: testResolvedPipeline(pipeline.DefaultOutputModule), output: testRunOutputWithout(func(output *pipeline.RunOutput) { output.Manifest.PipelineID = "" }), directory: "output"},
{name: "mismatched pipeline IDs", resolved: testResolvedPipeline(pipeline.DefaultOutputModule), output: testRunOutputWithout(func(output *pipeline.RunOutput) { output.Manifest.PipelineID = "other" }), directory: "output"},
{name: "blank validation status", resolved: testResolvedPipeline(pipeline.DefaultOutputModule), output: testRunOutputWithout(func(output *pipeline.RunOutput) { output.Manifest.ValidationStatus = " " }), directory: "output"},
{name: "blank output directory", resolved: testResolvedPipeline(pipeline.DefaultOutputModule), output: testRunOutput(), directory: " "},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if _, err := newRunResult(tt.resolved, tt.output, tt.directory, ""); err == nil {
t.Fatal("newRunResult() succeeded")
} }
}) continue
}
report := decodeCompletionReport(t, format, output.Bytes())
if report["pipeline"] != "demo" || report["output_directory"] != value.OutputDirectory || report["debug_directory"] != value.DebugDirectory || report["fallback_count"] != float64(2) || report["artifact_count"] != float64(3) || len(report) != 5 {
t.Fatalf("completion value: %#v", report)
}
} }
} }
func TestRunResultOmitsIndexFileForOtherOutputModules(t *testing.T) { func TestNativeCompletionDiagnosticsRetainDomainClassification(t *testing.T) {
result, err := newRunResult(testResolvedPipeline("test/output"), testRunOutputWithout(func(output *pipeline.RunOutput) { collection := contracts.DiagnosticCollection{}
output.OutputFiles = nil for _, disposition := range []contracts.DiagnosticDisposition{contracts.DiagnosticDispositionWarning, contracts.DiagnosticDispositionAdvisory, contracts.DiagnosticDispositionObservation} {
}), "output", "") category := contracts.DiagnosticCategoryDataQuality
if disposition == contracts.DiagnosticDispositionWarning {
category = contracts.DiagnosticCategoryFallback
} else if disposition == contracts.DiagnosticDispositionObservation {
category = contracts.DiagnosticCategoryNormalization
}
collection.Groups = append(collection.Groups, contracts.DiagnosticGroup{Disposition: disposition, Category: category, ReasonCode: string(disposition), Origin: contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract, StepID: "build", LaneID: "notes", ModuleKey: "producer"}, OccurrenceCount: 1, Samples: []contracts.DiagnosticSample{{Scope: "current source", Message: string(disposition) + " finding"}}})
}
if err := collection.Validate(); err != nil {
t.Fatal(err)
}
var output, diagnostics bytes.Buffer
app, err := commandkit.New(commandkit.Identity{Name: "fixture"}, commandkit.NewCommand(commandkit.CommandSpec[struct{}]{Name: "report", Run: func(invocation commandkit.Invocation, _ struct{}) commandkit.Outcome {
reportRunDiagnostics(invocation.Reporter(), collection)
return reportRunResult(invocation, runResult{Pipeline: "demo", ArtifactCount: 1, OutputDirectory: "/artifacts/run-one"})
}}))
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if result.IndexFile != "" { outcome := app.Execute(context.Background(), []string{"report", "--verbose"}, commandkit.Streams{Out: &output, Err: &diagnostics})
t.Fatalf("index_file = %q", result.IndexFile) if outcome.Status != 0 || strings.Contains(output.String(), "finding") || strings.Count(diagnostics.String(), "WARN:") != 1 {
t.Fatalf("diagnostic classification: %#v stdout=%s stderr=%s", outcome, output.String(), diagnostics.String())
} }
encoded, err := encodeRunResult(result) for _, group := range collection.Groups {
if err != nil { if !strings.Contains(diagnostics.String(), string(group.Disposition)+" finding") {
t.Fatal(err) t.Fatalf("missing diagnostic: %s", diagnostics.String())
} }
var decoded map[string]any
if err := json.Unmarshal(encoded, &decoded); err != nil {
t.Fatal(err)
}
if _, ok := decoded["index_file"]; ok {
t.Fatalf("encoded non-JSON result contains index_file: %s", encoded)
}
if _, ok := decoded["debug_directory"]; ok {
t.Fatalf("encoded result without debug capture contains debug_directory: %s", encoded)
} }
} }
func TestRunResultProjectsOwnedValidationSummaries(t *testing.T) { func TestCompletedRunRenderingFailureRetainsPublicationEvenWhenQuiet(t *testing.T) {
output := testRunOutput() previous := runResultTemplate
output.Manifest.ValidationSummaries = []artifacts.ValidationSummary{{Status: "incomplete", IncompleteValidators: []string{"validator"}, ProducerAttemptCount: 1, TerminalAction: "warn_continue"}} t.Cleanup(func() { runResultTemplate = previous })
result, err := newRunResult(testResolvedPipeline(pipeline.DefaultOutputModule), output, "output", "") // Simulate a broken application-owned text template, not a new renderer seam.
if err != nil { runResultTemplate = template.Must(commandkit.ParseTextTemplate("invalid", "{{.MissingField}}", nil))
t.Fatal(err) for _, quiet := range []bool{false, true} {
} probe := &runProbe{}
output.Manifest.ValidationSummaries[0].IncompleteValidators[0] = "caller mutation" app, file, input, _ := runFixture(t, probe)
if got := result.ValidationSummaries[0].IncompleteValidators; len(got) != 1 || got[0] != "validator" { args := []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug"}
t.Fatalf("result validation summaries = %#v", result.ValidationSummaries) if quiet {
} args = append(args, "--quiet")
encoded, err := encodeRunResult(result) }
if err != nil { var output, diagnostics bytes.Buffer
t.Fatal(err) outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: &diagnostics})
} if outcome.Status != 1 || outcome.Cause != nil || outcome.Presentation.Output.RenderError == nil || outcome.Presentation.Output.State != commandkit.DeliveryIncomplete || probe.produced != 1 {
if !bytes.Contains(encoded, []byte(`"validation_summaries":[{"status":"incomplete","incomplete_validators":["validator"],"producer_attempt_count":1,"terminal_action":"warn_continue"}]`)) { t.Fatalf("render failure: %#v %s", outcome, diagnostics.String())
t.Fatalf("encoded result = %s", encoded) }
assertCompletedRunFiles(t, file, true)
artifacts := filepath.Join(filepath.Dir(file), "output-must-not-exist", "run-1")
if !strings.Contains(diagnostics.String(), artifacts) || (quiet && output.Len() != 0) {
t.Fatalf("render failure locations/quiet: %s", diagnostics.String())
}
} }
} }
func TestRunResultRequiresOneProductionIndexFile(t *testing.T) { type failingResultWriter struct {
tests := []struct { cause error
name string
files []contracts.OutputFile
}{
{name: "missing", files: nil},
{name: "duplicate", files: []contracts.OutputFile{{Name: "index.json"}, {Name: "index.json"}}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
output := testRunOutput()
output.OutputFiles = tt.files
if _, err := newRunResult(testResolvedPipeline(pipeline.DefaultOutputModule), output, "output", ""); err == nil {
t.Fatal("newRunResult() succeeded")
}
})
}
} }
func TestWriteRunResultCompletesAndReportsWriterFailure(t *testing.T) { func (writer failingResultWriter) Write([]byte) (int, error) { return 0, writer.cause }
content := []byte("result\n")
var target bytes.Buffer
if err := writeRunResult(partialResultWriter{writer: &target, limit: 2}, content); err != nil {
t.Fatal(err)
}
if got := target.String(); got != string(content) {
t.Fatalf("written result = %q", got)
}
writerErr := errors.New("result writer failed")
if err := writeRunResult(failingResultWriter{err: writerErr}, content); !errors.Is(err, writerErr) {
t.Fatalf("writeRunResult() error = %v", err)
}
if err := writeRunResult(zeroResultWriter{}, content); !errors.Is(err, io.ErrShortWrite) {
t.Fatalf("zero-progress error = %v", err)
}
}
func testResolvedPipeline(outputModule string) pipeline.ResolvedPipeline {
return pipeline.ResolvedPipeline{ID: "sample", Output: pipeline.ModuleBinding{Module: outputModule}}
}
func testRunOutput() pipeline.RunOutput {
return pipeline.RunOutput{
Manifest: artifacts.RunManifest{RunID: "run-123", PipelineID: "sample", ValidationStatus: "rejected"},
NormalizeOutputs: []contracts.SerializedOutput{{}, {}},
Rejected: []contracts.RejectedOutput{{}},
Diagnostics: contracts.DiagnosticCollection{Groups: []contracts.DiagnosticGroup{{
Disposition: contracts.DiagnosticDispositionWarning,
Category: contracts.DiagnosticCategoryFallback,
ReasonCode: "fallback",
Origin: contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageNormalize, StepID: "step", LaneID: "lane", ModuleKey: "module"},
OccurrenceCount: 1,
Samples: []contracts.DiagnosticSample{{Scope: "scope", Message: "message"}},
}}},
OutputFiles: []contracts.OutputFile{{Name: "index.json"}},
}
}
func testRunOutputWithout(change func(*pipeline.RunOutput)) pipeline.RunOutput {
output := testRunOutput()
change(&output)
return output
}
func mustWorkingDirectory(t *testing.T) string {
t.Helper()
workingDirectory, err := filepath.Abs(".")
if err != nil {
t.Fatal(err)
}
return workingDirectory
}
type partialResultWriter struct { type partialResultWriter struct {
writer io.Writer content bytes.Buffer
limit int cause error
} }
func (w partialResultWriter) Write(content []byte) (int, error) { func (writer *partialResultWriter) Write(content []byte) (int, error) {
if len(content) > w.limit { if len(content) > 8 {
content = content[:w.limit] content = content[:8]
} }
return w.writer.Write(content) n, _ := writer.content.Write(content)
return n, writer.cause
} }
type failingResultWriter struct{ err error } var _ io.Writer = (*partialResultWriter)(nil)
func (w failingResultWriter) Write([]byte) (int, error) { return 0, w.err } func TestPipelinePreflightClassifiesOnlyIdentifiableInvalidSelections(t *testing.T) {
// Real native missing-resource errors retain their category separately from
type zeroResultWriter struct{} // private lookup causes. A joined operational branch must still win.
probe := &runProbe{}
func (zeroResultWriter) Write([]byte) (int, error) { return 0, nil } app, file, input, _ := runFixture(t, probe)
var diagnostics bytes.Buffer
missing := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--llm-profile", "absent"}, commandkit.Streams{Out: io.Discard, Err: &diagnostics}).Cause
for _, test := range []struct {
cause error
status int
}{
{missing, 2},
{errors.Join(missing, context.Canceled), 1},
{errors.Join(missing, errors.New("unavailable inspection")), 1},
{errors.New("unavailable inspection"), 1},
} {
outcome := pipelineResourceFailure(test.cause)
if outcome.Status != test.status || !errors.Is(outcome.Cause, test.cause) {
t.Fatalf("preflight classification: %#v", outcome)
}
}
}

View File

@@ -12,6 +12,21 @@ import (
"go.maximumdirect.net/promptkit" "go.maximumdirect.net/promptkit"
) )
// SelectionError identifies an absent configured native resource. Callers can
// distinguish this invalid selection from unavailable inspection or cancellation
// without inspecting diagnostic text or the kit's private lookup causes.
type SelectionError struct{ Cause error }
func (err *SelectionError) Error() string { return err.Cause.Error() }
func (err *SelectionError) Unwrap() error { return err.Cause }
func resourceFinding(err error) error {
if !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) && (errors.Is(err, promptkit.ErrPromptNotFound) || errors.Is(err, promptkit.ErrProfileNotFound)) {
return &SelectionError{Cause: err}
}
return err
}
// RequiresEngine reports selected model work, including nested validators. // RequiresEngine reports selected model work, including nested validators.
// Deterministic pipelines need no engine merely to validate their composition. // Deterministic pipelines need no engine merely to validate their composition.
func RequiresEngine(resolved pipeline.ResolvedPipeline) bool { func RequiresEngine(resolved pipeline.ResolvedPipeline) bool {
@@ -69,7 +84,7 @@ func Validate(ctx context.Context, resolved pipeline.ResolvedPipeline, catalog p
} }
if profile != "" { if profile != "" {
if _, err := engine.InspectProfile(ctx, profile); err != nil { if _, err := engine.InspectProfile(ctx, profile); err != nil {
findings = append(findings, fmt.Errorf("%s profile %q: %w", scope, profile, err)) findings = append(findings, resourceFinding(fmt.Errorf("%s profile %q: %w", scope, profile, err)))
} }
} }
for _, id := range prompts { for _, id := range prompts {
@@ -78,7 +93,7 @@ func Validate(ctx context.Context, resolved pipeline.ResolvedPipeline, catalog p
} }
prompt, err := engine.InspectPrompt(ctx, id) prompt, err := engine.InspectPrompt(ctx, id)
if err != nil { if err != nil {
findings = append(findings, fmt.Errorf("%s prompt %q: %w", scope, id, err)) findings = append(findings, resourceFinding(fmt.Errorf("%s prompt %q: %w", scope, id, err)))
continue continue
} }
if profile != "" { if profile != "" {
@@ -90,7 +105,7 @@ func Validate(ctx context.Context, resolved pipeline.ResolvedPipeline, catalog p
continue continue
} }
if _, err := engine.InspectProfile(ctx, *prompt.DefaultProfileID); err != nil { if _, err := engine.InspectProfile(ctx, *prompt.DefaultProfileID); err != nil {
findings = append(findings, fmt.Errorf("%s prompt %q default profile %q: %w", scope, id, *prompt.DefaultProfileID, err)) findings = append(findings, resourceFinding(fmt.Errorf("%s prompt %q default profile %q: %w", scope, id, *prompt.DefaultProfileID, err)))
} }
} }
} }