diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 8a36833b..84924f3a 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1606,6 +1606,30 @@ input-hash session stability across changed reference bundles. ## Stage 34: Finish Publication, Native Completion Reports, And Failure Outcomes +**Status:** Complete. Replaced the receipt/encoder/index and validation-ledger +reporting model with one native completion value and text template. Publication +and requested capture finish before reporting; JSON/YAML share the same value, +omit zero fallback counts and disabled debug paths, and never create reporting +sidecars. Qualified fallback summaries remain visible independently of warning +verbosity. Native progress, configuration diagnostics, and process findings use +stderr without promoting domain advisories to warnings. Operation causes remain +separate from native rendering/output/diagnostic delivery evidence. Failed capture +or presentation retains published files and available paths without model retries +or repeat terminalization. Identifiable preflight selections now retain a typed +classification alongside native causes; cancellation and unknown/mixed failures +remain operational. Replaced obsolete receipt tests with real-kit/fake-provider +coverage for formats, counts, native modes/progress, quiet rendering failure, +partial/failed stdout, failed diagnostics, cancellation, publication/capture +ordering, and primary/capture/delivery cause preservation. +Focused CLI/pipeline tests, supporting preflight/debug/cache/contracts/assets/ +generic/Seriatim tests, focused vet/race checks, and diff checks pass. Added real +executable format/fallback tests using a local fake HTTP provider; their setup and +the full CLI/executable checks remain blocked by legacy `framework/llm` importing +the removed reporting package and old ConfigKit's removed YAML `plugin/limit` +dependency. Those subprocess results are not claimed passing. Stage 35 owns +remaining legacy code/dependency/fixture removal and whole-tree compilation; +assembled and final verification must execute the pending checks. + **Goal:** subprocesses receive artifacts plus the selected small native report. **Inspect/change:** `run_result.go`, `run_terminal.go`, physical output placement, diff --git a/internal/cli/native_process_test.go b/internal/cli/native_process_test.go index 4afe5e9e..7ade7681 100644 --- a/internal/cli/native_process_test.go +++ b/internal/cli/native_process_test.go @@ -2,21 +2,34 @@ package cli import ( "bytes" + "encoding/json" "errors" + "fmt" + "net/http" + "net/http/httptest" "os" "os/exec" "path/filepath" "runtime" + "strings" "testing" + + "go.maximumdirect.net/commandkit" ) -func TestExecutableRunRoutesDeclareTheSameRequiredInputs(t *testing.T) { +func buildNativeExecutable(t *testing.T) string { + t.Helper() binary := filepath.Join(t.TempDir(), "notarius") build := exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"), "build", "-o", binary, "../../cmd/notarius") build.Env = append(os.Environ(), "GOWORK=off") if output, err := build.CombinedOutput(); err != nil { t.Fatalf("build executable: %v\n%s", err, output) } + return binary +} + +func TestExecutableRunRoutesDeclareTheSameRequiredInputs(t *testing.T) { + binary := buildNativeExecutable(t) for _, route := range [][]string{{"run"}, {"pipeline", "run"}} { help := exec.Command(binary, append(append([]string(nil), route...), "--help")...) output, err := help.CombinedOutput() @@ -31,3 +44,68 @@ func TestExecutableRunRoutesDeclareTheSameRequiredInputs(t *testing.T) { } } } + +func TestExecutableReportsDomainArtifactsAndQualifiedFallbacks(t *testing.T) { + binary := buildNativeExecutable(t) + for _, fallback := range []bool{false, true} { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if fallback { + writer.WriteHeader(http.StatusServiceUnavailable) + return + } + writer.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(writer).Encode(map[string]any{"id": "local-fake", "object": "chat.completion", "choices": []any{map[string]any{"index": 0, "message": map[string]any{"role": "assistant", "content": `{"spell_casts":[]}`}, "finish_reason": "stop"}}}) + })) + t.Cleanup(server.Close) + for _, format := range []commandkit.Format{commandkit.FormatText, commandkit.FormatJSON, commandkit.FormatYAML} { + t.Run(fmt.Sprintf("%s/fallback=%t", format, fallback), func(t *testing.T) { + root := t.TempDir() + configuration := fmt.Sprintf(`version: 6 +output: {directory: output} +pipelines: + demo: + input: {module: seriatim} + chunk: {module: generic} + llm_profile: local-fake + mode: permissive + max_additional_turns: 0 + artifacts: + spells: + extract: {module: dnd/spells} +promptkit: + profiles: + - id: local-fake + endpoint: %q + model: fake-model + api_key_required: false +`, server.URL+"/v1") + file := filepath.Join(root, "config.yml") + input := filepath.Join(root, "input.json") + for path, content := range map[string]string{file: configuration, input: `{"metadata":{"id":"report-source"},"segments":[{"id":1,"start":0,"end":1,"speaker":"GM","text":"The party rests."}]}`} { + if err := os.WriteFile(path, []byte(content), 0600); err != nil { + t.Fatal(err) + } + } + command := exec.Command(binary, "run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--llm-transient-retries=0", "--format", string(format), "--verbose") + var output, diagnostics bytes.Buffer + command.Stdout, command.Stderr = &output, &diagnostics + if err := command.Run(); err != nil { + t.Fatalf("subprocess: %v stdout=%s stderr=%s", err, output.String(), diagnostics.String()) + } + if format == commandkit.FormatText { + if !strings.Contains(output.String(), "produced 1 artifacts") || fallback != strings.Contains(output.String(), "Success with fallback:") { + t.Fatalf("completion text: %s", output.String()) + } + } else { + report := decodeCompletionReport(t, format, output.Bytes()) + if report["artifact_count"] != float64(1) || report["pipeline"] != "demo" || fallback && report["fallback_count"] != float64(1) || !fallback && report["fallback_count"] != nil { + t.Fatalf("completion report: %#v", report) + } + if _, err := os.Stat(filepath.Join(report["output_directory"].(string), "lanes", "spells.json")); err != nil { + t.Fatal(err) + } + } + }) + } + } +} diff --git a/internal/cli/pipeline_commands.go b/internal/cli/pipeline_commands.go index 73b5afaf..612c6912 100644 --- a/internal/cli/pipeline_commands.go +++ b/internal/cli/pipeline_commands.go @@ -107,7 +107,7 @@ func pipelineCommands(newState func() *nativeLoaders, catalog func() (pipeline.M } } if err := preflight.Validate(invocation.Context, effective.ResolvedPipeline, modules, engine); err != nil { - return pipelineInvalidConfiguration(err) + return pipelineResourceFailure(err) } return reportPipelineInspection(invocation, commandkit.Report{Value: struct { Pipeline string `json:"pipeline"` @@ -124,16 +124,56 @@ var pipelineValidationTemplate = template.Must(commandkit.ParseTextTemplate("pip func reportPipelineInspection(invocation commandkit.Invocation, report commandkit.Report) commandkit.Outcome { if err := invocation.Reporter().Report(report); err != nil { - return commandkit.Failure("Could not report pipeline inspection.").WithCause(err) + return commandkit.Failure("Could not report pipeline inspection.") } return commandkit.Success() } func pipelineConfigurationFailure(err error) commandkit.Outcome { + outcome := commandkit.Failure(err.Error()).WithCause(err) if ckconfig.ClassifyConfigurationError(err) == commandkit.OutcomeUsage { + outcome = commandkit.Usage(err.Error()).WithCause(err) + } + var load *configkit.LoadError + if errors.As(err, &load) { + _, explanation := ckconfig.AdaptDiagnostics(err.Error()+"\n", load.Sources(), load.Diagnostics()) + outcome = outcome.WithExplanation(explanation.Message, explanation.Details...) + } + return outcome +} + +func pipelineResourceFailure(err error) commandkit.Outcome { + if preflightInputError(err) { return commandkit.Usage(err.Error()).WithCause(err) } - return commandkit.Failure(err.Error()).WithCause(err) + return pipelineConfigurationFailure(err) +} + +// Only identifiable missing native selections are invalid preflight input. +// Unknown failures and any operational branch of a joined error remain failures. +func preflightInputError(err error) bool { + if err == nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return false + } + if _, ok := err.(*preflight.SelectionError); ok { + return true + } + if joined, ok := err.(interface{ Unwrap() []error }); ok { + found := false + for _, child := range joined.Unwrap() { + if child != nil { + found = true + if !preflightInputError(child) { + return false + } + } + } + return found + } + if wrapped, ok := err.(interface{ Unwrap() error }); ok { + return preflightInputError(wrapped.Unwrap()) + } + return false } func pipelineInvalidConfiguration(err error) commandkit.Outcome { diff --git a/internal/cli/pipeline_commands_test.go b/internal/cli/pipeline_commands_test.go index eba10936..59b3b7f1 100644 --- a/internal/cli/pipeline_commands_test.go +++ b/internal/cli/pipeline_commands_test.go @@ -262,7 +262,7 @@ func TestPipelineInspectionReportFailureRemainsOperational(t *testing.T) { primary := errors.New("inspection stdout delivery failed") var diagnostics bytes.Buffer outcome := app.Execute(context.Background(), []string{"pipeline", "list", "--config", file}, commandkit.Streams{Out: inspectionFailWriter{primary}, Err: &diagnostics}) - if outcome.Status != 1 || !errors.Is(outcome.Cause, primary) { + if outcome.Status != 1 || outcome.Cause != nil || !errors.Is(outcome.Presentation.Output.WriteError, primary) { t.Fatalf("delivery failure classified as invalid configuration: %#v %s", outcome, diagnostics.String()) } } diff --git a/internal/cli/pipeline_run.go b/internal/cli/pipeline_run.go index 581e56ce..321b7da3 100644 --- a/internal/cli/pipeline_run.go +++ b/internal/cli/pipeline_run.go @@ -11,6 +11,7 @@ import ( frameworkdebug "gitea.maximumdirect.net/eric/notarius/internal/framework/debug" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/preflight" + "go.maximumdirect.net/ckconfig" "go.maximumdirect.net/commandkit" "go.maximumdirect.net/configkit" "go.maximumdirect.net/promptkit" @@ -31,6 +32,14 @@ func pipelineRunCommand(version string, newState func() *nativeLoaders, modules if err != nil { return pipelineConfigurationFailure(err) } + diagnostics, _ := ckconfig.AdaptDiagnostics("Configuration\n", result.Sources(), result.Diagnostics()) + for _, diagnostic := range diagnostics { + if diagnostic.Severity == commandkit.SeverityWarning { + _ = invocation.Reporter().AtEnd().Warn("", diagnostic.Write) + } else { + _ = invocation.Reporter().AtEnd().Info("", diagnostic.Write) + } + } loaded, err := config.FromResult(result) if err != nil { return pipelineInvalidConfiguration(err) @@ -53,20 +62,20 @@ func pipelineRunCommand(version string, newState func() *nativeLoaders, modules } finish := func(outcome commandkit.Outcome) commandkit.Outcome { if err := capture.Finalize(captureOutcome(), outcome.Cause); err != nil { - message := err.Error() - if published { - message += fmt.Sprintf("\nArtifacts: %s", outputDirectory) - } - if capture.Path() != "" { - message += fmt.Sprintf("\nDebug capture: %s", capture.Path()) - } if outcome.Status == 0 { - outcome = commandkit.Failure(message) + outcome = commandkit.Failure(err.Error()) } else { - outcome = outcome.WithExplanation(message) + outcome = outcome.WithExplanation(err.Error(), outcome.Explanation.Details...) } outcome = outcome.WithCause(err) } + if outcome.Status != 0 { + availableOutput := "" + if published { + availableOutput = outputDirectory + } + outcome = outcome.WithExplanation(outcome.Explanation.Message+runLocations(availableOutput, capture.Path()), outcome.Explanation.Details...) + } return outcome } capture, err = startRunDebug(input.Debug, cfg.Debug.Directory, runID, startedAt) @@ -107,13 +116,13 @@ func pipelineRunCommand(version string, newState func() *nativeLoaders, modules } } if err := preflight.Validate(invocation.Context, effective.ResolvedPipeline, catalog, engine); err != nil { - return finish(pipelineInvalidConfiguration(err)) + return finish(pipelineResourceFailure(err)) } workingDirectory, err := os.Getwd() if err != nil { return finish(commandkit.Failure(err.Error()).WithCause(err)) } - materialized, diagnostics, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{ConfigPath: loaded.MainSourcePath(), WorkingDir: workingDirectory}) + materialized, referenceDiagnostics, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{ConfigPath: loaded.MainSourcePath(), WorkingDir: workingDirectory}) if err != nil { return finish(commandkit.Failure(err.Error()).WithCause(err)) } @@ -138,7 +147,9 @@ func pipelineRunCommand(version string, newState func() *nativeLoaders, modules recorder = newRunEvidenceRecorder(capture, result) } group := &promptkit.ExecutionGroup{} - output, err = pipeline.New().Run(invocation.Context, pipeline.RunInput{Prepared: prepared, Path: input.InputFile.Value, RawInput: raw, SessionID: sessionID, RunID: runID, StartedAt: startedAt, Diagnostics: diagnostics, Cache: cache, Debug: recorder, Workers: cfg.Concurrency.Workers, Executions: group, Execution: contracts.ExecutionOptions{Execution: input.Overrides.Execution.Execution, TransientRetries: input.Overrides.Execution.TransientRetries}}) + _ = invocation.Reporter().Progress(fmt.Sprintf("Running pipeline %q.", effective.ResolvedPipeline.ID)) + output, err = pipeline.New().Run(invocation.Context, pipeline.RunInput{Prepared: prepared, Path: input.InputFile.Value, RawInput: raw, SessionID: sessionID, RunID: runID, StartedAt: startedAt, Diagnostics: referenceDiagnostics, Cache: cache, Debug: recorder, Workers: cfg.Concurrency.Workers, Executions: group, Execution: contracts.ExecutionOptions{Execution: input.Overrides.Execution.Execution, TransientRetries: input.Overrides.Execution.TransientRetries}}) + reportRunDiagnostics(invocation.Reporter(), output.Diagnostics) if err != nil { return finish(commandkit.Failure(err.Error()).WithCause(err)) } @@ -146,7 +157,11 @@ func pipelineRunCommand(version string, newState func() *nativeLoaders, modules return finish(commandkit.Failure(err.Error()).WithCause(err)) } published = true - return finish(commandkit.Success()) + outcome := finish(commandkit.Success()) + if outcome.Status != 0 { + return outcome + } + return reportRunResult(invocation, runResult{Pipeline: effective.ResolvedPipeline.ID, ArtifactCount: output.ArtifactCount(), OutputDirectory: outputDirectory, DebugDirectory: capture.Path(), FallbackCount: output.FallbackCount}) }, }) } diff --git a/internal/cli/pipeline_run_test.go b/internal/cli/pipeline_run_test.go index f715d619..64502f9d 100644 --- a/internal/cli/pipeline_run_test.go +++ b/internal/cli/pipeline_run_test.go @@ -31,12 +31,20 @@ type runProbe struct { parsed int produced int correctFirst bool + providerErr error + onGenerate func() } func (probe *runProbe) Generate(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { probe.mu.Lock() defer probe.mu.Unlock() probe.requests = append(probe.requests, request) + if probe.onGenerate != nil { + probe.onGenerate() + } + if probe.providerErr != nil { + return nil, probe.providerErr + } value := "accepted" if request.Prompt.Messages[len(request.Prompt.Messages)-1].Content != "Assess." { probe.produced++ @@ -98,7 +106,9 @@ func (producer *runProducer) Extract(ctx context.Context, request contracts.Type return &value, nil, err }, func(context.Context, []string) (promptkit.RunRequest, error) { return promptkit.RunRequest{PromptID: "inspect-correction", ProfileID: request.LLMProfile, SessionID: request.SessionID, OnPrepared: producer.probe.onPrepared}, nil - }, nil) + }, func(context.Context, *inspectionArtifact) (inspectionArtifact, error) { + return inspectionArtifact{Value: "fallback"}, nil + }) return contracts.TypedExtractionResult[inspectionArtifact]{Value: value, Outcome: outcome}, err } func (producer *runProducer) Merge(_ context.Context, request contracts.TypedMergeRequest[inspectionArtifact]) (contracts.TypedMergeResult[inspectionArtifact], error) { diff --git a/internal/cli/run_result.go b/internal/cli/run_result.go index b421d7e0..74714aa9 100644 --- a/internal/cli/run_result.go +++ b/internal/cli/run_result.go @@ -1,142 +1,78 @@ package cli import ( - "encoding/json" "fmt" "io" - "path/filepath" "strings" + "text/template" - "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" - "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" + "go.maximumdirect.net/commandkit" ) -const runResultSchemaVersion = "notarius.run-result.v2" - +// runResult is the small completion value. Domain files, not a saved report or +// execution ledger, are the durable deliverables. type runResult struct { - SchemaVersion string `json:"schema_version"` - RunID string `json:"run_id"` - PipelineID string `json:"pipeline_id"` - OutputDirectory string `json:"output_directory"` - IndexFile string `json:"index_file,omitempty"` - NormalizedOutputCount int `json:"normalized_output_count"` - RejectedOutputCount int `json:"rejected_output_count"` - WarningGroupCount int `json:"warning_group_count"` - WarningOccurrenceCount int `json:"warning_occurrence_count"` - DiagnosticGroupCount int `json:"diagnostic_group_count"` - DiagnosticOccurrenceCount int `json:"diagnostic_occurrence_count"` - DiagnosticsTruncated bool `json:"diagnostics_truncated"` - ValidationStatus string `json:"validation_status"` - ValidationSummaries []artifacts.ValidationSummary `json:"validation_summaries,omitempty"` - DebugDirectory string `json:"debug_directory,omitempty"` + Pipeline string `json:"pipeline"` + ArtifactCount int `json:"artifact_count"` + OutputDirectory string `json:"output_directory"` + DebugDirectory string `json:"debug_directory,omitempty"` + FallbackCount int `json:"fallback_count,omitempty"` } -func newRunResult(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput, outputDirectory, debugDirectory string) (runResult, error) { - diagnosticProjection, err := contracts.ProjectDiagnosticCollection(output.Diagnostics) - if err != nil { - return runResult{}, fmt.Errorf("summarize run diagnostics: %w", err) +var runResultTemplate = template.Must(commandkit.ParseTextTemplate("run-completion", `{{if .FallbackCount}}Success with fallback: {{else}}Success: {{end}}{{.Pipeline}} produced {{.ArtifactCount}} artifacts in {{.OutputDirectory}} +{{if .FallbackCount}}{{.FallbackCount}} operations used fallback. +{{end}}{{if .DebugDirectory}}Debug capture: {{.DebugDirectory}} +{{end}}`, nil)) + +func reportRunResult(invocation commandkit.Invocation, result runResult) commandkit.Outcome { + message := "Could not present the completed run." + runLocations(result.OutputDirectory, result.DebugDirectory) + if err := invocation.Reporter().Report(commandkit.Report{Value: result, TextTemplate: runResultTemplate}); err != nil { + // The reporter retains render/write failures in Presentation. They are + // not a replacement operation Cause, and capture is already finalized. + return commandkit.Failure(message) } - return newRunResultWithDiagnostics(resolved, output, outputDirectory, debugDirectory, diagnosticProjection) + // Deferred diagnostic delivery can still fail during native finalization. + // Supply useful locations without presenting a failure on successful runs. + return commandkit.Success().WithExplanation(message) } -func newRunResultWithDiagnostics(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput, outputDirectory, debugDirectory string, diagnosticProjection contracts.DiagnosticProjection) (runResult, error) { - if strings.TrimSpace(output.Manifest.RunID) == "" { - return runResult{}, fmt.Errorf("run result requires a run ID") +func runLocations(outputDirectory, debugDirectory string) string { + var locations string + if outputDirectory != "" { + locations += "\nArtifacts: " + outputDirectory } - if strings.TrimSpace(resolved.ID) == "" { - return runResult{}, fmt.Errorf("run result requires a resolved pipeline ID") - } - if strings.TrimSpace(output.Manifest.PipelineID) == "" { - return runResult{}, fmt.Errorf("run result requires a manifest pipeline ID") - } - if output.Manifest.PipelineID != resolved.ID { - return runResult{}, fmt.Errorf("run result pipeline ID does not match resolved pipeline") - } - if strings.TrimSpace(output.Manifest.ValidationStatus) == "" { - return runResult{}, fmt.Errorf("run result requires a validation status") - } - if strings.TrimSpace(outputDirectory) == "" { - return runResult{}, fmt.Errorf("run result requires an output directory") + if debugDirectory != "" { + locations += "\nDebug capture: " + debugDirectory } + return locations +} - absOutputDirectory, err := filepath.Abs(outputDirectory) - if err != nil { - return runResult{}, fmt.Errorf("make output directory absolute: %w", err) - } - - result := runResult{ - SchemaVersion: runResultSchemaVersion, - RunID: output.Manifest.RunID, - PipelineID: output.Manifest.PipelineID, - OutputDirectory: absOutputDirectory, - NormalizedOutputCount: len(output.NormalizeOutputs), - RejectedOutputCount: len(output.Rejected), - WarningGroupCount: len(diagnosticProjection.Warnings), - WarningOccurrenceCount: diagnosticProjection.WarningOccurrenceCount, - DiagnosticGroupCount: len(diagnosticProjection.Diagnostics), - DiagnosticOccurrenceCount: diagnosticProjection.DiagnosticOccurrenceCount, - DiagnosticsTruncated: output.Diagnostics.Truncated, - ValidationStatus: output.Manifest.ValidationStatus, - ValidationSummaries: cloneValidationSummaries(output.Manifest.ValidationSummaries), - } - - if strings.TrimSpace(debugDirectory) != "" { - absDebugDirectory, err := filepath.Abs(debugDirectory) - if err != nil { - return runResult{}, fmt.Errorf("make debug directory absolute: %w", err) - } - result.DebugDirectory = absDebugDirectory - } - - if resolved.Output.Module == pipeline.DefaultOutputModule { - indexCount := 0 - for _, file := range output.OutputFiles { - if file.Name == "index.json" { - indexCount++ +func reportRunDiagnostics(reporter *commandkit.Reporter, collection contracts.DiagnosticCollection) { + for _, group := range collection.Groups { + origin := group.Origin + scope := []string{string(origin.Stage)} + for _, value := range []string{origin.StepID, origin.LaneID, origin.ModuleKey, origin.ValidatorKey} { + if value != "" { + scope = append(scope, value) } } - if indexCount != 1 { - return runResult{}, fmt.Errorf("production JSON output must contain exactly one index.json file") + heading := strings.Join(scope, " / ") + ": " + group.ReasonCode + detail := func(writer io.Writer) error { + for _, sample := range group.Samples { + if _, err := fmt.Fprintf(writer, "%s: %s\n", sample.Scope, sample.Message); err != nil { + return err + } + } + return nil } - result.IndexFile = "index.json" - } - - return result, nil -} - -func cloneValidationSummaries(summaries []artifacts.ValidationSummary) []artifacts.ValidationSummary { - if len(summaries) == 0 { - return nil - } - cloned := make([]artifacts.ValidationSummary, len(summaries)) - for index, summary := range summaries { - cloned[index] = artifacts.CloneValidationSummary(summary) - } - return cloned -} - -func encodeRunResult(result runResult) ([]byte, error) { - encoded, err := json.Marshal(result) - if err != nil { - return nil, fmt.Errorf("encode run result: %w", err) - } - return append(encoded, '\n'), nil -} - -func writeRunResult(writer io.Writer, content []byte) error { - for len(content) > 0 { - written, err := writer.Write(content) - if written < 0 || written > len(content) { - return io.ErrShortWrite - } - content = content[written:] - if err != nil { - return err - } - if written == 0 { - return io.ErrShortWrite + if group.Disposition == contracts.DiagnosticDispositionWarning { + _ = reporter.AtEnd().Warn(heading, detail) + } else { + _ = reporter.AtEnd().Info(heading, detail) } } - return nil + if collection.Truncated { + _ = reporter.AtEnd().Info("Additional pipeline diagnostics were omitted.") + } } diff --git a/internal/cli/run_result_command_test.go b/internal/cli/run_result_command_test.go index 3c4bc523..bf43598f 100644 --- a/internal/cli/run_result_command_test.go +++ b/internal/cli/run_result_command_test.go @@ -2,198 +2,318 @@ package cli import ( "bytes" + "context" "encoding/json" "errors" + "io" "os" "path/filepath" "strings" + "sync" "testing" - "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" - alwaysreject "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/validate/always_reject" + "go.maximumdirect.net/commandkit" + "go.maximumdirect.net/configkit" ) -func TestMaintainedMinimalInvocationEmitsRunResult(t *testing.T) { - outputRoot := filepath.Join(t.TempDir(), "output") - var stdout, stderr strings.Builder - code := RunWithOptions([]string{ - "run", "dnd-session", - "--config", repositoryPath("examples", "dnd-minimal.config.yml"), - "--input", repositoryPath("examples", "seriatim-minimal-transcript.json"), - "--only", "spells", "--chunk_cache", "bypass", "--output-dir", outputRoot, "--json", - }, &stdout, &stderr, productionRunOptions(t, &productionFakeLLMClient{})) - if code != 0 || stderr.Len() != 0 { - t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) - } - - receipt := decodeRunResultDocument(t, stdout.String()) - if got := receipt["schema_version"]; got != "notarius.run-result.v2" { - t.Fatalf("schema_version = %q", got) - } - if got := receipt["run_id"]; got != productionRunID { - t.Fatalf("run_id = %q", got) - } - if got := receipt["pipeline_id"]; got != "dnd-session" { - t.Fatalf("pipeline_id = %q", got) - } - if got := receipt["index_file"]; got != "index.json" { - t.Fatalf("index_file = %q", got) - } - if got := receipt["normalized_output_count"]; got != float64(1) { - t.Fatalf("normalized_output_count = %v", got) - } - if got := receipt["rejected_output_count"]; got != float64(0) { - t.Fatalf("rejected_output_count = %v", got) - } - if got := receipt["warning_group_count"]; got != float64(0) || receipt["warning_occurrence_count"] != float64(0) || receipt["diagnostic_group_count"] != float64(0) || receipt["diagnostic_occurrence_count"] != float64(0) || receipt["diagnostics_truncated"] != false { - t.Fatalf("diagnostic counts = %#v", receipt) - } - if got := receipt["validation_status"]; got != "approved" { - t.Fatalf("validation_status = %q", got) - } - - outputDirectory, ok := receipt["output_directory"].(string) - if !ok || !filepath.IsAbs(outputDirectory) || outputDirectory != filepath.Join(outputRoot, productionRunID) { - t.Fatalf("output_directory = %q", receipt["output_directory"]) - } - indexFile := receipt["index_file"].(string) - assertFile(t, filepath.Join(outputDirectory, indexFile)) -} - -func TestRunResultReportsWarningsAndDebugBundle(t *testing.T) { - roots := newStateTestRoots(t) - harness := newStateTestHarness() - harness.chunkDiagnostics = []contracts.ProducerDiagnostic{stateTestDiagnostic("chunk", "contract-warning", "warning retained")} - var stdout, stderr bytes.Buffer - code := RunWithOptions([]string{ - "run", "sample", "--config", roots.config, "--input", roots.input, - "--chunk_cache", "bypass", "--debug", "--json", - }, &stdout, &stderr, harness.options()) - if code != 0 || !strings.Contains(stderr.String(), "1 warning group(s), 1 occurrence(s)") || strings.Contains(stderr.String(), "warnings.json") { - t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) - } - - receipt := decodeRunResultDocument(t, stdout.String()) - if got := receipt["warning_group_count"]; got != float64(1) || receipt["warning_occurrence_count"] != float64(1) || receipt["diagnostic_group_count"] != float64(0) || receipt["diagnostic_occurrence_count"] != float64(0) || receipt["diagnostics_truncated"] != false { - t.Fatalf("diagnostic counts = %#v", receipt) - } - debugDirectory, ok := receipt["debug_directory"].(string) - if !ok || !filepath.IsAbs(debugDirectory) || debugDirectory != onlyChildDir(t, roots.debug) { - t.Fatalf("debug_directory = %q", receipt["debug_directory"]) - } - if strings.Contains(stdout.String(), "complete:") || strings.Contains(stdout.String(), "debug=") { - t.Fatalf("machine stdout contains human reporting: %q", stdout.String()) - } -} - -func TestRunResultReportsSuccessfulRejection(t *testing.T) { - roots := newStateTestRoots(t) - configBytes, err := os.ReadFile(roots.config) - if err != nil { - t.Fatal(err) - } - configBytes = []byte(replaceRequiredOnce(t, string(configBytes), " normalize:\n module: test/normalize\n", " normalize:\n module: test/normalize\n validators:\n - module: generic/always_reject\n validation_policy:\n semantic_rejection: reject_output\n")) - if err := os.WriteFile(roots.config, configBytes, 0o600); err != nil { - t.Fatal(err) - } - - harness := newStateTestHarness() - opts := harness.options() - if err := alwaysreject.RegisterTyped[stateTestArtifact](opts.Registries.Validators, stateTestArtifactKind); err != nil { - t.Fatal(err) - } - opts.Catalog = catalogFromRegistries(opts.Registries) - - var stdout, stderr bytes.Buffer - code := RunWithOptions([]string{ - "run", "sample", "--config", roots.config, "--input", roots.input, - "--chunk_cache", "bypass", "--json", - }, &stdout, &stderr, opts) - if code != 0 || stderr.Len() != 0 { - t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) - } - receipt := decodeRunResultDocument(t, stdout.String()) - if got := receipt["normalized_output_count"]; got != float64(0) { - t.Fatalf("normalized_output_count = %v", got) - } - if got := receipt["rejected_output_count"]; got != float64(1) { - t.Fatalf("rejected_output_count = %v", got) - } - if got := receipt["validation_status"]; got != "rejected" { - t.Fatalf("validation_status = %q", got) - } -} - -func TestRunResultIsAbsentForSyntaxAndRuntimeFailures(t *testing.T) { - t.Run("syntax", func(t *testing.T) { - var stdout, stderr bytes.Buffer - code := RunWithOptions([]string{"run", "sample", "--json"}, &stdout, &stderr, newStateTestHarness().options()) - if code != 2 || stdout.Len() != 0 || stderr.Len() == 0 { - t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) - } - }) - - t.Run("runtime", func(t *testing.T) { - roots := newStateTestRoots(t) - harness := newStateTestHarness() - harness.extractErr = errors.New("injected extraction failure") - var stdout, stderr bytes.Buffer - code := RunWithOptions([]string{ - "run", "sample", "--config", roots.config, "--input", roots.input, - "--chunk_cache", "bypass", "--json", - }, &stdout, &stderr, harness.options()) - if code != 1 || stdout.Len() != 0 || stderr.Len() == 0 { - t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) - } - }) -} - -func TestRunResultDeliveryFailureRetainsPublishedBundles(t *testing.T) { - roots := newStateTestRoots(t) - writerErr := errors.New("result writer sentinel") - stdout := &resultDeliveryWriter{err: writerErr} - var stderr bytes.Buffer - code := RunWithOptions([]string{ - "run", "sample", "--config", roots.config, "--input", roots.input, - "--chunk_cache", "bypass", "--debug", "--json", - }, stdout, &stderr, newStateTestHarness().options()) - if code != 1 || !strings.Contains(stderr.String(), "write run result") || strings.Contains(stderr.String(), writerErr.Error()) { - t.Fatalf("code=%d stderr=%q", code, stderr.String()) - } - if stdout.accepted.Len() != 0 { - t.Fatalf("accepted stdout = %q", stdout.accepted.String()) - } - assertStateTestOutput(t, roots.output) - debugBundle := onlyChildDir(t, roots.debug) - report := readStateTestRunReport(t, debugBundle) - if !report.Succeeded { - t.Fatalf("debug report = %#v, want successful persisted run", report) - } - if strings.Contains(readAllFiles(t, debugBundle), writerErr.Error()) { - t.Fatalf("debug bundle contains result writer error") - } -} - -func decodeRunResultDocument(t *testing.T, stdout string) map[string]any { +func decodeCompletionReport(t *testing.T, format commandkit.Format, content []byte) map[string]any { t.Helper() - if strings.Count(stdout, "\n") != 1 { - t.Fatalf("stdout = %q, want one JSON document", stdout) + if format == commandkit.FormatYAML { + value, err := commandkit.DecodeYAMLReport(bytes.NewReader(content)) + if err != nil { + t.Fatalf("decode native YAML report: %v\n%s", err, content) + } + content, err = json.Marshal(value) + if err != nil { + t.Fatal(err) + } } - var receipt map[string]any - if err := json.Unmarshal([]byte(stdout), &receipt); err != nil { - t.Fatalf("decode run result: %v; stdout=%q", err, stdout) + var report map[string]any + if err := json.Unmarshal(content, &report); err != nil { + t.Fatalf("decode native report: %v\n%s", err, content) } - return receipt + return report } -type resultDeliveryWriter struct { - err error - accepted bytes.Buffer +func assertCompletedRunFiles(t *testing.T, file string, captured bool) { + t.Helper() + run := filepath.Join(filepath.Dir(file), "output-must-not-exist", "run-1") + if content, err := os.ReadFile(filepath.Join(run, "lanes", "notes.json")); err != nil || len(content) == 0 { + t.Fatalf("published domain artifact: %s %v", content, err) + } + files, err := os.ReadDir(run) + if err != nil || len(files) != 1 || files[0].Name() != "lanes" { + t.Fatalf("unexpected reporting sidecars: %#v %v", files, err) + } + if captured { + debug := filepath.Join(filepath.Dir(file), "debug-must-not-exist", "run-1") + content, err := os.ReadFile(filepath.Join(debug, "outcome.json")) + if err != nil { + t.Fatal(err) + } + var terminal struct { + Outcome struct { + Published bool `json:"published"` + } `json:"outcome"` + Error string `json:"error"` + CaptureError string `json:"capture_error"` + } + if err := json.Unmarshal(content, &terminal); err != nil || !terminal.Outcome.Published || terminal.Error != "" || terminal.CaptureError != "" { + t.Fatalf("capture was not completed before reporting: %s %v", content, err) + } + } } -func (w *resultDeliveryWriter) Write(content []byte) (int, error) { - if w.err != nil { - return 0, w.err - } - return w.accepted.Write(content) +type completedRunWriter struct { + t *testing.T + file string + captured bool + content bytes.Buffer +} + +func (writer *completedRunWriter) Write(content []byte) (int, error) { + assertCompletedRunFiles(writer.t, writer.file, writer.captured) + return writer.content.Write(content) +} + +func TestNativeRunCompletionFormatsFollowPublicationAndCapture(t *testing.T) { + for _, format := range []commandkit.Format{commandkit.FormatText, commandkit.FormatJSON, commandkit.FormatYAML} { + t.Run(string(format), func(t *testing.T) { + probe := &runProbe{} + app, file, input, _ := runFixture(t, probe) + output := &completedRunWriter{t: t, file: file, captured: true} + var diagnostics bytes.Buffer + outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--format", string(format)}, commandkit.Streams{Out: output, Err: &diagnostics}) + if outcome.Status != 0 || !strings.Contains(diagnostics.String(), "Running pipeline") { + t.Fatalf("completion: %#v stderr=%s", outcome, diagnostics.String()) + } + directory := filepath.Join(filepath.Dir(file), "output-must-not-exist", "run-1") + debug := filepath.Join(filepath.Dir(file), "debug-must-not-exist", "run-1") + if format == commandkit.FormatText { + if !strings.Contains(output.content.String(), "Success: demo produced 1 artifacts in "+directory) || !strings.Contains(output.content.String(), "Debug capture: "+debug) || strings.Contains(output.content.String(), "fallback") { + t.Fatalf("native completion text: %s", output.content.String()) + } + } else { + report := decodeCompletionReport(t, format, output.content.Bytes()) + if len(report) != 4 || report["pipeline"] != "demo" || report["artifact_count"] != float64(1) || report["output_directory"] != directory || report["debug_directory"] != debug { + t.Fatalf("native report: %#v", report) + } + } + if probe.produced != 1 { + t.Fatalf("reporting reran a producer: %d", probe.produced) + } + }) + } +} + +func TestNativeRunDisclosesFallbackIndependentlyOfDiagnosticVisibility(t *testing.T) { + for _, format := range []commandkit.Format{commandkit.FormatText, commandkit.FormatJSON, commandkit.FormatYAML} { + for _, verbose := range []bool{false, true} { + probe := &runProbe{correctFirst: true} + app, file, input, _ := runFixture(t, probe) + args := []string{"pipeline", "run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--set", `/pipelines/demo/mode="permissive"`, "--set", "/pipelines/demo/max_additional_turns=0", "--format", string(format)} + if verbose { + args = append(args, "--verbose") + } + var output, diagnostics bytes.Buffer + outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: &diagnostics}) + if outcome.Status != 0 || probe.produced != 1 { + t.Fatalf("permissive completion: %#v %s", outcome, diagnostics.String()) + } + if verbose && !strings.Contains(diagnostics.String(), "WARN:") || !verbose && diagnostics.Len() != 0 { + t.Fatalf("native warning visibility: %s", diagnostics.String()) + } + if format == commandkit.FormatText { + if !strings.Contains(output.String(), "Success with fallback:") || !strings.Contains(output.String(), "1 operations used fallback.") { + t.Fatalf("fallback text: %s", output.String()) + } + } else { + report := decodeCompletionReport(t, format, output.Bytes()) + if report["fallback_count"] != float64(1) || report["artifact_count"] != float64(1) || len(report) != 4 { + t.Fatalf("fallback report: %#v", report) + } + } + assertCompletedRunFiles(t, file, false) + } + } +} + +func TestNativeRunQuietAndDebugUseNativeModeOrdering(t *testing.T) { + for _, test := range []struct { + flags []string + quiet bool + progress bool + }{ + {[]string{"--debug", "--quiet"}, true, false}, + {[]string{"--quiet", "--debug"}, false, true}, + {[]string{"--debug", "--progress=false"}, false, false}, + } { + probe := &runProbe{} + app, file, input, _ := runFixture(t, probe) + var output, diagnostics bytes.Buffer + args := append([]string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--format", "json"}, test.flags...) + outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: &diagnostics}) + if outcome.Status != 0 || test.quiet != (output.Len() == 0) || test.progress != strings.Contains(diagnostics.String(), "Running pipeline") { + t.Fatalf("native mode order: %#v output=%s stderr=%s", outcome, output.String(), diagnostics.String()) + } + assertCompletedRunFiles(t, file, true) + } +} + +func TestNativeRunCancellationKeepsItsCauseAndPublishesNothing(t *testing.T) { + probe := &runProbe{} + app, file, input, _ := runFixture(t, probe) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + probe.onGenerate = cancel + var output, diagnostics bytes.Buffer + outcome := app.Execute(ctx, []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--quiet"}, commandkit.Streams{Out: &output, Err: &diagnostics}) + if outcome.Status != 1 || !errors.Is(outcome.Cause, context.Canceled) || output.Len() != 0 || diagnostics.Len() == 0 { + t.Fatalf("cancellation: %#v stdout=%s stderr=%s", outcome, output.String(), diagnostics.String()) + } + if _, err := os.Stat(filepath.Join(filepath.Dir(file), "output-must-not-exist")); !os.IsNotExist(err) { + t.Fatalf("cancellation published artifacts: %v", err) + } +} + +func TestNativeRunPrimaryFailureSurvivesCaptureAndDiagnosticFailures(t *testing.T) { + primary := errors.New("provider failure sentinel") + delivery := errors.New("diagnostic failure sentinel") + probe := &runProbe{providerErr: primary} + app, file, input, _ := runFixture(t, probe) + debug := filepath.Join(filepath.Dir(file), "debug-must-not-exist", "run-1") + probe.onGenerate = func() { + if err := os.Mkdir(filepath.Join(debug, "outcome.json"), 0700); err != nil { + t.Error(err) + } + } + var output bytes.Buffer + outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--llm-transient-retries=0"}, commandkit.Streams{Out: &output, Err: failingResultWriter{cause: delivery}}) + var captureFailure *os.LinkError + if outcome.Status != 1 || !errors.Is(outcome.Cause, primary) || !errors.As(outcome.Cause, &captureFailure) || errors.Is(outcome.Cause, delivery) || !errors.Is(outcome.Presentation.Diagnostics.WriteError, delivery) || output.Len() != 0 || len(probe.requests) != 1 { + t.Fatalf("primary/capture/delivery failure: %#v", outcome) + } + if !strings.Contains(outcome.Explanation.Message, debug) { + t.Fatalf("failed run capture location unavailable: %#v", outcome) + } +} + +func TestNativeRunReportDeliveryFailuresRetainArtifactsAndNativePresentation(t *testing.T) { + sentinel := errors.New("result delivery sentinel") + for _, partial := range []bool{false, true} { + probe := &runProbe{} + app, file, input, _ := runFixture(t, probe) + var output io.Writer = failingResultWriter{cause: sentinel} + prefix := &partialResultWriter{cause: sentinel} + if partial { + output = prefix + } + var diagnostics bytes.Buffer + outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--format", "json"}, commandkit.Streams{Out: output, Err: &diagnostics}) + if outcome.Status != 1 || outcome.Cause != nil || !errors.Is(outcome.Presentation.Output.WriteError, sentinel) || outcome.Presentation.Output.State != commandkit.DeliveryIncomplete || probe.produced != 1 || partial && prefix.content.Len() == 0 { + t.Fatalf("delivery failure: %#v %s", outcome, diagnostics.String()) + } + assertCompletedRunFiles(t, file, true) + for _, root := range []string{"output-must-not-exist", "debug-must-not-exist"} { + if !strings.Contains(diagnostics.String(), filepath.Join(filepath.Dir(file), root, "run-1")) { + t.Fatalf("missing available path after delivery failure: %s", diagnostics.String()) + } + } + } +} + +func TestNativeRunDiagnosticDeliveryFailureIsNotAnOperationFailure(t *testing.T) { + sentinel := errors.New("diagnostic delivery sentinel") + probe := &runProbe{correctFirst: true} + app, file, input, _ := runFixture(t, probe) + var output bytes.Buffer + outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--set", `/pipelines/demo/mode="permissive"`, "--set", "/pipelines/demo/max_additional_turns=0", "--format", "json", "--verbose"}, commandkit.Streams{Out: &output, Err: failingResultWriter{cause: sentinel}}) + if outcome.Status != 1 || outcome.Cause != nil || !errors.Is(outcome.Presentation.Diagnostics.WriteError, sentinel) || outcome.Presentation.Diagnostics.State != commandkit.DeliveryIncomplete { + t.Fatalf("diagnostic delivery: %#v", outcome) + } + if report := decodeCompletionReport(t, commandkit.FormatJSON, output.Bytes()); report["fallback_count"] != float64(1) { + t.Fatalf("completed report was replaced by a delivery envelope: %#v", report) + } + if !strings.Contains(outcome.Explanation.Message, filepath.Join(filepath.Dir(file), "output-must-not-exist", "run-1")) { + t.Fatalf("presentation failure lost available artifacts: %#v", outcome) + } + assertCompletedRunFiles(t, file, false) +} + +func TestNativeRunCaptureFailureAfterPublicationDoesNotReportSuccess(t *testing.T) { + probe := &runProbe{} + app, file, input, _ := runFixture(t, probe) + var once sync.Once + debug := filepath.Join(filepath.Dir(file), "debug-must-not-exist", "run-1") + probe.onGenerate = func() { + once.Do(func() { + // The external provider boundary runs after capture allocation but + // before terminalization. Make only the terminal file unusable. + if err := os.Mkdir(filepath.Join(debug, "outcome.json"), 0700); err != nil { + t.Fatal(err) + } + }) + } + var output, diagnostics bytes.Buffer + outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--format", "json"}, commandkit.Streams{Out: &output, Err: &diagnostics}) + if outcome.Status != 1 || outcome.Cause == nil || output.Len() != 0 || probe.produced != 1 { + t.Fatalf("post-publication capture failure: %#v stdout=%s stderr=%s", outcome, output.String(), diagnostics.String()) + } + assertCompletedRunFiles(t, file, false) + if !strings.Contains(diagnostics.String(), debug) || !strings.Contains(diagnostics.String(), filepath.Join(filepath.Dir(file), "output-must-not-exist", "run-1")) { + t.Fatalf("post-publication locations: %s", diagnostics.String()) + } + if _, err := os.Stat(filepath.Join(debug, "executions.json")); err != nil { + t.Fatalf("existing capture was removed: %v", err) + } +} + +func TestNativeRunFailureCategoriesPreserveCausesWithoutSuccessReports(t *testing.T) { + for _, test := range []struct { + name string + configure func(*testing.T, *runProbe, string, string) []string + status int + cause error + }{ + {"missing configuration", func(t *testing.T, _ *runProbe, _, input string) []string { + return []string{"run", "demo", "--input-file", input, "--config", filepath.Join(t.TempDir(), "missing.yml")} + }, 1, nil}, + {"invalid configuration", func(t *testing.T, _ *runProbe, file, input string) []string { + return []string{"run", "demo", "--config", file, "--input-file", input, "--set", "/version=5"} + }, 2, nil}, + {"unknown pipeline", func(_ *testing.T, _ *runProbe, file, input string) []string { + return []string{"run", "missing", "--config", file, "--input-file", input} + }, 2, nil}, + {"missing input", func(t *testing.T, _ *runProbe, file, _ string) []string { + return []string{"run", "demo", "--config", file, "--input-file", filepath.Join(t.TempDir(), "missing.txt")} + }, 1, os.ErrNotExist}, + {"publication failure", func(t *testing.T, _ *runProbe, file, input string) []string { + path := filepath.Join(t.TempDir(), "occupied") + if err := os.WriteFile(path, []byte("keep"), 0600); err != nil { + t.Fatal(err) + } + return []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--output-dir", path} + }, 1, nil}, + {"provider failure", func(_ *testing.T, probe *runProbe, file, input string) []string { + probe.providerErr = io.ErrUnexpectedEOF + return []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--llm-transient-retries=0"} + }, 1, io.ErrUnexpectedEOF}, + } { + t.Run(test.name, func(t *testing.T) { + probe := &runProbe{} + app, file, input, _ := runFixture(t, probe) + var output, diagnostics bytes.Buffer + args := append(test.configure(t, probe, file, input), "--quiet", "--format", "json") + outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: &diagnostics}) + if outcome.Status != test.status || test.cause != nil && !errors.Is(outcome.Cause, test.cause) || output.Len() != 0 || diagnostics.Len() == 0 { + t.Fatalf("failure category: %#v stdout=%s stderr=%s", outcome, output.String(), diagnostics.String()) + } + if test.name == "missing configuration" || test.name == "invalid configuration" { + var load *configkit.LoadError + if !errors.As(outcome.Cause, &load) { + t.Fatalf("lost native load cause: %#v", outcome) + } + } + }) + } } diff --git a/internal/cli/run_result_test.go b/internal/cli/run_result_test.go index dec64dd2..9b223590 100644 --- a/internal/cli/run_result_test.go +++ b/internal/cli/run_result_test.go @@ -2,229 +2,136 @@ package cli import ( "bytes" - "encoding/json" + "context" "errors" "io" "path/filepath" + "strings" "testing" + "text/template" - "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" - "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" + "go.maximumdirect.net/commandkit" ) -func TestRunResultEncodesRequiredFieldsAndCounts(t *testing.T) { - result, err := newRunResult(testResolvedPipeline(pipeline.DefaultOutputModule), testRunOutput(), "relative-output", "relative-debug") - if err != nil { - t.Fatal(err) - } - - encoded, err := encodeRunResult(result) - if err != nil { - t.Fatal(err) - } - if encoded[len(encoded)-1] != '\n' || bytes.Count(encoded, []byte{'\n'}) != 1 { - t.Fatalf("encoded result is not one newline-terminated object: %q", encoded) - } - - var decoded map[string]any - if err := json.Unmarshal(encoded, &decoded); err != nil { - t.Fatal(err) - } - if got := decoded["schema_version"]; got != runResultSchemaVersion { - t.Fatalf("schema_version = %q", got) - } - if got := decoded["run_id"]; got != "run-123" { - t.Fatalf("run_id = %q", got) - } - if got := decoded["pipeline_id"]; got != "sample" { - t.Fatalf("pipeline_id = %q", got) - } - if got := decoded["validation_status"]; got != "rejected" { - t.Fatalf("validation_status = %q", got) - } - if got := decoded["index_file"]; got != "index.json" { - t.Fatalf("index_file = %q", got) - } - if got := decoded["normalized_output_count"]; got != float64(2) { - t.Fatalf("normalized_output_count = %v", got) - } - if got := decoded["rejected_output_count"]; got != float64(1) { - t.Fatalf("rejected_output_count = %v", got) - } - if got := decoded["warning_group_count"]; got != float64(1) || decoded["warning_occurrence_count"] != float64(1) || decoded["diagnostic_group_count"] != float64(0) || decoded["diagnostic_occurrence_count"] != float64(0) || decoded["diagnostics_truncated"] != false { - t.Fatalf("diagnostic counts = %#v", decoded) - } - if got := decoded["validation_summaries"]; got != nil { - t.Fatalf("validation_summaries = %#v, want omitted when empty", got) - } - if got := decoded["output_directory"]; got != filepath.Join(mustWorkingDirectory(t), "relative-output") { - t.Fatalf("output_directory = %q", got) - } - if got := decoded["debug_directory"]; got != filepath.Join(mustWorkingDirectory(t), "relative-debug") { - t.Fatalf("debug_directory = %q", got) - } -} - -func TestRunResultRejectsInvalidRequiredValues(t *testing.T) { - tests := []struct { - name string - resolved pipeline.ResolvedPipeline - output pipeline.RunOutput - directory string - }{ - {name: "blank run ID", resolved: testResolvedPipeline(pipeline.DefaultOutputModule), output: testRunOutputWithout(func(output *pipeline.RunOutput) { output.Manifest.RunID = " " }), directory: "output"}, - {name: "blank resolved pipeline ID", resolved: pipeline.ResolvedPipeline{Output: pipeline.ModuleBinding{Module: pipeline.DefaultOutputModule}}, output: testRunOutput(), directory: "output"}, - {name: "blank manifest pipeline ID", resolved: testResolvedPipeline(pipeline.DefaultOutputModule), output: testRunOutputWithout(func(output *pipeline.RunOutput) { output.Manifest.PipelineID = "" }), directory: "output"}, - {name: "mismatched pipeline IDs", resolved: testResolvedPipeline(pipeline.DefaultOutputModule), output: testRunOutputWithout(func(output *pipeline.RunOutput) { output.Manifest.PipelineID = "other" }), directory: "output"}, - {name: "blank validation status", resolved: testResolvedPipeline(pipeline.DefaultOutputModule), output: testRunOutputWithout(func(output *pipeline.RunOutput) { output.Manifest.ValidationStatus = " " }), directory: "output"}, - {name: "blank output directory", resolved: testResolvedPipeline(pipeline.DefaultOutputModule), output: testRunOutput(), directory: " "}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if _, err := newRunResult(tt.resolved, tt.output, tt.directory, ""); err == nil { - t.Fatal("newRunResult() succeeded") +func TestRunResultUsesOneNativeValueAcrossFormats(t *testing.T) { + value := runResult{Pipeline: "demo", ArtifactCount: 3, OutputDirectory: "/artifacts/run-one", DebugDirectory: "/debug/run-one", FallbackCount: 2} + for _, format := range []commandkit.Format{commandkit.FormatText, commandkit.FormatJSON, commandkit.FormatYAML} { + var output bytes.Buffer + if err := commandkit.RenderReport(&output, commandkit.Report{Value: value, TextTemplate: runResultTemplate}, format); err != nil { + t.Fatal(err) + } + if format == commandkit.FormatText { + if !strings.Contains(output.String(), "Success with fallback: demo produced 3 artifacts in /artifacts/run-one") || !strings.Contains(output.String(), "2 operations used fallback.") || !strings.Contains(output.String(), "Debug capture: /debug/run-one") { + t.Fatalf("fallback text: %s", output.String()) } - }) + continue + } + report := decodeCompletionReport(t, format, output.Bytes()) + if report["pipeline"] != "demo" || report["output_directory"] != value.OutputDirectory || report["debug_directory"] != value.DebugDirectory || report["fallback_count"] != float64(2) || report["artifact_count"] != float64(3) || len(report) != 5 { + t.Fatalf("completion value: %#v", report) + } } } -func TestRunResultOmitsIndexFileForOtherOutputModules(t *testing.T) { - result, err := newRunResult(testResolvedPipeline("test/output"), testRunOutputWithout(func(output *pipeline.RunOutput) { - output.OutputFiles = nil - }), "output", "") +func TestNativeCompletionDiagnosticsRetainDomainClassification(t *testing.T) { + collection := contracts.DiagnosticCollection{} + for _, disposition := range []contracts.DiagnosticDisposition{contracts.DiagnosticDispositionWarning, contracts.DiagnosticDispositionAdvisory, contracts.DiagnosticDispositionObservation} { + category := contracts.DiagnosticCategoryDataQuality + if disposition == contracts.DiagnosticDispositionWarning { + category = contracts.DiagnosticCategoryFallback + } else if disposition == contracts.DiagnosticDispositionObservation { + category = contracts.DiagnosticCategoryNormalization + } + collection.Groups = append(collection.Groups, contracts.DiagnosticGroup{Disposition: disposition, Category: category, ReasonCode: string(disposition), Origin: contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract, StepID: "build", LaneID: "notes", ModuleKey: "producer"}, OccurrenceCount: 1, Samples: []contracts.DiagnosticSample{{Scope: "current source", Message: string(disposition) + " finding"}}}) + } + if err := collection.Validate(); err != nil { + t.Fatal(err) + } + var output, diagnostics bytes.Buffer + app, err := commandkit.New(commandkit.Identity{Name: "fixture"}, commandkit.NewCommand(commandkit.CommandSpec[struct{}]{Name: "report", Run: func(invocation commandkit.Invocation, _ struct{}) commandkit.Outcome { + reportRunDiagnostics(invocation.Reporter(), collection) + return reportRunResult(invocation, runResult{Pipeline: "demo", ArtifactCount: 1, OutputDirectory: "/artifacts/run-one"}) + }})) if err != nil { t.Fatal(err) } - if result.IndexFile != "" { - t.Fatalf("index_file = %q", result.IndexFile) + outcome := app.Execute(context.Background(), []string{"report", "--verbose"}, commandkit.Streams{Out: &output, Err: &diagnostics}) + if outcome.Status != 0 || strings.Contains(output.String(), "finding") || strings.Count(diagnostics.String(), "WARN:") != 1 { + t.Fatalf("diagnostic classification: %#v stdout=%s stderr=%s", outcome, output.String(), diagnostics.String()) } - encoded, err := encodeRunResult(result) - if err != nil { - t.Fatal(err) - } - var decoded map[string]any - if err := json.Unmarshal(encoded, &decoded); err != nil { - t.Fatal(err) - } - if _, ok := decoded["index_file"]; ok { - t.Fatalf("encoded non-JSON result contains index_file: %s", encoded) - } - if _, ok := decoded["debug_directory"]; ok { - t.Fatalf("encoded result without debug capture contains debug_directory: %s", encoded) + for _, group := range collection.Groups { + if !strings.Contains(diagnostics.String(), string(group.Disposition)+" finding") { + t.Fatalf("missing diagnostic: %s", diagnostics.String()) + } } } -func TestRunResultProjectsOwnedValidationSummaries(t *testing.T) { - output := testRunOutput() - output.Manifest.ValidationSummaries = []artifacts.ValidationSummary{{Status: "incomplete", IncompleteValidators: []string{"validator"}, ProducerAttemptCount: 1, TerminalAction: "warn_continue"}} - result, err := newRunResult(testResolvedPipeline(pipeline.DefaultOutputModule), output, "output", "") - if err != nil { - t.Fatal(err) - } - output.Manifest.ValidationSummaries[0].IncompleteValidators[0] = "caller mutation" - if got := result.ValidationSummaries[0].IncompleteValidators; len(got) != 1 || got[0] != "validator" { - t.Fatalf("result validation summaries = %#v", result.ValidationSummaries) - } - encoded, err := encodeRunResult(result) - if err != nil { - t.Fatal(err) - } - if !bytes.Contains(encoded, []byte(`"validation_summaries":[{"status":"incomplete","incomplete_validators":["validator"],"producer_attempt_count":1,"terminal_action":"warn_continue"}]`)) { - t.Fatalf("encoded result = %s", encoded) +func TestCompletedRunRenderingFailureRetainsPublicationEvenWhenQuiet(t *testing.T) { + previous := runResultTemplate + t.Cleanup(func() { runResultTemplate = previous }) + // Simulate a broken application-owned text template, not a new renderer seam. + runResultTemplate = template.Must(commandkit.ParseTextTemplate("invalid", "{{.MissingField}}", nil)) + for _, quiet := range []bool{false, true} { + probe := &runProbe{} + app, file, input, _ := runFixture(t, probe) + args := []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug"} + if quiet { + args = append(args, "--quiet") + } + var output, diagnostics bytes.Buffer + outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: &diagnostics}) + if outcome.Status != 1 || outcome.Cause != nil || outcome.Presentation.Output.RenderError == nil || outcome.Presentation.Output.State != commandkit.DeliveryIncomplete || probe.produced != 1 { + t.Fatalf("render failure: %#v %s", outcome, diagnostics.String()) + } + assertCompletedRunFiles(t, file, true) + artifacts := filepath.Join(filepath.Dir(file), "output-must-not-exist", "run-1") + if !strings.Contains(diagnostics.String(), artifacts) || (quiet && output.Len() != 0) { + t.Fatalf("render failure locations/quiet: %s", diagnostics.String()) + } } } -func TestRunResultRequiresOneProductionIndexFile(t *testing.T) { - tests := []struct { - name string - files []contracts.OutputFile - }{ - {name: "missing", files: nil}, - {name: "duplicate", files: []contracts.OutputFile{{Name: "index.json"}, {Name: "index.json"}}}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - output := testRunOutput() - output.OutputFiles = tt.files - if _, err := newRunResult(testResolvedPipeline(pipeline.DefaultOutputModule), output, "output", ""); err == nil { - t.Fatal("newRunResult() succeeded") - } - }) - } +type failingResultWriter struct { + cause error } -func TestWriteRunResultCompletesAndReportsWriterFailure(t *testing.T) { - content := []byte("result\n") - var target bytes.Buffer - if err := writeRunResult(partialResultWriter{writer: &target, limit: 2}, content); err != nil { - t.Fatal(err) - } - if got := target.String(); got != string(content) { - t.Fatalf("written result = %q", got) - } - - writerErr := errors.New("result writer failed") - if err := writeRunResult(failingResultWriter{err: writerErr}, content); !errors.Is(err, writerErr) { - t.Fatalf("writeRunResult() error = %v", err) - } - if err := writeRunResult(zeroResultWriter{}, content); !errors.Is(err, io.ErrShortWrite) { - t.Fatalf("zero-progress error = %v", err) - } -} - -func testResolvedPipeline(outputModule string) pipeline.ResolvedPipeline { - return pipeline.ResolvedPipeline{ID: "sample", Output: pipeline.ModuleBinding{Module: outputModule}} -} - -func testRunOutput() pipeline.RunOutput { - return pipeline.RunOutput{ - Manifest: artifacts.RunManifest{RunID: "run-123", PipelineID: "sample", ValidationStatus: "rejected"}, - NormalizeOutputs: []contracts.SerializedOutput{{}, {}}, - Rejected: []contracts.RejectedOutput{{}}, - Diagnostics: contracts.DiagnosticCollection{Groups: []contracts.DiagnosticGroup{{ - Disposition: contracts.DiagnosticDispositionWarning, - Category: contracts.DiagnosticCategoryFallback, - ReasonCode: "fallback", - Origin: contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageNormalize, StepID: "step", LaneID: "lane", ModuleKey: "module"}, - OccurrenceCount: 1, - Samples: []contracts.DiagnosticSample{{Scope: "scope", Message: "message"}}, - }}}, - OutputFiles: []contracts.OutputFile{{Name: "index.json"}}, - } -} - -func testRunOutputWithout(change func(*pipeline.RunOutput)) pipeline.RunOutput { - output := testRunOutput() - change(&output) - return output -} - -func mustWorkingDirectory(t *testing.T) string { - t.Helper() - workingDirectory, err := filepath.Abs(".") - if err != nil { - t.Fatal(err) - } - return workingDirectory -} +func (writer failingResultWriter) Write([]byte) (int, error) { return 0, writer.cause } type partialResultWriter struct { - writer io.Writer - limit int + content bytes.Buffer + cause error } -func (w partialResultWriter) Write(content []byte) (int, error) { - if len(content) > w.limit { - content = content[:w.limit] +func (writer *partialResultWriter) Write(content []byte) (int, error) { + if len(content) > 8 { + content = content[:8] } - return w.writer.Write(content) + n, _ := writer.content.Write(content) + return n, writer.cause } -type failingResultWriter struct{ err error } +var _ io.Writer = (*partialResultWriter)(nil) -func (w failingResultWriter) Write([]byte) (int, error) { return 0, w.err } - -type zeroResultWriter struct{} - -func (zeroResultWriter) Write([]byte) (int, error) { return 0, nil } +func TestPipelinePreflightClassifiesOnlyIdentifiableInvalidSelections(t *testing.T) { + // Real native missing-resource errors retain their category separately from + // private lookup causes. A joined operational branch must still win. + probe := &runProbe{} + app, file, input, _ := runFixture(t, probe) + var diagnostics bytes.Buffer + missing := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--llm-profile", "absent"}, commandkit.Streams{Out: io.Discard, Err: &diagnostics}).Cause + for _, test := range []struct { + cause error + status int + }{ + {missing, 2}, + {errors.Join(missing, context.Canceled), 1}, + {errors.Join(missing, errors.New("unavailable inspection")), 1}, + {errors.New("unavailable inspection"), 1}, + } { + outcome := pipelineResourceFailure(test.cause) + if outcome.Status != test.status || !errors.Is(outcome.Cause, test.cause) { + t.Fatalf("preflight classification: %#v", outcome) + } + } +} diff --git a/internal/preflight/preflight.go b/internal/preflight/preflight.go index 64464ce0..3a2379f3 100644 --- a/internal/preflight/preflight.go +++ b/internal/preflight/preflight.go @@ -12,6 +12,21 @@ import ( "go.maximumdirect.net/promptkit" ) +// SelectionError identifies an absent configured native resource. Callers can +// distinguish this invalid selection from unavailable inspection or cancellation +// without inspecting diagnostic text or the kit's private lookup causes. +type SelectionError struct{ Cause error } + +func (err *SelectionError) Error() string { return err.Cause.Error() } +func (err *SelectionError) Unwrap() error { return err.Cause } + +func resourceFinding(err error) error { + if !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) && (errors.Is(err, promptkit.ErrPromptNotFound) || errors.Is(err, promptkit.ErrProfileNotFound)) { + return &SelectionError{Cause: err} + } + return err +} + // RequiresEngine reports selected model work, including nested validators. // Deterministic pipelines need no engine merely to validate their composition. func RequiresEngine(resolved pipeline.ResolvedPipeline) bool { @@ -69,7 +84,7 @@ func Validate(ctx context.Context, resolved pipeline.ResolvedPipeline, catalog p } if profile != "" { if _, err := engine.InspectProfile(ctx, profile); err != nil { - findings = append(findings, fmt.Errorf("%s profile %q: %w", scope, profile, err)) + findings = append(findings, resourceFinding(fmt.Errorf("%s profile %q: %w", scope, profile, err))) } } for _, id := range prompts { @@ -78,7 +93,7 @@ func Validate(ctx context.Context, resolved pipeline.ResolvedPipeline, catalog p } prompt, err := engine.InspectPrompt(ctx, id) if err != nil { - findings = append(findings, fmt.Errorf("%s prompt %q: %w", scope, id, err)) + findings = append(findings, resourceFinding(fmt.Errorf("%s prompt %q: %w", scope, id, err))) continue } if profile != "" { @@ -90,7 +105,7 @@ func Validate(ctx context.Context, resolved pipeline.ResolvedPipeline, catalog p continue } if _, err := engine.InspectProfile(ctx, *prompt.DefaultProfileID); err != nil { - findings = append(findings, fmt.Errorf("%s prompt %q default profile %q: %w", scope, id, *prompt.DefaultProfileID, err)) + findings = append(findings, resourceFinding(fmt.Errorf("%s prompt %q default profile %q: %w", scope, id, *prompt.DefaultProfileID, err))) } } }