Report completed runs through native presentation and preserve failure outcomes
This commit is contained in:
@@ -2,21 +2,34 @@ package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/commandkit"
|
||||
)
|
||||
|
||||
func TestExecutableRunRoutesDeclareTheSameRequiredInputs(t *testing.T) {
|
||||
func buildNativeExecutable(t *testing.T) string {
|
||||
t.Helper()
|
||||
binary := filepath.Join(t.TempDir(), "notarius")
|
||||
build := exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"), "build", "-o", binary, "../../cmd/notarius")
|
||||
build.Env = append(os.Environ(), "GOWORK=off")
|
||||
if output, err := build.CombinedOutput(); err != nil {
|
||||
t.Fatalf("build executable: %v\n%s", err, output)
|
||||
}
|
||||
return binary
|
||||
}
|
||||
|
||||
func TestExecutableRunRoutesDeclareTheSameRequiredInputs(t *testing.T) {
|
||||
binary := buildNativeExecutable(t)
|
||||
for _, route := range [][]string{{"run"}, {"pipeline", "run"}} {
|
||||
help := exec.Command(binary, append(append([]string(nil), route...), "--help")...)
|
||||
output, err := help.CombinedOutput()
|
||||
@@ -31,3 +44,68 @@ func TestExecutableRunRoutesDeclareTheSameRequiredInputs(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutableReportsDomainArtifactsAndQualifiedFallbacks(t *testing.T) {
|
||||
binary := buildNativeExecutable(t)
|
||||
for _, fallback := range []bool{false, true} {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
if fallback {
|
||||
writer.WriteHeader(http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
writer.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(writer).Encode(map[string]any{"id": "local-fake", "object": "chat.completion", "choices": []any{map[string]any{"index": 0, "message": map[string]any{"role": "assistant", "content": `{"spell_casts":[]}`}, "finish_reason": "stop"}}})
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
for _, format := range []commandkit.Format{commandkit.FormatText, commandkit.FormatJSON, commandkit.FormatYAML} {
|
||||
t.Run(fmt.Sprintf("%s/fallback=%t", format, fallback), func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
configuration := fmt.Sprintf(`version: 6
|
||||
output: {directory: output}
|
||||
pipelines:
|
||||
demo:
|
||||
input: {module: seriatim}
|
||||
chunk: {module: generic}
|
||||
llm_profile: local-fake
|
||||
mode: permissive
|
||||
max_additional_turns: 0
|
||||
artifacts:
|
||||
spells:
|
||||
extract: {module: dnd/spells}
|
||||
promptkit:
|
||||
profiles:
|
||||
- id: local-fake
|
||||
endpoint: %q
|
||||
model: fake-model
|
||||
api_key_required: false
|
||||
`, server.URL+"/v1")
|
||||
file := filepath.Join(root, "config.yml")
|
||||
input := filepath.Join(root, "input.json")
|
||||
for path, content := range map[string]string{file: configuration, input: `{"metadata":{"id":"report-source"},"segments":[{"id":1,"start":0,"end":1,"speaker":"GM","text":"The party rests."}]}`} {
|
||||
if err := os.WriteFile(path, []byte(content), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
command := exec.Command(binary, "run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--llm-transient-retries=0", "--format", string(format), "--verbose")
|
||||
var output, diagnostics bytes.Buffer
|
||||
command.Stdout, command.Stderr = &output, &diagnostics
|
||||
if err := command.Run(); err != nil {
|
||||
t.Fatalf("subprocess: %v stdout=%s stderr=%s", err, output.String(), diagnostics.String())
|
||||
}
|
||||
if format == commandkit.FormatText {
|
||||
if !strings.Contains(output.String(), "produced 1 artifacts") || fallback != strings.Contains(output.String(), "Success with fallback:") {
|
||||
t.Fatalf("completion text: %s", output.String())
|
||||
}
|
||||
} else {
|
||||
report := decodeCompletionReport(t, format, output.Bytes())
|
||||
if report["artifact_count"] != float64(1) || report["pipeline"] != "demo" || fallback && report["fallback_count"] != float64(1) || !fallback && report["fallback_count"] != nil {
|
||||
t.Fatalf("completion report: %#v", report)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(report["output_directory"].(string), "lanes", "spells.json")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -107,7 +107,7 @@ func pipelineCommands(newState func() *nativeLoaders, catalog func() (pipeline.M
|
||||
}
|
||||
}
|
||||
if err := preflight.Validate(invocation.Context, effective.ResolvedPipeline, modules, engine); err != nil {
|
||||
return pipelineInvalidConfiguration(err)
|
||||
return pipelineResourceFailure(err)
|
||||
}
|
||||
return reportPipelineInspection(invocation, commandkit.Report{Value: struct {
|
||||
Pipeline string `json:"pipeline"`
|
||||
@@ -124,16 +124,56 @@ var pipelineValidationTemplate = template.Must(commandkit.ParseTextTemplate("pip
|
||||
|
||||
func reportPipelineInspection(invocation commandkit.Invocation, report commandkit.Report) commandkit.Outcome {
|
||||
if err := invocation.Reporter().Report(report); err != nil {
|
||||
return commandkit.Failure("Could not report pipeline inspection.").WithCause(err)
|
||||
return commandkit.Failure("Could not report pipeline inspection.")
|
||||
}
|
||||
return commandkit.Success()
|
||||
}
|
||||
|
||||
func pipelineConfigurationFailure(err error) commandkit.Outcome {
|
||||
outcome := commandkit.Failure(err.Error()).WithCause(err)
|
||||
if ckconfig.ClassifyConfigurationError(err) == commandkit.OutcomeUsage {
|
||||
outcome = commandkit.Usage(err.Error()).WithCause(err)
|
||||
}
|
||||
var load *configkit.LoadError
|
||||
if errors.As(err, &load) {
|
||||
_, explanation := ckconfig.AdaptDiagnostics(err.Error()+"\n", load.Sources(), load.Diagnostics())
|
||||
outcome = outcome.WithExplanation(explanation.Message, explanation.Details...)
|
||||
}
|
||||
return outcome
|
||||
}
|
||||
|
||||
func pipelineResourceFailure(err error) commandkit.Outcome {
|
||||
if preflightInputError(err) {
|
||||
return commandkit.Usage(err.Error()).WithCause(err)
|
||||
}
|
||||
return commandkit.Failure(err.Error()).WithCause(err)
|
||||
return pipelineConfigurationFailure(err)
|
||||
}
|
||||
|
||||
// Only identifiable missing native selections are invalid preflight input.
|
||||
// Unknown failures and any operational branch of a joined error remain failures.
|
||||
func preflightInputError(err error) bool {
|
||||
if err == nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return false
|
||||
}
|
||||
if _, ok := err.(*preflight.SelectionError); ok {
|
||||
return true
|
||||
}
|
||||
if joined, ok := err.(interface{ Unwrap() []error }); ok {
|
||||
found := false
|
||||
for _, child := range joined.Unwrap() {
|
||||
if child != nil {
|
||||
found = true
|
||||
if !preflightInputError(child) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return found
|
||||
}
|
||||
if wrapped, ok := err.(interface{ Unwrap() error }); ok {
|
||||
return preflightInputError(wrapped.Unwrap())
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func pipelineInvalidConfiguration(err error) commandkit.Outcome {
|
||||
|
||||
@@ -262,7 +262,7 @@ func TestPipelineInspectionReportFailureRemainsOperational(t *testing.T) {
|
||||
primary := errors.New("inspection stdout delivery failed")
|
||||
var diagnostics bytes.Buffer
|
||||
outcome := app.Execute(context.Background(), []string{"pipeline", "list", "--config", file}, commandkit.Streams{Out: inspectionFailWriter{primary}, Err: &diagnostics})
|
||||
if outcome.Status != 1 || !errors.Is(outcome.Cause, primary) {
|
||||
if outcome.Status != 1 || outcome.Cause != nil || !errors.Is(outcome.Presentation.Output.WriteError, primary) {
|
||||
t.Fatalf("delivery failure classified as invalid configuration: %#v %s", outcome, diagnostics.String())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
frameworkdebug "gitea.maximumdirect.net/eric/notarius/internal/framework/debug"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/preflight"
|
||||
"go.maximumdirect.net/ckconfig"
|
||||
"go.maximumdirect.net/commandkit"
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
@@ -31,6 +32,14 @@ func pipelineRunCommand(version string, newState func() *nativeLoaders, modules
|
||||
if err != nil {
|
||||
return pipelineConfigurationFailure(err)
|
||||
}
|
||||
diagnostics, _ := ckconfig.AdaptDiagnostics("Configuration\n", result.Sources(), result.Diagnostics())
|
||||
for _, diagnostic := range diagnostics {
|
||||
if diagnostic.Severity == commandkit.SeverityWarning {
|
||||
_ = invocation.Reporter().AtEnd().Warn("", diagnostic.Write)
|
||||
} else {
|
||||
_ = invocation.Reporter().AtEnd().Info("", diagnostic.Write)
|
||||
}
|
||||
}
|
||||
loaded, err := config.FromResult(result)
|
||||
if err != nil {
|
||||
return pipelineInvalidConfiguration(err)
|
||||
@@ -53,20 +62,20 @@ func pipelineRunCommand(version string, newState func() *nativeLoaders, modules
|
||||
}
|
||||
finish := func(outcome commandkit.Outcome) commandkit.Outcome {
|
||||
if err := capture.Finalize(captureOutcome(), outcome.Cause); err != nil {
|
||||
message := err.Error()
|
||||
if published {
|
||||
message += fmt.Sprintf("\nArtifacts: %s", outputDirectory)
|
||||
}
|
||||
if capture.Path() != "" {
|
||||
message += fmt.Sprintf("\nDebug capture: %s", capture.Path())
|
||||
}
|
||||
if outcome.Status == 0 {
|
||||
outcome = commandkit.Failure(message)
|
||||
outcome = commandkit.Failure(err.Error())
|
||||
} else {
|
||||
outcome = outcome.WithExplanation(message)
|
||||
outcome = outcome.WithExplanation(err.Error(), outcome.Explanation.Details...)
|
||||
}
|
||||
outcome = outcome.WithCause(err)
|
||||
}
|
||||
if outcome.Status != 0 {
|
||||
availableOutput := ""
|
||||
if published {
|
||||
availableOutput = outputDirectory
|
||||
}
|
||||
outcome = outcome.WithExplanation(outcome.Explanation.Message+runLocations(availableOutput, capture.Path()), outcome.Explanation.Details...)
|
||||
}
|
||||
return outcome
|
||||
}
|
||||
capture, err = startRunDebug(input.Debug, cfg.Debug.Directory, runID, startedAt)
|
||||
@@ -107,13 +116,13 @@ func pipelineRunCommand(version string, newState func() *nativeLoaders, modules
|
||||
}
|
||||
}
|
||||
if err := preflight.Validate(invocation.Context, effective.ResolvedPipeline, catalog, engine); err != nil {
|
||||
return finish(pipelineInvalidConfiguration(err))
|
||||
return finish(pipelineResourceFailure(err))
|
||||
}
|
||||
workingDirectory, err := os.Getwd()
|
||||
if err != nil {
|
||||
return finish(commandkit.Failure(err.Error()).WithCause(err))
|
||||
}
|
||||
materialized, diagnostics, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{ConfigPath: loaded.MainSourcePath(), WorkingDir: workingDirectory})
|
||||
materialized, referenceDiagnostics, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{ConfigPath: loaded.MainSourcePath(), WorkingDir: workingDirectory})
|
||||
if err != nil {
|
||||
return finish(commandkit.Failure(err.Error()).WithCause(err))
|
||||
}
|
||||
@@ -138,7 +147,9 @@ func pipelineRunCommand(version string, newState func() *nativeLoaders, modules
|
||||
recorder = newRunEvidenceRecorder(capture, result)
|
||||
}
|
||||
group := &promptkit.ExecutionGroup{}
|
||||
output, err = pipeline.New().Run(invocation.Context, pipeline.RunInput{Prepared: prepared, Path: input.InputFile.Value, RawInput: raw, SessionID: sessionID, RunID: runID, StartedAt: startedAt, Diagnostics: diagnostics, Cache: cache, Debug: recorder, Workers: cfg.Concurrency.Workers, Executions: group, Execution: contracts.ExecutionOptions{Execution: input.Overrides.Execution.Execution, TransientRetries: input.Overrides.Execution.TransientRetries}})
|
||||
_ = invocation.Reporter().Progress(fmt.Sprintf("Running pipeline %q.", effective.ResolvedPipeline.ID))
|
||||
output, err = pipeline.New().Run(invocation.Context, pipeline.RunInput{Prepared: prepared, Path: input.InputFile.Value, RawInput: raw, SessionID: sessionID, RunID: runID, StartedAt: startedAt, Diagnostics: referenceDiagnostics, Cache: cache, Debug: recorder, Workers: cfg.Concurrency.Workers, Executions: group, Execution: contracts.ExecutionOptions{Execution: input.Overrides.Execution.Execution, TransientRetries: input.Overrides.Execution.TransientRetries}})
|
||||
reportRunDiagnostics(invocation.Reporter(), output.Diagnostics)
|
||||
if err != nil {
|
||||
return finish(commandkit.Failure(err.Error()).WithCause(err))
|
||||
}
|
||||
@@ -146,7 +157,11 @@ func pipelineRunCommand(version string, newState func() *nativeLoaders, modules
|
||||
return finish(commandkit.Failure(err.Error()).WithCause(err))
|
||||
}
|
||||
published = true
|
||||
return finish(commandkit.Success())
|
||||
outcome := finish(commandkit.Success())
|
||||
if outcome.Status != 0 {
|
||||
return outcome
|
||||
}
|
||||
return reportRunResult(invocation, runResult{Pipeline: effective.ResolvedPipeline.ID, ArtifactCount: output.ArtifactCount(), OutputDirectory: outputDirectory, DebugDirectory: capture.Path(), FallbackCount: output.FallbackCount})
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
@@ -31,12 +31,20 @@ type runProbe struct {
|
||||
parsed int
|
||||
produced int
|
||||
correctFirst bool
|
||||
providerErr error
|
||||
onGenerate func()
|
||||
}
|
||||
|
||||
func (probe *runProbe) Generate(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
probe.mu.Lock()
|
||||
defer probe.mu.Unlock()
|
||||
probe.requests = append(probe.requests, request)
|
||||
if probe.onGenerate != nil {
|
||||
probe.onGenerate()
|
||||
}
|
||||
if probe.providerErr != nil {
|
||||
return nil, probe.providerErr
|
||||
}
|
||||
value := "accepted"
|
||||
if request.Prompt.Messages[len(request.Prompt.Messages)-1].Content != "Assess." {
|
||||
probe.produced++
|
||||
@@ -98,7 +106,9 @@ func (producer *runProducer) Extract(ctx context.Context, request contracts.Type
|
||||
return &value, nil, err
|
||||
}, func(context.Context, []string) (promptkit.RunRequest, error) {
|
||||
return promptkit.RunRequest{PromptID: "inspect-correction", ProfileID: request.LLMProfile, SessionID: request.SessionID, OnPrepared: producer.probe.onPrepared}, nil
|
||||
}, nil)
|
||||
}, func(context.Context, *inspectionArtifact) (inspectionArtifact, error) {
|
||||
return inspectionArtifact{Value: "fallback"}, nil
|
||||
})
|
||||
return contracts.TypedExtractionResult[inspectionArtifact]{Value: value, Outcome: outcome}, err
|
||||
}
|
||||
func (producer *runProducer) Merge(_ context.Context, request contracts.TypedMergeRequest[inspectionArtifact]) (contracts.TypedMergeResult[inspectionArtifact], error) {
|
||||
|
||||
@@ -1,142 +1,78 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"text/template"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"go.maximumdirect.net/commandkit"
|
||||
)
|
||||
|
||||
const runResultSchemaVersion = "notarius.run-result.v2"
|
||||
|
||||
// runResult is the small completion value. Domain files, not a saved report or
|
||||
// execution ledger, are the durable deliverables.
|
||||
type runResult struct {
|
||||
SchemaVersion string `json:"schema_version"`
|
||||
RunID string `json:"run_id"`
|
||||
PipelineID string `json:"pipeline_id"`
|
||||
OutputDirectory string `json:"output_directory"`
|
||||
IndexFile string `json:"index_file,omitempty"`
|
||||
NormalizedOutputCount int `json:"normalized_output_count"`
|
||||
RejectedOutputCount int `json:"rejected_output_count"`
|
||||
WarningGroupCount int `json:"warning_group_count"`
|
||||
WarningOccurrenceCount int `json:"warning_occurrence_count"`
|
||||
DiagnosticGroupCount int `json:"diagnostic_group_count"`
|
||||
DiagnosticOccurrenceCount int `json:"diagnostic_occurrence_count"`
|
||||
DiagnosticsTruncated bool `json:"diagnostics_truncated"`
|
||||
ValidationStatus string `json:"validation_status"`
|
||||
ValidationSummaries []artifacts.ValidationSummary `json:"validation_summaries,omitempty"`
|
||||
DebugDirectory string `json:"debug_directory,omitempty"`
|
||||
Pipeline string `json:"pipeline"`
|
||||
ArtifactCount int `json:"artifact_count"`
|
||||
OutputDirectory string `json:"output_directory"`
|
||||
DebugDirectory string `json:"debug_directory,omitempty"`
|
||||
FallbackCount int `json:"fallback_count,omitempty"`
|
||||
}
|
||||
|
||||
func newRunResult(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput, outputDirectory, debugDirectory string) (runResult, error) {
|
||||
diagnosticProjection, err := contracts.ProjectDiagnosticCollection(output.Diagnostics)
|
||||
if err != nil {
|
||||
return runResult{}, fmt.Errorf("summarize run diagnostics: %w", err)
|
||||
var runResultTemplate = template.Must(commandkit.ParseTextTemplate("run-completion", `{{if .FallbackCount}}Success with fallback: {{else}}Success: {{end}}{{.Pipeline}} produced {{.ArtifactCount}} artifacts in {{.OutputDirectory}}
|
||||
{{if .FallbackCount}}{{.FallbackCount}} operations used fallback.
|
||||
{{end}}{{if .DebugDirectory}}Debug capture: {{.DebugDirectory}}
|
||||
{{end}}`, nil))
|
||||
|
||||
func reportRunResult(invocation commandkit.Invocation, result runResult) commandkit.Outcome {
|
||||
message := "Could not present the completed run." + runLocations(result.OutputDirectory, result.DebugDirectory)
|
||||
if err := invocation.Reporter().Report(commandkit.Report{Value: result, TextTemplate: runResultTemplate}); err != nil {
|
||||
// The reporter retains render/write failures in Presentation. They are
|
||||
// not a replacement operation Cause, and capture is already finalized.
|
||||
return commandkit.Failure(message)
|
||||
}
|
||||
return newRunResultWithDiagnostics(resolved, output, outputDirectory, debugDirectory, diagnosticProjection)
|
||||
// Deferred diagnostic delivery can still fail during native finalization.
|
||||
// Supply useful locations without presenting a failure on successful runs.
|
||||
return commandkit.Success().WithExplanation(message)
|
||||
}
|
||||
|
||||
func newRunResultWithDiagnostics(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput, outputDirectory, debugDirectory string, diagnosticProjection contracts.DiagnosticProjection) (runResult, error) {
|
||||
if strings.TrimSpace(output.Manifest.RunID) == "" {
|
||||
return runResult{}, fmt.Errorf("run result requires a run ID")
|
||||
func runLocations(outputDirectory, debugDirectory string) string {
|
||||
var locations string
|
||||
if outputDirectory != "" {
|
||||
locations += "\nArtifacts: " + outputDirectory
|
||||
}
|
||||
if strings.TrimSpace(resolved.ID) == "" {
|
||||
return runResult{}, fmt.Errorf("run result requires a resolved pipeline ID")
|
||||
}
|
||||
if strings.TrimSpace(output.Manifest.PipelineID) == "" {
|
||||
return runResult{}, fmt.Errorf("run result requires a manifest pipeline ID")
|
||||
}
|
||||
if output.Manifest.PipelineID != resolved.ID {
|
||||
return runResult{}, fmt.Errorf("run result pipeline ID does not match resolved pipeline")
|
||||
}
|
||||
if strings.TrimSpace(output.Manifest.ValidationStatus) == "" {
|
||||
return runResult{}, fmt.Errorf("run result requires a validation status")
|
||||
}
|
||||
if strings.TrimSpace(outputDirectory) == "" {
|
||||
return runResult{}, fmt.Errorf("run result requires an output directory")
|
||||
if debugDirectory != "" {
|
||||
locations += "\nDebug capture: " + debugDirectory
|
||||
}
|
||||
return locations
|
||||
}
|
||||
|
||||
absOutputDirectory, err := filepath.Abs(outputDirectory)
|
||||
if err != nil {
|
||||
return runResult{}, fmt.Errorf("make output directory absolute: %w", err)
|
||||
}
|
||||
|
||||
result := runResult{
|
||||
SchemaVersion: runResultSchemaVersion,
|
||||
RunID: output.Manifest.RunID,
|
||||
PipelineID: output.Manifest.PipelineID,
|
||||
OutputDirectory: absOutputDirectory,
|
||||
NormalizedOutputCount: len(output.NormalizeOutputs),
|
||||
RejectedOutputCount: len(output.Rejected),
|
||||
WarningGroupCount: len(diagnosticProjection.Warnings),
|
||||
WarningOccurrenceCount: diagnosticProjection.WarningOccurrenceCount,
|
||||
DiagnosticGroupCount: len(diagnosticProjection.Diagnostics),
|
||||
DiagnosticOccurrenceCount: diagnosticProjection.DiagnosticOccurrenceCount,
|
||||
DiagnosticsTruncated: output.Diagnostics.Truncated,
|
||||
ValidationStatus: output.Manifest.ValidationStatus,
|
||||
ValidationSummaries: cloneValidationSummaries(output.Manifest.ValidationSummaries),
|
||||
}
|
||||
|
||||
if strings.TrimSpace(debugDirectory) != "" {
|
||||
absDebugDirectory, err := filepath.Abs(debugDirectory)
|
||||
if err != nil {
|
||||
return runResult{}, fmt.Errorf("make debug directory absolute: %w", err)
|
||||
}
|
||||
result.DebugDirectory = absDebugDirectory
|
||||
}
|
||||
|
||||
if resolved.Output.Module == pipeline.DefaultOutputModule {
|
||||
indexCount := 0
|
||||
for _, file := range output.OutputFiles {
|
||||
if file.Name == "index.json" {
|
||||
indexCount++
|
||||
func reportRunDiagnostics(reporter *commandkit.Reporter, collection contracts.DiagnosticCollection) {
|
||||
for _, group := range collection.Groups {
|
||||
origin := group.Origin
|
||||
scope := []string{string(origin.Stage)}
|
||||
for _, value := range []string{origin.StepID, origin.LaneID, origin.ModuleKey, origin.ValidatorKey} {
|
||||
if value != "" {
|
||||
scope = append(scope, value)
|
||||
}
|
||||
}
|
||||
if indexCount != 1 {
|
||||
return runResult{}, fmt.Errorf("production JSON output must contain exactly one index.json file")
|
||||
heading := strings.Join(scope, " / ") + ": " + group.ReasonCode
|
||||
detail := func(writer io.Writer) error {
|
||||
for _, sample := range group.Samples {
|
||||
if _, err := fmt.Fprintf(writer, "%s: %s\n", sample.Scope, sample.Message); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
result.IndexFile = "index.json"
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func cloneValidationSummaries(summaries []artifacts.ValidationSummary) []artifacts.ValidationSummary {
|
||||
if len(summaries) == 0 {
|
||||
return nil
|
||||
}
|
||||
cloned := make([]artifacts.ValidationSummary, len(summaries))
|
||||
for index, summary := range summaries {
|
||||
cloned[index] = artifacts.CloneValidationSummary(summary)
|
||||
}
|
||||
return cloned
|
||||
}
|
||||
|
||||
func encodeRunResult(result runResult) ([]byte, error) {
|
||||
encoded, err := json.Marshal(result)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("encode run result: %w", err)
|
||||
}
|
||||
return append(encoded, '\n'), nil
|
||||
}
|
||||
|
||||
func writeRunResult(writer io.Writer, content []byte) error {
|
||||
for len(content) > 0 {
|
||||
written, err := writer.Write(content)
|
||||
if written < 0 || written > len(content) {
|
||||
return io.ErrShortWrite
|
||||
}
|
||||
content = content[written:]
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if written == 0 {
|
||||
return io.ErrShortWrite
|
||||
if group.Disposition == contracts.DiagnosticDispositionWarning {
|
||||
_ = reporter.AtEnd().Warn(heading, detail)
|
||||
} else {
|
||||
_ = reporter.AtEnd().Info(heading, detail)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
if collection.Truncated {
|
||||
_ = reporter.AtEnd().Info("Additional pipeline diagnostics were omitted.")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,198 +2,318 @@ package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
alwaysreject "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/validate/always_reject"
|
||||
"go.maximumdirect.net/commandkit"
|
||||
"go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
func TestMaintainedMinimalInvocationEmitsRunResult(t *testing.T) {
|
||||
outputRoot := filepath.Join(t.TempDir(), "output")
|
||||
var stdout, stderr strings.Builder
|
||||
code := RunWithOptions([]string{
|
||||
"run", "dnd-session",
|
||||
"--config", repositoryPath("examples", "dnd-minimal.config.yml"),
|
||||
"--input", repositoryPath("examples", "seriatim-minimal-transcript.json"),
|
||||
"--only", "spells", "--chunk_cache", "bypass", "--output-dir", outputRoot, "--json",
|
||||
}, &stdout, &stderr, productionRunOptions(t, &productionFakeLLMClient{}))
|
||||
if code != 0 || stderr.Len() != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
|
||||
receipt := decodeRunResultDocument(t, stdout.String())
|
||||
if got := receipt["schema_version"]; got != "notarius.run-result.v2" {
|
||||
t.Fatalf("schema_version = %q", got)
|
||||
}
|
||||
if got := receipt["run_id"]; got != productionRunID {
|
||||
t.Fatalf("run_id = %q", got)
|
||||
}
|
||||
if got := receipt["pipeline_id"]; got != "dnd-session" {
|
||||
t.Fatalf("pipeline_id = %q", got)
|
||||
}
|
||||
if got := receipt["index_file"]; got != "index.json" {
|
||||
t.Fatalf("index_file = %q", got)
|
||||
}
|
||||
if got := receipt["normalized_output_count"]; got != float64(1) {
|
||||
t.Fatalf("normalized_output_count = %v", got)
|
||||
}
|
||||
if got := receipt["rejected_output_count"]; got != float64(0) {
|
||||
t.Fatalf("rejected_output_count = %v", got)
|
||||
}
|
||||
if got := receipt["warning_group_count"]; got != float64(0) || receipt["warning_occurrence_count"] != float64(0) || receipt["diagnostic_group_count"] != float64(0) || receipt["diagnostic_occurrence_count"] != float64(0) || receipt["diagnostics_truncated"] != false {
|
||||
t.Fatalf("diagnostic counts = %#v", receipt)
|
||||
}
|
||||
if got := receipt["validation_status"]; got != "approved" {
|
||||
t.Fatalf("validation_status = %q", got)
|
||||
}
|
||||
|
||||
outputDirectory, ok := receipt["output_directory"].(string)
|
||||
if !ok || !filepath.IsAbs(outputDirectory) || outputDirectory != filepath.Join(outputRoot, productionRunID) {
|
||||
t.Fatalf("output_directory = %q", receipt["output_directory"])
|
||||
}
|
||||
indexFile := receipt["index_file"].(string)
|
||||
assertFile(t, filepath.Join(outputDirectory, indexFile))
|
||||
}
|
||||
|
||||
func TestRunResultReportsWarningsAndDebugBundle(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
harness := newStateTestHarness()
|
||||
harness.chunkDiagnostics = []contracts.ProducerDiagnostic{stateTestDiagnostic("chunk", "contract-warning", "warning retained")}
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{
|
||||
"run", "sample", "--config", roots.config, "--input", roots.input,
|
||||
"--chunk_cache", "bypass", "--debug", "--json",
|
||||
}, &stdout, &stderr, harness.options())
|
||||
if code != 0 || !strings.Contains(stderr.String(), "1 warning group(s), 1 occurrence(s)") || strings.Contains(stderr.String(), "warnings.json") {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
|
||||
receipt := decodeRunResultDocument(t, stdout.String())
|
||||
if got := receipt["warning_group_count"]; got != float64(1) || receipt["warning_occurrence_count"] != float64(1) || receipt["diagnostic_group_count"] != float64(0) || receipt["diagnostic_occurrence_count"] != float64(0) || receipt["diagnostics_truncated"] != false {
|
||||
t.Fatalf("diagnostic counts = %#v", receipt)
|
||||
}
|
||||
debugDirectory, ok := receipt["debug_directory"].(string)
|
||||
if !ok || !filepath.IsAbs(debugDirectory) || debugDirectory != onlyChildDir(t, roots.debug) {
|
||||
t.Fatalf("debug_directory = %q", receipt["debug_directory"])
|
||||
}
|
||||
if strings.Contains(stdout.String(), "complete:") || strings.Contains(stdout.String(), "debug=") {
|
||||
t.Fatalf("machine stdout contains human reporting: %q", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunResultReportsSuccessfulRejection(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
configBytes, err := os.ReadFile(roots.config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
configBytes = []byte(replaceRequiredOnce(t, string(configBytes), " normalize:\n module: test/normalize\n", " normalize:\n module: test/normalize\n validators:\n - module: generic/always_reject\n validation_policy:\n semantic_rejection: reject_output\n"))
|
||||
if err := os.WriteFile(roots.config, configBytes, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
harness := newStateTestHarness()
|
||||
opts := harness.options()
|
||||
if err := alwaysreject.RegisterTyped[stateTestArtifact](opts.Registries.Validators, stateTestArtifactKind); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts.Catalog = catalogFromRegistries(opts.Registries)
|
||||
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{
|
||||
"run", "sample", "--config", roots.config, "--input", roots.input,
|
||||
"--chunk_cache", "bypass", "--json",
|
||||
}, &stdout, &stderr, opts)
|
||||
if code != 0 || stderr.Len() != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
receipt := decodeRunResultDocument(t, stdout.String())
|
||||
if got := receipt["normalized_output_count"]; got != float64(0) {
|
||||
t.Fatalf("normalized_output_count = %v", got)
|
||||
}
|
||||
if got := receipt["rejected_output_count"]; got != float64(1) {
|
||||
t.Fatalf("rejected_output_count = %v", got)
|
||||
}
|
||||
if got := receipt["validation_status"]; got != "rejected" {
|
||||
t.Fatalf("validation_status = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunResultIsAbsentForSyntaxAndRuntimeFailures(t *testing.T) {
|
||||
t.Run("syntax", func(t *testing.T) {
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "sample", "--json"}, &stdout, &stderr, newStateTestHarness().options())
|
||||
if code != 2 || stdout.Len() != 0 || stderr.Len() == 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("runtime", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
harness := newStateTestHarness()
|
||||
harness.extractErr = errors.New("injected extraction failure")
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{
|
||||
"run", "sample", "--config", roots.config, "--input", roots.input,
|
||||
"--chunk_cache", "bypass", "--json",
|
||||
}, &stdout, &stderr, harness.options())
|
||||
if code != 1 || stdout.Len() != 0 || stderr.Len() == 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestRunResultDeliveryFailureRetainsPublishedBundles(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
writerErr := errors.New("result writer sentinel")
|
||||
stdout := &resultDeliveryWriter{err: writerErr}
|
||||
var stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{
|
||||
"run", "sample", "--config", roots.config, "--input", roots.input,
|
||||
"--chunk_cache", "bypass", "--debug", "--json",
|
||||
}, stdout, &stderr, newStateTestHarness().options())
|
||||
if code != 1 || !strings.Contains(stderr.String(), "write run result") || strings.Contains(stderr.String(), writerErr.Error()) {
|
||||
t.Fatalf("code=%d stderr=%q", code, stderr.String())
|
||||
}
|
||||
if stdout.accepted.Len() != 0 {
|
||||
t.Fatalf("accepted stdout = %q", stdout.accepted.String())
|
||||
}
|
||||
assertStateTestOutput(t, roots.output)
|
||||
debugBundle := onlyChildDir(t, roots.debug)
|
||||
report := readStateTestRunReport(t, debugBundle)
|
||||
if !report.Succeeded {
|
||||
t.Fatalf("debug report = %#v, want successful persisted run", report)
|
||||
}
|
||||
if strings.Contains(readAllFiles(t, debugBundle), writerErr.Error()) {
|
||||
t.Fatalf("debug bundle contains result writer error")
|
||||
}
|
||||
}
|
||||
|
||||
func decodeRunResultDocument(t *testing.T, stdout string) map[string]any {
|
||||
func decodeCompletionReport(t *testing.T, format commandkit.Format, content []byte) map[string]any {
|
||||
t.Helper()
|
||||
if strings.Count(stdout, "\n") != 1 {
|
||||
t.Fatalf("stdout = %q, want one JSON document", stdout)
|
||||
if format == commandkit.FormatYAML {
|
||||
value, err := commandkit.DecodeYAMLReport(bytes.NewReader(content))
|
||||
if err != nil {
|
||||
t.Fatalf("decode native YAML report: %v\n%s", err, content)
|
||||
}
|
||||
content, err = json.Marshal(value)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
var receipt map[string]any
|
||||
if err := json.Unmarshal([]byte(stdout), &receipt); err != nil {
|
||||
t.Fatalf("decode run result: %v; stdout=%q", err, stdout)
|
||||
var report map[string]any
|
||||
if err := json.Unmarshal(content, &report); err != nil {
|
||||
t.Fatalf("decode native report: %v\n%s", err, content)
|
||||
}
|
||||
return receipt
|
||||
return report
|
||||
}
|
||||
|
||||
type resultDeliveryWriter struct {
|
||||
err error
|
||||
accepted bytes.Buffer
|
||||
func assertCompletedRunFiles(t *testing.T, file string, captured bool) {
|
||||
t.Helper()
|
||||
run := filepath.Join(filepath.Dir(file), "output-must-not-exist", "run-1")
|
||||
if content, err := os.ReadFile(filepath.Join(run, "lanes", "notes.json")); err != nil || len(content) == 0 {
|
||||
t.Fatalf("published domain artifact: %s %v", content, err)
|
||||
}
|
||||
files, err := os.ReadDir(run)
|
||||
if err != nil || len(files) != 1 || files[0].Name() != "lanes" {
|
||||
t.Fatalf("unexpected reporting sidecars: %#v %v", files, err)
|
||||
}
|
||||
if captured {
|
||||
debug := filepath.Join(filepath.Dir(file), "debug-must-not-exist", "run-1")
|
||||
content, err := os.ReadFile(filepath.Join(debug, "outcome.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var terminal struct {
|
||||
Outcome struct {
|
||||
Published bool `json:"published"`
|
||||
} `json:"outcome"`
|
||||
Error string `json:"error"`
|
||||
CaptureError string `json:"capture_error"`
|
||||
}
|
||||
if err := json.Unmarshal(content, &terminal); err != nil || !terminal.Outcome.Published || terminal.Error != "" || terminal.CaptureError != "" {
|
||||
t.Fatalf("capture was not completed before reporting: %s %v", content, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *resultDeliveryWriter) Write(content []byte) (int, error) {
|
||||
if w.err != nil {
|
||||
return 0, w.err
|
||||
}
|
||||
return w.accepted.Write(content)
|
||||
type completedRunWriter struct {
|
||||
t *testing.T
|
||||
file string
|
||||
captured bool
|
||||
content bytes.Buffer
|
||||
}
|
||||
|
||||
func (writer *completedRunWriter) Write(content []byte) (int, error) {
|
||||
assertCompletedRunFiles(writer.t, writer.file, writer.captured)
|
||||
return writer.content.Write(content)
|
||||
}
|
||||
|
||||
func TestNativeRunCompletionFormatsFollowPublicationAndCapture(t *testing.T) {
|
||||
for _, format := range []commandkit.Format{commandkit.FormatText, commandkit.FormatJSON, commandkit.FormatYAML} {
|
||||
t.Run(string(format), func(t *testing.T) {
|
||||
probe := &runProbe{}
|
||||
app, file, input, _ := runFixture(t, probe)
|
||||
output := &completedRunWriter{t: t, file: file, captured: true}
|
||||
var diagnostics bytes.Buffer
|
||||
outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--format", string(format)}, commandkit.Streams{Out: output, Err: &diagnostics})
|
||||
if outcome.Status != 0 || !strings.Contains(diagnostics.String(), "Running pipeline") {
|
||||
t.Fatalf("completion: %#v stderr=%s", outcome, diagnostics.String())
|
||||
}
|
||||
directory := filepath.Join(filepath.Dir(file), "output-must-not-exist", "run-1")
|
||||
debug := filepath.Join(filepath.Dir(file), "debug-must-not-exist", "run-1")
|
||||
if format == commandkit.FormatText {
|
||||
if !strings.Contains(output.content.String(), "Success: demo produced 1 artifacts in "+directory) || !strings.Contains(output.content.String(), "Debug capture: "+debug) || strings.Contains(output.content.String(), "fallback") {
|
||||
t.Fatalf("native completion text: %s", output.content.String())
|
||||
}
|
||||
} else {
|
||||
report := decodeCompletionReport(t, format, output.content.Bytes())
|
||||
if len(report) != 4 || report["pipeline"] != "demo" || report["artifact_count"] != float64(1) || report["output_directory"] != directory || report["debug_directory"] != debug {
|
||||
t.Fatalf("native report: %#v", report)
|
||||
}
|
||||
}
|
||||
if probe.produced != 1 {
|
||||
t.Fatalf("reporting reran a producer: %d", probe.produced)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeRunDisclosesFallbackIndependentlyOfDiagnosticVisibility(t *testing.T) {
|
||||
for _, format := range []commandkit.Format{commandkit.FormatText, commandkit.FormatJSON, commandkit.FormatYAML} {
|
||||
for _, verbose := range []bool{false, true} {
|
||||
probe := &runProbe{correctFirst: true}
|
||||
app, file, input, _ := runFixture(t, probe)
|
||||
args := []string{"pipeline", "run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--set", `/pipelines/demo/mode="permissive"`, "--set", "/pipelines/demo/max_additional_turns=0", "--format", string(format)}
|
||||
if verbose {
|
||||
args = append(args, "--verbose")
|
||||
}
|
||||
var output, diagnostics bytes.Buffer
|
||||
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: &diagnostics})
|
||||
if outcome.Status != 0 || probe.produced != 1 {
|
||||
t.Fatalf("permissive completion: %#v %s", outcome, diagnostics.String())
|
||||
}
|
||||
if verbose && !strings.Contains(diagnostics.String(), "WARN:") || !verbose && diagnostics.Len() != 0 {
|
||||
t.Fatalf("native warning visibility: %s", diagnostics.String())
|
||||
}
|
||||
if format == commandkit.FormatText {
|
||||
if !strings.Contains(output.String(), "Success with fallback:") || !strings.Contains(output.String(), "1 operations used fallback.") {
|
||||
t.Fatalf("fallback text: %s", output.String())
|
||||
}
|
||||
} else {
|
||||
report := decodeCompletionReport(t, format, output.Bytes())
|
||||
if report["fallback_count"] != float64(1) || report["artifact_count"] != float64(1) || len(report) != 4 {
|
||||
t.Fatalf("fallback report: %#v", report)
|
||||
}
|
||||
}
|
||||
assertCompletedRunFiles(t, file, false)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeRunQuietAndDebugUseNativeModeOrdering(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
flags []string
|
||||
quiet bool
|
||||
progress bool
|
||||
}{
|
||||
{[]string{"--debug", "--quiet"}, true, false},
|
||||
{[]string{"--quiet", "--debug"}, false, true},
|
||||
{[]string{"--debug", "--progress=false"}, false, false},
|
||||
} {
|
||||
probe := &runProbe{}
|
||||
app, file, input, _ := runFixture(t, probe)
|
||||
var output, diagnostics bytes.Buffer
|
||||
args := append([]string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--format", "json"}, test.flags...)
|
||||
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: &diagnostics})
|
||||
if outcome.Status != 0 || test.quiet != (output.Len() == 0) || test.progress != strings.Contains(diagnostics.String(), "Running pipeline") {
|
||||
t.Fatalf("native mode order: %#v output=%s stderr=%s", outcome, output.String(), diagnostics.String())
|
||||
}
|
||||
assertCompletedRunFiles(t, file, true)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeRunCancellationKeepsItsCauseAndPublishesNothing(t *testing.T) {
|
||||
probe := &runProbe{}
|
||||
app, file, input, _ := runFixture(t, probe)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
probe.onGenerate = cancel
|
||||
var output, diagnostics bytes.Buffer
|
||||
outcome := app.Execute(ctx, []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--quiet"}, commandkit.Streams{Out: &output, Err: &diagnostics})
|
||||
if outcome.Status != 1 || !errors.Is(outcome.Cause, context.Canceled) || output.Len() != 0 || diagnostics.Len() == 0 {
|
||||
t.Fatalf("cancellation: %#v stdout=%s stderr=%s", outcome, output.String(), diagnostics.String())
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(filepath.Dir(file), "output-must-not-exist")); !os.IsNotExist(err) {
|
||||
t.Fatalf("cancellation published artifacts: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeRunPrimaryFailureSurvivesCaptureAndDiagnosticFailures(t *testing.T) {
|
||||
primary := errors.New("provider failure sentinel")
|
||||
delivery := errors.New("diagnostic failure sentinel")
|
||||
probe := &runProbe{providerErr: primary}
|
||||
app, file, input, _ := runFixture(t, probe)
|
||||
debug := filepath.Join(filepath.Dir(file), "debug-must-not-exist", "run-1")
|
||||
probe.onGenerate = func() {
|
||||
if err := os.Mkdir(filepath.Join(debug, "outcome.json"), 0700); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
var output bytes.Buffer
|
||||
outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--llm-transient-retries=0"}, commandkit.Streams{Out: &output, Err: failingResultWriter{cause: delivery}})
|
||||
var captureFailure *os.LinkError
|
||||
if outcome.Status != 1 || !errors.Is(outcome.Cause, primary) || !errors.As(outcome.Cause, &captureFailure) || errors.Is(outcome.Cause, delivery) || !errors.Is(outcome.Presentation.Diagnostics.WriteError, delivery) || output.Len() != 0 || len(probe.requests) != 1 {
|
||||
t.Fatalf("primary/capture/delivery failure: %#v", outcome)
|
||||
}
|
||||
if !strings.Contains(outcome.Explanation.Message, debug) {
|
||||
t.Fatalf("failed run capture location unavailable: %#v", outcome)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeRunReportDeliveryFailuresRetainArtifactsAndNativePresentation(t *testing.T) {
|
||||
sentinel := errors.New("result delivery sentinel")
|
||||
for _, partial := range []bool{false, true} {
|
||||
probe := &runProbe{}
|
||||
app, file, input, _ := runFixture(t, probe)
|
||||
var output io.Writer = failingResultWriter{cause: sentinel}
|
||||
prefix := &partialResultWriter{cause: sentinel}
|
||||
if partial {
|
||||
output = prefix
|
||||
}
|
||||
var diagnostics bytes.Buffer
|
||||
outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--format", "json"}, commandkit.Streams{Out: output, Err: &diagnostics})
|
||||
if outcome.Status != 1 || outcome.Cause != nil || !errors.Is(outcome.Presentation.Output.WriteError, sentinel) || outcome.Presentation.Output.State != commandkit.DeliveryIncomplete || probe.produced != 1 || partial && prefix.content.Len() == 0 {
|
||||
t.Fatalf("delivery failure: %#v %s", outcome, diagnostics.String())
|
||||
}
|
||||
assertCompletedRunFiles(t, file, true)
|
||||
for _, root := range []string{"output-must-not-exist", "debug-must-not-exist"} {
|
||||
if !strings.Contains(diagnostics.String(), filepath.Join(filepath.Dir(file), root, "run-1")) {
|
||||
t.Fatalf("missing available path after delivery failure: %s", diagnostics.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeRunDiagnosticDeliveryFailureIsNotAnOperationFailure(t *testing.T) {
|
||||
sentinel := errors.New("diagnostic delivery sentinel")
|
||||
probe := &runProbe{correctFirst: true}
|
||||
app, file, input, _ := runFixture(t, probe)
|
||||
var output bytes.Buffer
|
||||
outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--set", `/pipelines/demo/mode="permissive"`, "--set", "/pipelines/demo/max_additional_turns=0", "--format", "json", "--verbose"}, commandkit.Streams{Out: &output, Err: failingResultWriter{cause: sentinel}})
|
||||
if outcome.Status != 1 || outcome.Cause != nil || !errors.Is(outcome.Presentation.Diagnostics.WriteError, sentinel) || outcome.Presentation.Diagnostics.State != commandkit.DeliveryIncomplete {
|
||||
t.Fatalf("diagnostic delivery: %#v", outcome)
|
||||
}
|
||||
if report := decodeCompletionReport(t, commandkit.FormatJSON, output.Bytes()); report["fallback_count"] != float64(1) {
|
||||
t.Fatalf("completed report was replaced by a delivery envelope: %#v", report)
|
||||
}
|
||||
if !strings.Contains(outcome.Explanation.Message, filepath.Join(filepath.Dir(file), "output-must-not-exist", "run-1")) {
|
||||
t.Fatalf("presentation failure lost available artifacts: %#v", outcome)
|
||||
}
|
||||
assertCompletedRunFiles(t, file, false)
|
||||
}
|
||||
|
||||
func TestNativeRunCaptureFailureAfterPublicationDoesNotReportSuccess(t *testing.T) {
|
||||
probe := &runProbe{}
|
||||
app, file, input, _ := runFixture(t, probe)
|
||||
var once sync.Once
|
||||
debug := filepath.Join(filepath.Dir(file), "debug-must-not-exist", "run-1")
|
||||
probe.onGenerate = func() {
|
||||
once.Do(func() {
|
||||
// The external provider boundary runs after capture allocation but
|
||||
// before terminalization. Make only the terminal file unusable.
|
||||
if err := os.Mkdir(filepath.Join(debug, "outcome.json"), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
var output, diagnostics bytes.Buffer
|
||||
outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--format", "json"}, commandkit.Streams{Out: &output, Err: &diagnostics})
|
||||
if outcome.Status != 1 || outcome.Cause == nil || output.Len() != 0 || probe.produced != 1 {
|
||||
t.Fatalf("post-publication capture failure: %#v stdout=%s stderr=%s", outcome, output.String(), diagnostics.String())
|
||||
}
|
||||
assertCompletedRunFiles(t, file, false)
|
||||
if !strings.Contains(diagnostics.String(), debug) || !strings.Contains(diagnostics.String(), filepath.Join(filepath.Dir(file), "output-must-not-exist", "run-1")) {
|
||||
t.Fatalf("post-publication locations: %s", diagnostics.String())
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(debug, "executions.json")); err != nil {
|
||||
t.Fatalf("existing capture was removed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeRunFailureCategoriesPreserveCausesWithoutSuccessReports(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
configure func(*testing.T, *runProbe, string, string) []string
|
||||
status int
|
||||
cause error
|
||||
}{
|
||||
{"missing configuration", func(t *testing.T, _ *runProbe, _, input string) []string {
|
||||
return []string{"run", "demo", "--input-file", input, "--config", filepath.Join(t.TempDir(), "missing.yml")}
|
||||
}, 1, nil},
|
||||
{"invalid configuration", func(t *testing.T, _ *runProbe, file, input string) []string {
|
||||
return []string{"run", "demo", "--config", file, "--input-file", input, "--set", "/version=5"}
|
||||
}, 2, nil},
|
||||
{"unknown pipeline", func(_ *testing.T, _ *runProbe, file, input string) []string {
|
||||
return []string{"run", "missing", "--config", file, "--input-file", input}
|
||||
}, 2, nil},
|
||||
{"missing input", func(t *testing.T, _ *runProbe, file, _ string) []string {
|
||||
return []string{"run", "demo", "--config", file, "--input-file", filepath.Join(t.TempDir(), "missing.txt")}
|
||||
}, 1, os.ErrNotExist},
|
||||
{"publication failure", func(t *testing.T, _ *runProbe, file, input string) []string {
|
||||
path := filepath.Join(t.TempDir(), "occupied")
|
||||
if err := os.WriteFile(path, []byte("keep"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--output-dir", path}
|
||||
}, 1, nil},
|
||||
{"provider failure", func(_ *testing.T, probe *runProbe, file, input string) []string {
|
||||
probe.providerErr = io.ErrUnexpectedEOF
|
||||
return []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--llm-transient-retries=0"}
|
||||
}, 1, io.ErrUnexpectedEOF},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
probe := &runProbe{}
|
||||
app, file, input, _ := runFixture(t, probe)
|
||||
var output, diagnostics bytes.Buffer
|
||||
args := append(test.configure(t, probe, file, input), "--quiet", "--format", "json")
|
||||
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: &diagnostics})
|
||||
if outcome.Status != test.status || test.cause != nil && !errors.Is(outcome.Cause, test.cause) || output.Len() != 0 || diagnostics.Len() == 0 {
|
||||
t.Fatalf("failure category: %#v stdout=%s stderr=%s", outcome, output.String(), diagnostics.String())
|
||||
}
|
||||
if test.name == "missing configuration" || test.name == "invalid configuration" {
|
||||
var load *configkit.LoadError
|
||||
if !errors.As(outcome.Cause, &load) {
|
||||
t.Fatalf("lost native load cause: %#v", outcome)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,229 +2,136 @@ package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"text/template"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"go.maximumdirect.net/commandkit"
|
||||
)
|
||||
|
||||
func TestRunResultEncodesRequiredFieldsAndCounts(t *testing.T) {
|
||||
result, err := newRunResult(testResolvedPipeline(pipeline.DefaultOutputModule), testRunOutput(), "relative-output", "relative-debug")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
encoded, err := encodeRunResult(result)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if encoded[len(encoded)-1] != '\n' || bytes.Count(encoded, []byte{'\n'}) != 1 {
|
||||
t.Fatalf("encoded result is not one newline-terminated object: %q", encoded)
|
||||
}
|
||||
|
||||
var decoded map[string]any
|
||||
if err := json.Unmarshal(encoded, &decoded); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := decoded["schema_version"]; got != runResultSchemaVersion {
|
||||
t.Fatalf("schema_version = %q", got)
|
||||
}
|
||||
if got := decoded["run_id"]; got != "run-123" {
|
||||
t.Fatalf("run_id = %q", got)
|
||||
}
|
||||
if got := decoded["pipeline_id"]; got != "sample" {
|
||||
t.Fatalf("pipeline_id = %q", got)
|
||||
}
|
||||
if got := decoded["validation_status"]; got != "rejected" {
|
||||
t.Fatalf("validation_status = %q", got)
|
||||
}
|
||||
if got := decoded["index_file"]; got != "index.json" {
|
||||
t.Fatalf("index_file = %q", got)
|
||||
}
|
||||
if got := decoded["normalized_output_count"]; got != float64(2) {
|
||||
t.Fatalf("normalized_output_count = %v", got)
|
||||
}
|
||||
if got := decoded["rejected_output_count"]; got != float64(1) {
|
||||
t.Fatalf("rejected_output_count = %v", got)
|
||||
}
|
||||
if got := decoded["warning_group_count"]; got != float64(1) || decoded["warning_occurrence_count"] != float64(1) || decoded["diagnostic_group_count"] != float64(0) || decoded["diagnostic_occurrence_count"] != float64(0) || decoded["diagnostics_truncated"] != false {
|
||||
t.Fatalf("diagnostic counts = %#v", decoded)
|
||||
}
|
||||
if got := decoded["validation_summaries"]; got != nil {
|
||||
t.Fatalf("validation_summaries = %#v, want omitted when empty", got)
|
||||
}
|
||||
if got := decoded["output_directory"]; got != filepath.Join(mustWorkingDirectory(t), "relative-output") {
|
||||
t.Fatalf("output_directory = %q", got)
|
||||
}
|
||||
if got := decoded["debug_directory"]; got != filepath.Join(mustWorkingDirectory(t), "relative-debug") {
|
||||
t.Fatalf("debug_directory = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunResultRejectsInvalidRequiredValues(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
resolved pipeline.ResolvedPipeline
|
||||
output pipeline.RunOutput
|
||||
directory string
|
||||
}{
|
||||
{name: "blank run ID", resolved: testResolvedPipeline(pipeline.DefaultOutputModule), output: testRunOutputWithout(func(output *pipeline.RunOutput) { output.Manifest.RunID = " " }), directory: "output"},
|
||||
{name: "blank resolved pipeline ID", resolved: pipeline.ResolvedPipeline{Output: pipeline.ModuleBinding{Module: pipeline.DefaultOutputModule}}, output: testRunOutput(), directory: "output"},
|
||||
{name: "blank manifest pipeline ID", resolved: testResolvedPipeline(pipeline.DefaultOutputModule), output: testRunOutputWithout(func(output *pipeline.RunOutput) { output.Manifest.PipelineID = "" }), directory: "output"},
|
||||
{name: "mismatched pipeline IDs", resolved: testResolvedPipeline(pipeline.DefaultOutputModule), output: testRunOutputWithout(func(output *pipeline.RunOutput) { output.Manifest.PipelineID = "other" }), directory: "output"},
|
||||
{name: "blank validation status", resolved: testResolvedPipeline(pipeline.DefaultOutputModule), output: testRunOutputWithout(func(output *pipeline.RunOutput) { output.Manifest.ValidationStatus = " " }), directory: "output"},
|
||||
{name: "blank output directory", resolved: testResolvedPipeline(pipeline.DefaultOutputModule), output: testRunOutput(), directory: " "},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if _, err := newRunResult(tt.resolved, tt.output, tt.directory, ""); err == nil {
|
||||
t.Fatal("newRunResult() succeeded")
|
||||
func TestRunResultUsesOneNativeValueAcrossFormats(t *testing.T) {
|
||||
value := runResult{Pipeline: "demo", ArtifactCount: 3, OutputDirectory: "/artifacts/run-one", DebugDirectory: "/debug/run-one", FallbackCount: 2}
|
||||
for _, format := range []commandkit.Format{commandkit.FormatText, commandkit.FormatJSON, commandkit.FormatYAML} {
|
||||
var output bytes.Buffer
|
||||
if err := commandkit.RenderReport(&output, commandkit.Report{Value: value, TextTemplate: runResultTemplate}, format); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if format == commandkit.FormatText {
|
||||
if !strings.Contains(output.String(), "Success with fallback: demo produced 3 artifacts in /artifacts/run-one") || !strings.Contains(output.String(), "2 operations used fallback.") || !strings.Contains(output.String(), "Debug capture: /debug/run-one") {
|
||||
t.Fatalf("fallback text: %s", output.String())
|
||||
}
|
||||
})
|
||||
continue
|
||||
}
|
||||
report := decodeCompletionReport(t, format, output.Bytes())
|
||||
if report["pipeline"] != "demo" || report["output_directory"] != value.OutputDirectory || report["debug_directory"] != value.DebugDirectory || report["fallback_count"] != float64(2) || report["artifact_count"] != float64(3) || len(report) != 5 {
|
||||
t.Fatalf("completion value: %#v", report)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunResultOmitsIndexFileForOtherOutputModules(t *testing.T) {
|
||||
result, err := newRunResult(testResolvedPipeline("test/output"), testRunOutputWithout(func(output *pipeline.RunOutput) {
|
||||
output.OutputFiles = nil
|
||||
}), "output", "")
|
||||
func TestNativeCompletionDiagnosticsRetainDomainClassification(t *testing.T) {
|
||||
collection := contracts.DiagnosticCollection{}
|
||||
for _, disposition := range []contracts.DiagnosticDisposition{contracts.DiagnosticDispositionWarning, contracts.DiagnosticDispositionAdvisory, contracts.DiagnosticDispositionObservation} {
|
||||
category := contracts.DiagnosticCategoryDataQuality
|
||||
if disposition == contracts.DiagnosticDispositionWarning {
|
||||
category = contracts.DiagnosticCategoryFallback
|
||||
} else if disposition == contracts.DiagnosticDispositionObservation {
|
||||
category = contracts.DiagnosticCategoryNormalization
|
||||
}
|
||||
collection.Groups = append(collection.Groups, contracts.DiagnosticGroup{Disposition: disposition, Category: category, ReasonCode: string(disposition), Origin: contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract, StepID: "build", LaneID: "notes", ModuleKey: "producer"}, OccurrenceCount: 1, Samples: []contracts.DiagnosticSample{{Scope: "current source", Message: string(disposition) + " finding"}}})
|
||||
}
|
||||
if err := collection.Validate(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var output, diagnostics bytes.Buffer
|
||||
app, err := commandkit.New(commandkit.Identity{Name: "fixture"}, commandkit.NewCommand(commandkit.CommandSpec[struct{}]{Name: "report", Run: func(invocation commandkit.Invocation, _ struct{}) commandkit.Outcome {
|
||||
reportRunDiagnostics(invocation.Reporter(), collection)
|
||||
return reportRunResult(invocation, runResult{Pipeline: "demo", ArtifactCount: 1, OutputDirectory: "/artifacts/run-one"})
|
||||
}}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.IndexFile != "" {
|
||||
t.Fatalf("index_file = %q", result.IndexFile)
|
||||
outcome := app.Execute(context.Background(), []string{"report", "--verbose"}, commandkit.Streams{Out: &output, Err: &diagnostics})
|
||||
if outcome.Status != 0 || strings.Contains(output.String(), "finding") || strings.Count(diagnostics.String(), "WARN:") != 1 {
|
||||
t.Fatalf("diagnostic classification: %#v stdout=%s stderr=%s", outcome, output.String(), diagnostics.String())
|
||||
}
|
||||
encoded, err := encodeRunResult(result)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var decoded map[string]any
|
||||
if err := json.Unmarshal(encoded, &decoded); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := decoded["index_file"]; ok {
|
||||
t.Fatalf("encoded non-JSON result contains index_file: %s", encoded)
|
||||
}
|
||||
if _, ok := decoded["debug_directory"]; ok {
|
||||
t.Fatalf("encoded result without debug capture contains debug_directory: %s", encoded)
|
||||
for _, group := range collection.Groups {
|
||||
if !strings.Contains(diagnostics.String(), string(group.Disposition)+" finding") {
|
||||
t.Fatalf("missing diagnostic: %s", diagnostics.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunResultProjectsOwnedValidationSummaries(t *testing.T) {
|
||||
output := testRunOutput()
|
||||
output.Manifest.ValidationSummaries = []artifacts.ValidationSummary{{Status: "incomplete", IncompleteValidators: []string{"validator"}, ProducerAttemptCount: 1, TerminalAction: "warn_continue"}}
|
||||
result, err := newRunResult(testResolvedPipeline(pipeline.DefaultOutputModule), output, "output", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
output.Manifest.ValidationSummaries[0].IncompleteValidators[0] = "caller mutation"
|
||||
if got := result.ValidationSummaries[0].IncompleteValidators; len(got) != 1 || got[0] != "validator" {
|
||||
t.Fatalf("result validation summaries = %#v", result.ValidationSummaries)
|
||||
}
|
||||
encoded, err := encodeRunResult(result)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Contains(encoded, []byte(`"validation_summaries":[{"status":"incomplete","incomplete_validators":["validator"],"producer_attempt_count":1,"terminal_action":"warn_continue"}]`)) {
|
||||
t.Fatalf("encoded result = %s", encoded)
|
||||
func TestCompletedRunRenderingFailureRetainsPublicationEvenWhenQuiet(t *testing.T) {
|
||||
previous := runResultTemplate
|
||||
t.Cleanup(func() { runResultTemplate = previous })
|
||||
// Simulate a broken application-owned text template, not a new renderer seam.
|
||||
runResultTemplate = template.Must(commandkit.ParseTextTemplate("invalid", "{{.MissingField}}", nil))
|
||||
for _, quiet := range []bool{false, true} {
|
||||
probe := &runProbe{}
|
||||
app, file, input, _ := runFixture(t, probe)
|
||||
args := []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug"}
|
||||
if quiet {
|
||||
args = append(args, "--quiet")
|
||||
}
|
||||
var output, diagnostics bytes.Buffer
|
||||
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: &diagnostics})
|
||||
if outcome.Status != 1 || outcome.Cause != nil || outcome.Presentation.Output.RenderError == nil || outcome.Presentation.Output.State != commandkit.DeliveryIncomplete || probe.produced != 1 {
|
||||
t.Fatalf("render failure: %#v %s", outcome, diagnostics.String())
|
||||
}
|
||||
assertCompletedRunFiles(t, file, true)
|
||||
artifacts := filepath.Join(filepath.Dir(file), "output-must-not-exist", "run-1")
|
||||
if !strings.Contains(diagnostics.String(), artifacts) || (quiet && output.Len() != 0) {
|
||||
t.Fatalf("render failure locations/quiet: %s", diagnostics.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunResultRequiresOneProductionIndexFile(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
files []contracts.OutputFile
|
||||
}{
|
||||
{name: "missing", files: nil},
|
||||
{name: "duplicate", files: []contracts.OutputFile{{Name: "index.json"}, {Name: "index.json"}}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
output := testRunOutput()
|
||||
output.OutputFiles = tt.files
|
||||
if _, err := newRunResult(testResolvedPipeline(pipeline.DefaultOutputModule), output, "output", ""); err == nil {
|
||||
t.Fatal("newRunResult() succeeded")
|
||||
}
|
||||
})
|
||||
}
|
||||
type failingResultWriter struct {
|
||||
cause error
|
||||
}
|
||||
|
||||
func TestWriteRunResultCompletesAndReportsWriterFailure(t *testing.T) {
|
||||
content := []byte("result\n")
|
||||
var target bytes.Buffer
|
||||
if err := writeRunResult(partialResultWriter{writer: &target, limit: 2}, content); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := target.String(); got != string(content) {
|
||||
t.Fatalf("written result = %q", got)
|
||||
}
|
||||
|
||||
writerErr := errors.New("result writer failed")
|
||||
if err := writeRunResult(failingResultWriter{err: writerErr}, content); !errors.Is(err, writerErr) {
|
||||
t.Fatalf("writeRunResult() error = %v", err)
|
||||
}
|
||||
if err := writeRunResult(zeroResultWriter{}, content); !errors.Is(err, io.ErrShortWrite) {
|
||||
t.Fatalf("zero-progress error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func testResolvedPipeline(outputModule string) pipeline.ResolvedPipeline {
|
||||
return pipeline.ResolvedPipeline{ID: "sample", Output: pipeline.ModuleBinding{Module: outputModule}}
|
||||
}
|
||||
|
||||
func testRunOutput() pipeline.RunOutput {
|
||||
return pipeline.RunOutput{
|
||||
Manifest: artifacts.RunManifest{RunID: "run-123", PipelineID: "sample", ValidationStatus: "rejected"},
|
||||
NormalizeOutputs: []contracts.SerializedOutput{{}, {}},
|
||||
Rejected: []contracts.RejectedOutput{{}},
|
||||
Diagnostics: contracts.DiagnosticCollection{Groups: []contracts.DiagnosticGroup{{
|
||||
Disposition: contracts.DiagnosticDispositionWarning,
|
||||
Category: contracts.DiagnosticCategoryFallback,
|
||||
ReasonCode: "fallback",
|
||||
Origin: contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageNormalize, StepID: "step", LaneID: "lane", ModuleKey: "module"},
|
||||
OccurrenceCount: 1,
|
||||
Samples: []contracts.DiagnosticSample{{Scope: "scope", Message: "message"}},
|
||||
}}},
|
||||
OutputFiles: []contracts.OutputFile{{Name: "index.json"}},
|
||||
}
|
||||
}
|
||||
|
||||
func testRunOutputWithout(change func(*pipeline.RunOutput)) pipeline.RunOutput {
|
||||
output := testRunOutput()
|
||||
change(&output)
|
||||
return output
|
||||
}
|
||||
|
||||
func mustWorkingDirectory(t *testing.T) string {
|
||||
t.Helper()
|
||||
workingDirectory, err := filepath.Abs(".")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return workingDirectory
|
||||
}
|
||||
func (writer failingResultWriter) Write([]byte) (int, error) { return 0, writer.cause }
|
||||
|
||||
type partialResultWriter struct {
|
||||
writer io.Writer
|
||||
limit int
|
||||
content bytes.Buffer
|
||||
cause error
|
||||
}
|
||||
|
||||
func (w partialResultWriter) Write(content []byte) (int, error) {
|
||||
if len(content) > w.limit {
|
||||
content = content[:w.limit]
|
||||
func (writer *partialResultWriter) Write(content []byte) (int, error) {
|
||||
if len(content) > 8 {
|
||||
content = content[:8]
|
||||
}
|
||||
return w.writer.Write(content)
|
||||
n, _ := writer.content.Write(content)
|
||||
return n, writer.cause
|
||||
}
|
||||
|
||||
type failingResultWriter struct{ err error }
|
||||
var _ io.Writer = (*partialResultWriter)(nil)
|
||||
|
||||
func (w failingResultWriter) Write([]byte) (int, error) { return 0, w.err }
|
||||
|
||||
type zeroResultWriter struct{}
|
||||
|
||||
func (zeroResultWriter) Write([]byte) (int, error) { return 0, nil }
|
||||
func TestPipelinePreflightClassifiesOnlyIdentifiableInvalidSelections(t *testing.T) {
|
||||
// Real native missing-resource errors retain their category separately from
|
||||
// private lookup causes. A joined operational branch must still win.
|
||||
probe := &runProbe{}
|
||||
app, file, input, _ := runFixture(t, probe)
|
||||
var diagnostics bytes.Buffer
|
||||
missing := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--llm-profile", "absent"}, commandkit.Streams{Out: io.Discard, Err: &diagnostics}).Cause
|
||||
for _, test := range []struct {
|
||||
cause error
|
||||
status int
|
||||
}{
|
||||
{missing, 2},
|
||||
{errors.Join(missing, context.Canceled), 1},
|
||||
{errors.Join(missing, errors.New("unavailable inspection")), 1},
|
||||
{errors.New("unavailable inspection"), 1},
|
||||
} {
|
||||
outcome := pipelineResourceFailure(test.cause)
|
||||
if outcome.Status != test.status || !errors.Is(outcome.Cause, test.cause) {
|
||||
t.Fatalf("preflight classification: %#v", outcome)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,6 +12,21 @@ import (
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
// SelectionError identifies an absent configured native resource. Callers can
|
||||
// distinguish this invalid selection from unavailable inspection or cancellation
|
||||
// without inspecting diagnostic text or the kit's private lookup causes.
|
||||
type SelectionError struct{ Cause error }
|
||||
|
||||
func (err *SelectionError) Error() string { return err.Cause.Error() }
|
||||
func (err *SelectionError) Unwrap() error { return err.Cause }
|
||||
|
||||
func resourceFinding(err error) error {
|
||||
if !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) && (errors.Is(err, promptkit.ErrPromptNotFound) || errors.Is(err, promptkit.ErrProfileNotFound)) {
|
||||
return &SelectionError{Cause: err}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// RequiresEngine reports selected model work, including nested validators.
|
||||
// Deterministic pipelines need no engine merely to validate their composition.
|
||||
func RequiresEngine(resolved pipeline.ResolvedPipeline) bool {
|
||||
@@ -69,7 +84,7 @@ func Validate(ctx context.Context, resolved pipeline.ResolvedPipeline, catalog p
|
||||
}
|
||||
if profile != "" {
|
||||
if _, err := engine.InspectProfile(ctx, profile); err != nil {
|
||||
findings = append(findings, fmt.Errorf("%s profile %q: %w", scope, profile, err))
|
||||
findings = append(findings, resourceFinding(fmt.Errorf("%s profile %q: %w", scope, profile, err)))
|
||||
}
|
||||
}
|
||||
for _, id := range prompts {
|
||||
@@ -78,7 +93,7 @@ func Validate(ctx context.Context, resolved pipeline.ResolvedPipeline, catalog p
|
||||
}
|
||||
prompt, err := engine.InspectPrompt(ctx, id)
|
||||
if err != nil {
|
||||
findings = append(findings, fmt.Errorf("%s prompt %q: %w", scope, id, err))
|
||||
findings = append(findings, resourceFinding(fmt.Errorf("%s prompt %q: %w", scope, id, err)))
|
||||
continue
|
||||
}
|
||||
if profile != "" {
|
||||
@@ -90,7 +105,7 @@ func Validate(ctx context.Context, resolved pipeline.ResolvedPipeline, catalog p
|
||||
continue
|
||||
}
|
||||
if _, err := engine.InspectProfile(ctx, *prompt.DefaultProfileID); err != nil {
|
||||
findings = append(findings, fmt.Errorf("%s prompt %q default profile %q: %w", scope, id, *prompt.DefaultProfileID, err))
|
||||
findings = append(findings, resourceFinding(fmt.Errorf("%s prompt %q default profile %q: %w", scope, id, *prompt.DefaultProfileID, err)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user