Report completed runs through native presentation and preserve failure outcomes
This commit is contained in:
@@ -1606,6 +1606,30 @@ input-hash session stability across changed reference bundles.
|
|||||||
|
|
||||||
## Stage 34: Finish Publication, Native Completion Reports, And Failure Outcomes
|
## Stage 34: Finish Publication, Native Completion Reports, And Failure Outcomes
|
||||||
|
|
||||||
|
**Status:** Complete. Replaced the receipt/encoder/index and validation-ledger
|
||||||
|
reporting model with one native completion value and text template. Publication
|
||||||
|
and requested capture finish before reporting; JSON/YAML share the same value,
|
||||||
|
omit zero fallback counts and disabled debug paths, and never create reporting
|
||||||
|
sidecars. Qualified fallback summaries remain visible independently of warning
|
||||||
|
verbosity. Native progress, configuration diagnostics, and process findings use
|
||||||
|
stderr without promoting domain advisories to warnings. Operation causes remain
|
||||||
|
separate from native rendering/output/diagnostic delivery evidence. Failed capture
|
||||||
|
or presentation retains published files and available paths without model retries
|
||||||
|
or repeat terminalization. Identifiable preflight selections now retain a typed
|
||||||
|
classification alongside native causes; cancellation and unknown/mixed failures
|
||||||
|
remain operational. Replaced obsolete receipt tests with real-kit/fake-provider
|
||||||
|
coverage for formats, counts, native modes/progress, quiet rendering failure,
|
||||||
|
partial/failed stdout, failed diagnostics, cancellation, publication/capture
|
||||||
|
ordering, and primary/capture/delivery cause preservation.
|
||||||
|
Focused CLI/pipeline tests, supporting preflight/debug/cache/contracts/assets/
|
||||||
|
generic/Seriatim tests, focused vet/race checks, and diff checks pass. Added real
|
||||||
|
executable format/fallback tests using a local fake HTTP provider; their setup and
|
||||||
|
the full CLI/executable checks remain blocked by legacy `framework/llm` importing
|
||||||
|
the removed reporting package and old ConfigKit's removed YAML `plugin/limit`
|
||||||
|
dependency. Those subprocess results are not claimed passing. Stage 35 owns
|
||||||
|
remaining legacy code/dependency/fixture removal and whole-tree compilation;
|
||||||
|
assembled and final verification must execute the pending checks.
|
||||||
|
|
||||||
**Goal:** subprocesses receive artifacts plus the selected small native report.
|
**Goal:** subprocesses receive artifacts plus the selected small native report.
|
||||||
|
|
||||||
**Inspect/change:** `run_result.go`, `run_terminal.go`, physical output placement,
|
**Inspect/change:** `run_result.go`, `run_terminal.go`, physical output placement,
|
||||||
|
|||||||
@@ -2,21 +2,34 @@ package cli
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"runtime"
|
"runtime"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/commandkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestExecutableRunRoutesDeclareTheSameRequiredInputs(t *testing.T) {
|
func buildNativeExecutable(t *testing.T) string {
|
||||||
|
t.Helper()
|
||||||
binary := filepath.Join(t.TempDir(), "notarius")
|
binary := filepath.Join(t.TempDir(), "notarius")
|
||||||
build := exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"), "build", "-o", binary, "../../cmd/notarius")
|
build := exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"), "build", "-o", binary, "../../cmd/notarius")
|
||||||
build.Env = append(os.Environ(), "GOWORK=off")
|
build.Env = append(os.Environ(), "GOWORK=off")
|
||||||
if output, err := build.CombinedOutput(); err != nil {
|
if output, err := build.CombinedOutput(); err != nil {
|
||||||
t.Fatalf("build executable: %v\n%s", err, output)
|
t.Fatalf("build executable: %v\n%s", err, output)
|
||||||
}
|
}
|
||||||
|
return binary
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecutableRunRoutesDeclareTheSameRequiredInputs(t *testing.T) {
|
||||||
|
binary := buildNativeExecutable(t)
|
||||||
for _, route := range [][]string{{"run"}, {"pipeline", "run"}} {
|
for _, route := range [][]string{{"run"}, {"pipeline", "run"}} {
|
||||||
help := exec.Command(binary, append(append([]string(nil), route...), "--help")...)
|
help := exec.Command(binary, append(append([]string(nil), route...), "--help")...)
|
||||||
output, err := help.CombinedOutput()
|
output, err := help.CombinedOutput()
|
||||||
@@ -31,3 +44,68 @@ func TestExecutableRunRoutesDeclareTheSameRequiredInputs(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExecutableReportsDomainArtifactsAndQualifiedFallbacks(t *testing.T) {
|
||||||
|
binary := buildNativeExecutable(t)
|
||||||
|
for _, fallback := range []bool{false, true} {
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||||
|
if fallback {
|
||||||
|
writer.WriteHeader(http.StatusServiceUnavailable)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writer.Header().Set("Content-Type", "application/json")
|
||||||
|
_ = json.NewEncoder(writer).Encode(map[string]any{"id": "local-fake", "object": "chat.completion", "choices": []any{map[string]any{"index": 0, "message": map[string]any{"role": "assistant", "content": `{"spell_casts":[]}`}, "finish_reason": "stop"}}})
|
||||||
|
}))
|
||||||
|
t.Cleanup(server.Close)
|
||||||
|
for _, format := range []commandkit.Format{commandkit.FormatText, commandkit.FormatJSON, commandkit.FormatYAML} {
|
||||||
|
t.Run(fmt.Sprintf("%s/fallback=%t", format, fallback), func(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
configuration := fmt.Sprintf(`version: 6
|
||||||
|
output: {directory: output}
|
||||||
|
pipelines:
|
||||||
|
demo:
|
||||||
|
input: {module: seriatim}
|
||||||
|
chunk: {module: generic}
|
||||||
|
llm_profile: local-fake
|
||||||
|
mode: permissive
|
||||||
|
max_additional_turns: 0
|
||||||
|
artifacts:
|
||||||
|
spells:
|
||||||
|
extract: {module: dnd/spells}
|
||||||
|
promptkit:
|
||||||
|
profiles:
|
||||||
|
- id: local-fake
|
||||||
|
endpoint: %q
|
||||||
|
model: fake-model
|
||||||
|
api_key_required: false
|
||||||
|
`, server.URL+"/v1")
|
||||||
|
file := filepath.Join(root, "config.yml")
|
||||||
|
input := filepath.Join(root, "input.json")
|
||||||
|
for path, content := range map[string]string{file: configuration, input: `{"metadata":{"id":"report-source"},"segments":[{"id":1,"start":0,"end":1,"speaker":"GM","text":"The party rests."}]}`} {
|
||||||
|
if err := os.WriteFile(path, []byte(content), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
command := exec.Command(binary, "run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--llm-transient-retries=0", "--format", string(format), "--verbose")
|
||||||
|
var output, diagnostics bytes.Buffer
|
||||||
|
command.Stdout, command.Stderr = &output, &diagnostics
|
||||||
|
if err := command.Run(); err != nil {
|
||||||
|
t.Fatalf("subprocess: %v stdout=%s stderr=%s", err, output.String(), diagnostics.String())
|
||||||
|
}
|
||||||
|
if format == commandkit.FormatText {
|
||||||
|
if !strings.Contains(output.String(), "produced 1 artifacts") || fallback != strings.Contains(output.String(), "Success with fallback:") {
|
||||||
|
t.Fatalf("completion text: %s", output.String())
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
report := decodeCompletionReport(t, format, output.Bytes())
|
||||||
|
if report["artifact_count"] != float64(1) || report["pipeline"] != "demo" || fallback && report["fallback_count"] != float64(1) || !fallback && report["fallback_count"] != nil {
|
||||||
|
t.Fatalf("completion report: %#v", report)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(report["output_directory"].(string), "lanes", "spells.json")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -107,7 +107,7 @@ func pipelineCommands(newState func() *nativeLoaders, catalog func() (pipeline.M
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if err := preflight.Validate(invocation.Context, effective.ResolvedPipeline, modules, engine); err != nil {
|
if err := preflight.Validate(invocation.Context, effective.ResolvedPipeline, modules, engine); err != nil {
|
||||||
return pipelineInvalidConfiguration(err)
|
return pipelineResourceFailure(err)
|
||||||
}
|
}
|
||||||
return reportPipelineInspection(invocation, commandkit.Report{Value: struct {
|
return reportPipelineInspection(invocation, commandkit.Report{Value: struct {
|
||||||
Pipeline string `json:"pipeline"`
|
Pipeline string `json:"pipeline"`
|
||||||
@@ -124,16 +124,56 @@ var pipelineValidationTemplate = template.Must(commandkit.ParseTextTemplate("pip
|
|||||||
|
|
||||||
func reportPipelineInspection(invocation commandkit.Invocation, report commandkit.Report) commandkit.Outcome {
|
func reportPipelineInspection(invocation commandkit.Invocation, report commandkit.Report) commandkit.Outcome {
|
||||||
if err := invocation.Reporter().Report(report); err != nil {
|
if err := invocation.Reporter().Report(report); err != nil {
|
||||||
return commandkit.Failure("Could not report pipeline inspection.").WithCause(err)
|
return commandkit.Failure("Could not report pipeline inspection.")
|
||||||
}
|
}
|
||||||
return commandkit.Success()
|
return commandkit.Success()
|
||||||
}
|
}
|
||||||
|
|
||||||
func pipelineConfigurationFailure(err error) commandkit.Outcome {
|
func pipelineConfigurationFailure(err error) commandkit.Outcome {
|
||||||
|
outcome := commandkit.Failure(err.Error()).WithCause(err)
|
||||||
if ckconfig.ClassifyConfigurationError(err) == commandkit.OutcomeUsage {
|
if ckconfig.ClassifyConfigurationError(err) == commandkit.OutcomeUsage {
|
||||||
|
outcome = commandkit.Usage(err.Error()).WithCause(err)
|
||||||
|
}
|
||||||
|
var load *configkit.LoadError
|
||||||
|
if errors.As(err, &load) {
|
||||||
|
_, explanation := ckconfig.AdaptDiagnostics(err.Error()+"\n", load.Sources(), load.Diagnostics())
|
||||||
|
outcome = outcome.WithExplanation(explanation.Message, explanation.Details...)
|
||||||
|
}
|
||||||
|
return outcome
|
||||||
|
}
|
||||||
|
|
||||||
|
func pipelineResourceFailure(err error) commandkit.Outcome {
|
||||||
|
if preflightInputError(err) {
|
||||||
return commandkit.Usage(err.Error()).WithCause(err)
|
return commandkit.Usage(err.Error()).WithCause(err)
|
||||||
}
|
}
|
||||||
return commandkit.Failure(err.Error()).WithCause(err)
|
return pipelineConfigurationFailure(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Only identifiable missing native selections are invalid preflight input.
|
||||||
|
// Unknown failures and any operational branch of a joined error remain failures.
|
||||||
|
func preflightInputError(err error) bool {
|
||||||
|
if err == nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if _, ok := err.(*preflight.SelectionError); ok {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if joined, ok := err.(interface{ Unwrap() []error }); ok {
|
||||||
|
found := false
|
||||||
|
for _, child := range joined.Unwrap() {
|
||||||
|
if child != nil {
|
||||||
|
found = true
|
||||||
|
if !preflightInputError(child) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return found
|
||||||
|
}
|
||||||
|
if wrapped, ok := err.(interface{ Unwrap() error }); ok {
|
||||||
|
return preflightInputError(wrapped.Unwrap())
|
||||||
|
}
|
||||||
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
func pipelineInvalidConfiguration(err error) commandkit.Outcome {
|
func pipelineInvalidConfiguration(err error) commandkit.Outcome {
|
||||||
|
|||||||
@@ -262,7 +262,7 @@ func TestPipelineInspectionReportFailureRemainsOperational(t *testing.T) {
|
|||||||
primary := errors.New("inspection stdout delivery failed")
|
primary := errors.New("inspection stdout delivery failed")
|
||||||
var diagnostics bytes.Buffer
|
var diagnostics bytes.Buffer
|
||||||
outcome := app.Execute(context.Background(), []string{"pipeline", "list", "--config", file}, commandkit.Streams{Out: inspectionFailWriter{primary}, Err: &diagnostics})
|
outcome := app.Execute(context.Background(), []string{"pipeline", "list", "--config", file}, commandkit.Streams{Out: inspectionFailWriter{primary}, Err: &diagnostics})
|
||||||
if outcome.Status != 1 || !errors.Is(outcome.Cause, primary) {
|
if outcome.Status != 1 || outcome.Cause != nil || !errors.Is(outcome.Presentation.Output.WriteError, primary) {
|
||||||
t.Fatalf("delivery failure classified as invalid configuration: %#v %s", outcome, diagnostics.String())
|
t.Fatalf("delivery failure classified as invalid configuration: %#v %s", outcome, diagnostics.String())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import (
|
|||||||
frameworkdebug "gitea.maximumdirect.net/eric/notarius/internal/framework/debug"
|
frameworkdebug "gitea.maximumdirect.net/eric/notarius/internal/framework/debug"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/preflight"
|
"gitea.maximumdirect.net/eric/notarius/internal/preflight"
|
||||||
|
"go.maximumdirect.net/ckconfig"
|
||||||
"go.maximumdirect.net/commandkit"
|
"go.maximumdirect.net/commandkit"
|
||||||
"go.maximumdirect.net/configkit"
|
"go.maximumdirect.net/configkit"
|
||||||
"go.maximumdirect.net/promptkit"
|
"go.maximumdirect.net/promptkit"
|
||||||
@@ -31,6 +32,14 @@ func pipelineRunCommand(version string, newState func() *nativeLoaders, modules
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return pipelineConfigurationFailure(err)
|
return pipelineConfigurationFailure(err)
|
||||||
}
|
}
|
||||||
|
diagnostics, _ := ckconfig.AdaptDiagnostics("Configuration\n", result.Sources(), result.Diagnostics())
|
||||||
|
for _, diagnostic := range diagnostics {
|
||||||
|
if diagnostic.Severity == commandkit.SeverityWarning {
|
||||||
|
_ = invocation.Reporter().AtEnd().Warn("", diagnostic.Write)
|
||||||
|
} else {
|
||||||
|
_ = invocation.Reporter().AtEnd().Info("", diagnostic.Write)
|
||||||
|
}
|
||||||
|
}
|
||||||
loaded, err := config.FromResult(result)
|
loaded, err := config.FromResult(result)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return pipelineInvalidConfiguration(err)
|
return pipelineInvalidConfiguration(err)
|
||||||
@@ -53,20 +62,20 @@ func pipelineRunCommand(version string, newState func() *nativeLoaders, modules
|
|||||||
}
|
}
|
||||||
finish := func(outcome commandkit.Outcome) commandkit.Outcome {
|
finish := func(outcome commandkit.Outcome) commandkit.Outcome {
|
||||||
if err := capture.Finalize(captureOutcome(), outcome.Cause); err != nil {
|
if err := capture.Finalize(captureOutcome(), outcome.Cause); err != nil {
|
||||||
message := err.Error()
|
|
||||||
if published {
|
|
||||||
message += fmt.Sprintf("\nArtifacts: %s", outputDirectory)
|
|
||||||
}
|
|
||||||
if capture.Path() != "" {
|
|
||||||
message += fmt.Sprintf("\nDebug capture: %s", capture.Path())
|
|
||||||
}
|
|
||||||
if outcome.Status == 0 {
|
if outcome.Status == 0 {
|
||||||
outcome = commandkit.Failure(message)
|
outcome = commandkit.Failure(err.Error())
|
||||||
} else {
|
} else {
|
||||||
outcome = outcome.WithExplanation(message)
|
outcome = outcome.WithExplanation(err.Error(), outcome.Explanation.Details...)
|
||||||
}
|
}
|
||||||
outcome = outcome.WithCause(err)
|
outcome = outcome.WithCause(err)
|
||||||
}
|
}
|
||||||
|
if outcome.Status != 0 {
|
||||||
|
availableOutput := ""
|
||||||
|
if published {
|
||||||
|
availableOutput = outputDirectory
|
||||||
|
}
|
||||||
|
outcome = outcome.WithExplanation(outcome.Explanation.Message+runLocations(availableOutput, capture.Path()), outcome.Explanation.Details...)
|
||||||
|
}
|
||||||
return outcome
|
return outcome
|
||||||
}
|
}
|
||||||
capture, err = startRunDebug(input.Debug, cfg.Debug.Directory, runID, startedAt)
|
capture, err = startRunDebug(input.Debug, cfg.Debug.Directory, runID, startedAt)
|
||||||
@@ -107,13 +116,13 @@ func pipelineRunCommand(version string, newState func() *nativeLoaders, modules
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if err := preflight.Validate(invocation.Context, effective.ResolvedPipeline, catalog, engine); err != nil {
|
if err := preflight.Validate(invocation.Context, effective.ResolvedPipeline, catalog, engine); err != nil {
|
||||||
return finish(pipelineInvalidConfiguration(err))
|
return finish(pipelineResourceFailure(err))
|
||||||
}
|
}
|
||||||
workingDirectory, err := os.Getwd()
|
workingDirectory, err := os.Getwd()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return finish(commandkit.Failure(err.Error()).WithCause(err))
|
return finish(commandkit.Failure(err.Error()).WithCause(err))
|
||||||
}
|
}
|
||||||
materialized, diagnostics, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{ConfigPath: loaded.MainSourcePath(), WorkingDir: workingDirectory})
|
materialized, referenceDiagnostics, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{ConfigPath: loaded.MainSourcePath(), WorkingDir: workingDirectory})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return finish(commandkit.Failure(err.Error()).WithCause(err))
|
return finish(commandkit.Failure(err.Error()).WithCause(err))
|
||||||
}
|
}
|
||||||
@@ -138,7 +147,9 @@ func pipelineRunCommand(version string, newState func() *nativeLoaders, modules
|
|||||||
recorder = newRunEvidenceRecorder(capture, result)
|
recorder = newRunEvidenceRecorder(capture, result)
|
||||||
}
|
}
|
||||||
group := &promptkit.ExecutionGroup{}
|
group := &promptkit.ExecutionGroup{}
|
||||||
output, err = pipeline.New().Run(invocation.Context, pipeline.RunInput{Prepared: prepared, Path: input.InputFile.Value, RawInput: raw, SessionID: sessionID, RunID: runID, StartedAt: startedAt, Diagnostics: diagnostics, Cache: cache, Debug: recorder, Workers: cfg.Concurrency.Workers, Executions: group, Execution: contracts.ExecutionOptions{Execution: input.Overrides.Execution.Execution, TransientRetries: input.Overrides.Execution.TransientRetries}})
|
_ = invocation.Reporter().Progress(fmt.Sprintf("Running pipeline %q.", effective.ResolvedPipeline.ID))
|
||||||
|
output, err = pipeline.New().Run(invocation.Context, pipeline.RunInput{Prepared: prepared, Path: input.InputFile.Value, RawInput: raw, SessionID: sessionID, RunID: runID, StartedAt: startedAt, Diagnostics: referenceDiagnostics, Cache: cache, Debug: recorder, Workers: cfg.Concurrency.Workers, Executions: group, Execution: contracts.ExecutionOptions{Execution: input.Overrides.Execution.Execution, TransientRetries: input.Overrides.Execution.TransientRetries}})
|
||||||
|
reportRunDiagnostics(invocation.Reporter(), output.Diagnostics)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return finish(commandkit.Failure(err.Error()).WithCause(err))
|
return finish(commandkit.Failure(err.Error()).WithCause(err))
|
||||||
}
|
}
|
||||||
@@ -146,7 +157,11 @@ func pipelineRunCommand(version string, newState func() *nativeLoaders, modules
|
|||||||
return finish(commandkit.Failure(err.Error()).WithCause(err))
|
return finish(commandkit.Failure(err.Error()).WithCause(err))
|
||||||
}
|
}
|
||||||
published = true
|
published = true
|
||||||
return finish(commandkit.Success())
|
outcome := finish(commandkit.Success())
|
||||||
|
if outcome.Status != 0 {
|
||||||
|
return outcome
|
||||||
|
}
|
||||||
|
return reportRunResult(invocation, runResult{Pipeline: effective.ResolvedPipeline.ID, ArtifactCount: output.ArtifactCount(), OutputDirectory: outputDirectory, DebugDirectory: capture.Path(), FallbackCount: output.FallbackCount})
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -31,12 +31,20 @@ type runProbe struct {
|
|||||||
parsed int
|
parsed int
|
||||||
produced int
|
produced int
|
||||||
correctFirst bool
|
correctFirst bool
|
||||||
|
providerErr error
|
||||||
|
onGenerate func()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (probe *runProbe) Generate(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
func (probe *runProbe) Generate(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||||
probe.mu.Lock()
|
probe.mu.Lock()
|
||||||
defer probe.mu.Unlock()
|
defer probe.mu.Unlock()
|
||||||
probe.requests = append(probe.requests, request)
|
probe.requests = append(probe.requests, request)
|
||||||
|
if probe.onGenerate != nil {
|
||||||
|
probe.onGenerate()
|
||||||
|
}
|
||||||
|
if probe.providerErr != nil {
|
||||||
|
return nil, probe.providerErr
|
||||||
|
}
|
||||||
value := "accepted"
|
value := "accepted"
|
||||||
if request.Prompt.Messages[len(request.Prompt.Messages)-1].Content != "Assess." {
|
if request.Prompt.Messages[len(request.Prompt.Messages)-1].Content != "Assess." {
|
||||||
probe.produced++
|
probe.produced++
|
||||||
@@ -98,7 +106,9 @@ func (producer *runProducer) Extract(ctx context.Context, request contracts.Type
|
|||||||
return &value, nil, err
|
return &value, nil, err
|
||||||
}, func(context.Context, []string) (promptkit.RunRequest, error) {
|
}, func(context.Context, []string) (promptkit.RunRequest, error) {
|
||||||
return promptkit.RunRequest{PromptID: "inspect-correction", ProfileID: request.LLMProfile, SessionID: request.SessionID, OnPrepared: producer.probe.onPrepared}, nil
|
return promptkit.RunRequest{PromptID: "inspect-correction", ProfileID: request.LLMProfile, SessionID: request.SessionID, OnPrepared: producer.probe.onPrepared}, nil
|
||||||
}, nil)
|
}, func(context.Context, *inspectionArtifact) (inspectionArtifact, error) {
|
||||||
|
return inspectionArtifact{Value: "fallback"}, nil
|
||||||
|
})
|
||||||
return contracts.TypedExtractionResult[inspectionArtifact]{Value: value, Outcome: outcome}, err
|
return contracts.TypedExtractionResult[inspectionArtifact]{Value: value, Outcome: outcome}, err
|
||||||
}
|
}
|
||||||
func (producer *runProducer) Merge(_ context.Context, request contracts.TypedMergeRequest[inspectionArtifact]) (contracts.TypedMergeResult[inspectionArtifact], error) {
|
func (producer *runProducer) Merge(_ context.Context, request contracts.TypedMergeRequest[inspectionArtifact]) (contracts.TypedMergeResult[inspectionArtifact], error) {
|
||||||
|
|||||||
@@ -1,142 +1,78 @@
|
|||||||
package cli
|
package cli
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
"strings"
|
||||||
|
"text/template"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
"go.maximumdirect.net/commandkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
const runResultSchemaVersion = "notarius.run-result.v2"
|
// runResult is the small completion value. Domain files, not a saved report or
|
||||||
|
// execution ledger, are the durable deliverables.
|
||||||
type runResult struct {
|
type runResult struct {
|
||||||
SchemaVersion string `json:"schema_version"`
|
Pipeline string `json:"pipeline"`
|
||||||
RunID string `json:"run_id"`
|
ArtifactCount int `json:"artifact_count"`
|
||||||
PipelineID string `json:"pipeline_id"`
|
OutputDirectory string `json:"output_directory"`
|
||||||
OutputDirectory string `json:"output_directory"`
|
DebugDirectory string `json:"debug_directory,omitempty"`
|
||||||
IndexFile string `json:"index_file,omitempty"`
|
FallbackCount int `json:"fallback_count,omitempty"`
|
||||||
NormalizedOutputCount int `json:"normalized_output_count"`
|
|
||||||
RejectedOutputCount int `json:"rejected_output_count"`
|
|
||||||
WarningGroupCount int `json:"warning_group_count"`
|
|
||||||
WarningOccurrenceCount int `json:"warning_occurrence_count"`
|
|
||||||
DiagnosticGroupCount int `json:"diagnostic_group_count"`
|
|
||||||
DiagnosticOccurrenceCount int `json:"diagnostic_occurrence_count"`
|
|
||||||
DiagnosticsTruncated bool `json:"diagnostics_truncated"`
|
|
||||||
ValidationStatus string `json:"validation_status"`
|
|
||||||
ValidationSummaries []artifacts.ValidationSummary `json:"validation_summaries,omitempty"`
|
|
||||||
DebugDirectory string `json:"debug_directory,omitempty"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func newRunResult(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput, outputDirectory, debugDirectory string) (runResult, error) {
|
var runResultTemplate = template.Must(commandkit.ParseTextTemplate("run-completion", `{{if .FallbackCount}}Success with fallback: {{else}}Success: {{end}}{{.Pipeline}} produced {{.ArtifactCount}} artifacts in {{.OutputDirectory}}
|
||||||
diagnosticProjection, err := contracts.ProjectDiagnosticCollection(output.Diagnostics)
|
{{if .FallbackCount}}{{.FallbackCount}} operations used fallback.
|
||||||
if err != nil {
|
{{end}}{{if .DebugDirectory}}Debug capture: {{.DebugDirectory}}
|
||||||
return runResult{}, fmt.Errorf("summarize run diagnostics: %w", err)
|
{{end}}`, nil))
|
||||||
|
|
||||||
|
func reportRunResult(invocation commandkit.Invocation, result runResult) commandkit.Outcome {
|
||||||
|
message := "Could not present the completed run." + runLocations(result.OutputDirectory, result.DebugDirectory)
|
||||||
|
if err := invocation.Reporter().Report(commandkit.Report{Value: result, TextTemplate: runResultTemplate}); err != nil {
|
||||||
|
// The reporter retains render/write failures in Presentation. They are
|
||||||
|
// not a replacement operation Cause, and capture is already finalized.
|
||||||
|
return commandkit.Failure(message)
|
||||||
}
|
}
|
||||||
return newRunResultWithDiagnostics(resolved, output, outputDirectory, debugDirectory, diagnosticProjection)
|
// Deferred diagnostic delivery can still fail during native finalization.
|
||||||
|
// Supply useful locations without presenting a failure on successful runs.
|
||||||
|
return commandkit.Success().WithExplanation(message)
|
||||||
}
|
}
|
||||||
|
|
||||||
func newRunResultWithDiagnostics(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput, outputDirectory, debugDirectory string, diagnosticProjection contracts.DiagnosticProjection) (runResult, error) {
|
func runLocations(outputDirectory, debugDirectory string) string {
|
||||||
if strings.TrimSpace(output.Manifest.RunID) == "" {
|
var locations string
|
||||||
return runResult{}, fmt.Errorf("run result requires a run ID")
|
if outputDirectory != "" {
|
||||||
|
locations += "\nArtifacts: " + outputDirectory
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(resolved.ID) == "" {
|
if debugDirectory != "" {
|
||||||
return runResult{}, fmt.Errorf("run result requires a resolved pipeline ID")
|
locations += "\nDebug capture: " + debugDirectory
|
||||||
}
|
|
||||||
if strings.TrimSpace(output.Manifest.PipelineID) == "" {
|
|
||||||
return runResult{}, fmt.Errorf("run result requires a manifest pipeline ID")
|
|
||||||
}
|
|
||||||
if output.Manifest.PipelineID != resolved.ID {
|
|
||||||
return runResult{}, fmt.Errorf("run result pipeline ID does not match resolved pipeline")
|
|
||||||
}
|
|
||||||
if strings.TrimSpace(output.Manifest.ValidationStatus) == "" {
|
|
||||||
return runResult{}, fmt.Errorf("run result requires a validation status")
|
|
||||||
}
|
|
||||||
if strings.TrimSpace(outputDirectory) == "" {
|
|
||||||
return runResult{}, fmt.Errorf("run result requires an output directory")
|
|
||||||
}
|
}
|
||||||
|
return locations
|
||||||
|
}
|
||||||
|
|
||||||
absOutputDirectory, err := filepath.Abs(outputDirectory)
|
func reportRunDiagnostics(reporter *commandkit.Reporter, collection contracts.DiagnosticCollection) {
|
||||||
if err != nil {
|
for _, group := range collection.Groups {
|
||||||
return runResult{}, fmt.Errorf("make output directory absolute: %w", err)
|
origin := group.Origin
|
||||||
}
|
scope := []string{string(origin.Stage)}
|
||||||
|
for _, value := range []string{origin.StepID, origin.LaneID, origin.ModuleKey, origin.ValidatorKey} {
|
||||||
result := runResult{
|
if value != "" {
|
||||||
SchemaVersion: runResultSchemaVersion,
|
scope = append(scope, value)
|
||||||
RunID: output.Manifest.RunID,
|
|
||||||
PipelineID: output.Manifest.PipelineID,
|
|
||||||
OutputDirectory: absOutputDirectory,
|
|
||||||
NormalizedOutputCount: len(output.NormalizeOutputs),
|
|
||||||
RejectedOutputCount: len(output.Rejected),
|
|
||||||
WarningGroupCount: len(diagnosticProjection.Warnings),
|
|
||||||
WarningOccurrenceCount: diagnosticProjection.WarningOccurrenceCount,
|
|
||||||
DiagnosticGroupCount: len(diagnosticProjection.Diagnostics),
|
|
||||||
DiagnosticOccurrenceCount: diagnosticProjection.DiagnosticOccurrenceCount,
|
|
||||||
DiagnosticsTruncated: output.Diagnostics.Truncated,
|
|
||||||
ValidationStatus: output.Manifest.ValidationStatus,
|
|
||||||
ValidationSummaries: cloneValidationSummaries(output.Manifest.ValidationSummaries),
|
|
||||||
}
|
|
||||||
|
|
||||||
if strings.TrimSpace(debugDirectory) != "" {
|
|
||||||
absDebugDirectory, err := filepath.Abs(debugDirectory)
|
|
||||||
if err != nil {
|
|
||||||
return runResult{}, fmt.Errorf("make debug directory absolute: %w", err)
|
|
||||||
}
|
|
||||||
result.DebugDirectory = absDebugDirectory
|
|
||||||
}
|
|
||||||
|
|
||||||
if resolved.Output.Module == pipeline.DefaultOutputModule {
|
|
||||||
indexCount := 0
|
|
||||||
for _, file := range output.OutputFiles {
|
|
||||||
if file.Name == "index.json" {
|
|
||||||
indexCount++
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if indexCount != 1 {
|
heading := strings.Join(scope, " / ") + ": " + group.ReasonCode
|
||||||
return runResult{}, fmt.Errorf("production JSON output must contain exactly one index.json file")
|
detail := func(writer io.Writer) error {
|
||||||
|
for _, sample := range group.Samples {
|
||||||
|
if _, err := fmt.Fprintf(writer, "%s: %s\n", sample.Scope, sample.Message); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
result.IndexFile = "index.json"
|
if group.Disposition == contracts.DiagnosticDispositionWarning {
|
||||||
}
|
_ = reporter.AtEnd().Warn(heading, detail)
|
||||||
|
} else {
|
||||||
return result, nil
|
_ = reporter.AtEnd().Info(heading, detail)
|
||||||
}
|
|
||||||
|
|
||||||
func cloneValidationSummaries(summaries []artifacts.ValidationSummary) []artifacts.ValidationSummary {
|
|
||||||
if len(summaries) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
cloned := make([]artifacts.ValidationSummary, len(summaries))
|
|
||||||
for index, summary := range summaries {
|
|
||||||
cloned[index] = artifacts.CloneValidationSummary(summary)
|
|
||||||
}
|
|
||||||
return cloned
|
|
||||||
}
|
|
||||||
|
|
||||||
func encodeRunResult(result runResult) ([]byte, error) {
|
|
||||||
encoded, err := json.Marshal(result)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("encode run result: %w", err)
|
|
||||||
}
|
|
||||||
return append(encoded, '\n'), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func writeRunResult(writer io.Writer, content []byte) error {
|
|
||||||
for len(content) > 0 {
|
|
||||||
written, err := writer.Write(content)
|
|
||||||
if written < 0 || written > len(content) {
|
|
||||||
return io.ErrShortWrite
|
|
||||||
}
|
|
||||||
content = content[written:]
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if written == 0 {
|
|
||||||
return io.ErrShortWrite
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil
|
if collection.Truncated {
|
||||||
|
_ = reporter.AtEnd().Info("Additional pipeline diagnostics were omitted.")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,198 +2,318 @@ package cli
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
"go.maximumdirect.net/commandkit"
|
||||||
alwaysreject "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/validate/always_reject"
|
"go.maximumdirect.net/configkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestMaintainedMinimalInvocationEmitsRunResult(t *testing.T) {
|
func decodeCompletionReport(t *testing.T, format commandkit.Format, content []byte) map[string]any {
|
||||||
outputRoot := filepath.Join(t.TempDir(), "output")
|
|
||||||
var stdout, stderr strings.Builder
|
|
||||||
code := RunWithOptions([]string{
|
|
||||||
"run", "dnd-session",
|
|
||||||
"--config", repositoryPath("examples", "dnd-minimal.config.yml"),
|
|
||||||
"--input", repositoryPath("examples", "seriatim-minimal-transcript.json"),
|
|
||||||
"--only", "spells", "--chunk_cache", "bypass", "--output-dir", outputRoot, "--json",
|
|
||||||
}, &stdout, &stderr, productionRunOptions(t, &productionFakeLLMClient{}))
|
|
||||||
if code != 0 || stderr.Len() != 0 {
|
|
||||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
receipt := decodeRunResultDocument(t, stdout.String())
|
|
||||||
if got := receipt["schema_version"]; got != "notarius.run-result.v2" {
|
|
||||||
t.Fatalf("schema_version = %q", got)
|
|
||||||
}
|
|
||||||
if got := receipt["run_id"]; got != productionRunID {
|
|
||||||
t.Fatalf("run_id = %q", got)
|
|
||||||
}
|
|
||||||
if got := receipt["pipeline_id"]; got != "dnd-session" {
|
|
||||||
t.Fatalf("pipeline_id = %q", got)
|
|
||||||
}
|
|
||||||
if got := receipt["index_file"]; got != "index.json" {
|
|
||||||
t.Fatalf("index_file = %q", got)
|
|
||||||
}
|
|
||||||
if got := receipt["normalized_output_count"]; got != float64(1) {
|
|
||||||
t.Fatalf("normalized_output_count = %v", got)
|
|
||||||
}
|
|
||||||
if got := receipt["rejected_output_count"]; got != float64(0) {
|
|
||||||
t.Fatalf("rejected_output_count = %v", got)
|
|
||||||
}
|
|
||||||
if got := receipt["warning_group_count"]; got != float64(0) || receipt["warning_occurrence_count"] != float64(0) || receipt["diagnostic_group_count"] != float64(0) || receipt["diagnostic_occurrence_count"] != float64(0) || receipt["diagnostics_truncated"] != false {
|
|
||||||
t.Fatalf("diagnostic counts = %#v", receipt)
|
|
||||||
}
|
|
||||||
if got := receipt["validation_status"]; got != "approved" {
|
|
||||||
t.Fatalf("validation_status = %q", got)
|
|
||||||
}
|
|
||||||
|
|
||||||
outputDirectory, ok := receipt["output_directory"].(string)
|
|
||||||
if !ok || !filepath.IsAbs(outputDirectory) || outputDirectory != filepath.Join(outputRoot, productionRunID) {
|
|
||||||
t.Fatalf("output_directory = %q", receipt["output_directory"])
|
|
||||||
}
|
|
||||||
indexFile := receipt["index_file"].(string)
|
|
||||||
assertFile(t, filepath.Join(outputDirectory, indexFile))
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunResultReportsWarningsAndDebugBundle(t *testing.T) {
|
|
||||||
roots := newStateTestRoots(t)
|
|
||||||
harness := newStateTestHarness()
|
|
||||||
harness.chunkDiagnostics = []contracts.ProducerDiagnostic{stateTestDiagnostic("chunk", "contract-warning", "warning retained")}
|
|
||||||
var stdout, stderr bytes.Buffer
|
|
||||||
code := RunWithOptions([]string{
|
|
||||||
"run", "sample", "--config", roots.config, "--input", roots.input,
|
|
||||||
"--chunk_cache", "bypass", "--debug", "--json",
|
|
||||||
}, &stdout, &stderr, harness.options())
|
|
||||||
if code != 0 || !strings.Contains(stderr.String(), "1 warning group(s), 1 occurrence(s)") || strings.Contains(stderr.String(), "warnings.json") {
|
|
||||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
receipt := decodeRunResultDocument(t, stdout.String())
|
|
||||||
if got := receipt["warning_group_count"]; got != float64(1) || receipt["warning_occurrence_count"] != float64(1) || receipt["diagnostic_group_count"] != float64(0) || receipt["diagnostic_occurrence_count"] != float64(0) || receipt["diagnostics_truncated"] != false {
|
|
||||||
t.Fatalf("diagnostic counts = %#v", receipt)
|
|
||||||
}
|
|
||||||
debugDirectory, ok := receipt["debug_directory"].(string)
|
|
||||||
if !ok || !filepath.IsAbs(debugDirectory) || debugDirectory != onlyChildDir(t, roots.debug) {
|
|
||||||
t.Fatalf("debug_directory = %q", receipt["debug_directory"])
|
|
||||||
}
|
|
||||||
if strings.Contains(stdout.String(), "complete:") || strings.Contains(stdout.String(), "debug=") {
|
|
||||||
t.Fatalf("machine stdout contains human reporting: %q", stdout.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunResultReportsSuccessfulRejection(t *testing.T) {
|
|
||||||
roots := newStateTestRoots(t)
|
|
||||||
configBytes, err := os.ReadFile(roots.config)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
configBytes = []byte(replaceRequiredOnce(t, string(configBytes), " normalize:\n module: test/normalize\n", " normalize:\n module: test/normalize\n validators:\n - module: generic/always_reject\n validation_policy:\n semantic_rejection: reject_output\n"))
|
|
||||||
if err := os.WriteFile(roots.config, configBytes, 0o600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
harness := newStateTestHarness()
|
|
||||||
opts := harness.options()
|
|
||||||
if err := alwaysreject.RegisterTyped[stateTestArtifact](opts.Registries.Validators, stateTestArtifactKind); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
opts.Catalog = catalogFromRegistries(opts.Registries)
|
|
||||||
|
|
||||||
var stdout, stderr bytes.Buffer
|
|
||||||
code := RunWithOptions([]string{
|
|
||||||
"run", "sample", "--config", roots.config, "--input", roots.input,
|
|
||||||
"--chunk_cache", "bypass", "--json",
|
|
||||||
}, &stdout, &stderr, opts)
|
|
||||||
if code != 0 || stderr.Len() != 0 {
|
|
||||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
||||||
}
|
|
||||||
receipt := decodeRunResultDocument(t, stdout.String())
|
|
||||||
if got := receipt["normalized_output_count"]; got != float64(0) {
|
|
||||||
t.Fatalf("normalized_output_count = %v", got)
|
|
||||||
}
|
|
||||||
if got := receipt["rejected_output_count"]; got != float64(1) {
|
|
||||||
t.Fatalf("rejected_output_count = %v", got)
|
|
||||||
}
|
|
||||||
if got := receipt["validation_status"]; got != "rejected" {
|
|
||||||
t.Fatalf("validation_status = %q", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunResultIsAbsentForSyntaxAndRuntimeFailures(t *testing.T) {
|
|
||||||
t.Run("syntax", func(t *testing.T) {
|
|
||||||
var stdout, stderr bytes.Buffer
|
|
||||||
code := RunWithOptions([]string{"run", "sample", "--json"}, &stdout, &stderr, newStateTestHarness().options())
|
|
||||||
if code != 2 || stdout.Len() != 0 || stderr.Len() == 0 {
|
|
||||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("runtime", func(t *testing.T) {
|
|
||||||
roots := newStateTestRoots(t)
|
|
||||||
harness := newStateTestHarness()
|
|
||||||
harness.extractErr = errors.New("injected extraction failure")
|
|
||||||
var stdout, stderr bytes.Buffer
|
|
||||||
code := RunWithOptions([]string{
|
|
||||||
"run", "sample", "--config", roots.config, "--input", roots.input,
|
|
||||||
"--chunk_cache", "bypass", "--json",
|
|
||||||
}, &stdout, &stderr, harness.options())
|
|
||||||
if code != 1 || stdout.Len() != 0 || stderr.Len() == 0 {
|
|
||||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunResultDeliveryFailureRetainsPublishedBundles(t *testing.T) {
|
|
||||||
roots := newStateTestRoots(t)
|
|
||||||
writerErr := errors.New("result writer sentinel")
|
|
||||||
stdout := &resultDeliveryWriter{err: writerErr}
|
|
||||||
var stderr bytes.Buffer
|
|
||||||
code := RunWithOptions([]string{
|
|
||||||
"run", "sample", "--config", roots.config, "--input", roots.input,
|
|
||||||
"--chunk_cache", "bypass", "--debug", "--json",
|
|
||||||
}, stdout, &stderr, newStateTestHarness().options())
|
|
||||||
if code != 1 || !strings.Contains(stderr.String(), "write run result") || strings.Contains(stderr.String(), writerErr.Error()) {
|
|
||||||
t.Fatalf("code=%d stderr=%q", code, stderr.String())
|
|
||||||
}
|
|
||||||
if stdout.accepted.Len() != 0 {
|
|
||||||
t.Fatalf("accepted stdout = %q", stdout.accepted.String())
|
|
||||||
}
|
|
||||||
assertStateTestOutput(t, roots.output)
|
|
||||||
debugBundle := onlyChildDir(t, roots.debug)
|
|
||||||
report := readStateTestRunReport(t, debugBundle)
|
|
||||||
if !report.Succeeded {
|
|
||||||
t.Fatalf("debug report = %#v, want successful persisted run", report)
|
|
||||||
}
|
|
||||||
if strings.Contains(readAllFiles(t, debugBundle), writerErr.Error()) {
|
|
||||||
t.Fatalf("debug bundle contains result writer error")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func decodeRunResultDocument(t *testing.T, stdout string) map[string]any {
|
|
||||||
t.Helper()
|
t.Helper()
|
||||||
if strings.Count(stdout, "\n") != 1 {
|
if format == commandkit.FormatYAML {
|
||||||
t.Fatalf("stdout = %q, want one JSON document", stdout)
|
value, err := commandkit.DecodeYAMLReport(bytes.NewReader(content))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("decode native YAML report: %v\n%s", err, content)
|
||||||
|
}
|
||||||
|
content, err = json.Marshal(value)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
var receipt map[string]any
|
var report map[string]any
|
||||||
if err := json.Unmarshal([]byte(stdout), &receipt); err != nil {
|
if err := json.Unmarshal(content, &report); err != nil {
|
||||||
t.Fatalf("decode run result: %v; stdout=%q", err, stdout)
|
t.Fatalf("decode native report: %v\n%s", err, content)
|
||||||
}
|
}
|
||||||
return receipt
|
return report
|
||||||
}
|
}
|
||||||
|
|
||||||
type resultDeliveryWriter struct {
|
func assertCompletedRunFiles(t *testing.T, file string, captured bool) {
|
||||||
err error
|
t.Helper()
|
||||||
accepted bytes.Buffer
|
run := filepath.Join(filepath.Dir(file), "output-must-not-exist", "run-1")
|
||||||
|
if content, err := os.ReadFile(filepath.Join(run, "lanes", "notes.json")); err != nil || len(content) == 0 {
|
||||||
|
t.Fatalf("published domain artifact: %s %v", content, err)
|
||||||
|
}
|
||||||
|
files, err := os.ReadDir(run)
|
||||||
|
if err != nil || len(files) != 1 || files[0].Name() != "lanes" {
|
||||||
|
t.Fatalf("unexpected reporting sidecars: %#v %v", files, err)
|
||||||
|
}
|
||||||
|
if captured {
|
||||||
|
debug := filepath.Join(filepath.Dir(file), "debug-must-not-exist", "run-1")
|
||||||
|
content, err := os.ReadFile(filepath.Join(debug, "outcome.json"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var terminal struct {
|
||||||
|
Outcome struct {
|
||||||
|
Published bool `json:"published"`
|
||||||
|
} `json:"outcome"`
|
||||||
|
Error string `json:"error"`
|
||||||
|
CaptureError string `json:"capture_error"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(content, &terminal); err != nil || !terminal.Outcome.Published || terminal.Error != "" || terminal.CaptureError != "" {
|
||||||
|
t.Fatalf("capture was not completed before reporting: %s %v", content, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (w *resultDeliveryWriter) Write(content []byte) (int, error) {
|
type completedRunWriter struct {
|
||||||
if w.err != nil {
|
t *testing.T
|
||||||
return 0, w.err
|
file string
|
||||||
}
|
captured bool
|
||||||
return w.accepted.Write(content)
|
content bytes.Buffer
|
||||||
|
}
|
||||||
|
|
||||||
|
func (writer *completedRunWriter) Write(content []byte) (int, error) {
|
||||||
|
assertCompletedRunFiles(writer.t, writer.file, writer.captured)
|
||||||
|
return writer.content.Write(content)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNativeRunCompletionFormatsFollowPublicationAndCapture(t *testing.T) {
|
||||||
|
for _, format := range []commandkit.Format{commandkit.FormatText, commandkit.FormatJSON, commandkit.FormatYAML} {
|
||||||
|
t.Run(string(format), func(t *testing.T) {
|
||||||
|
probe := &runProbe{}
|
||||||
|
app, file, input, _ := runFixture(t, probe)
|
||||||
|
output := &completedRunWriter{t: t, file: file, captured: true}
|
||||||
|
var diagnostics bytes.Buffer
|
||||||
|
outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--format", string(format)}, commandkit.Streams{Out: output, Err: &diagnostics})
|
||||||
|
if outcome.Status != 0 || !strings.Contains(diagnostics.String(), "Running pipeline") {
|
||||||
|
t.Fatalf("completion: %#v stderr=%s", outcome, diagnostics.String())
|
||||||
|
}
|
||||||
|
directory := filepath.Join(filepath.Dir(file), "output-must-not-exist", "run-1")
|
||||||
|
debug := filepath.Join(filepath.Dir(file), "debug-must-not-exist", "run-1")
|
||||||
|
if format == commandkit.FormatText {
|
||||||
|
if !strings.Contains(output.content.String(), "Success: demo produced 1 artifacts in "+directory) || !strings.Contains(output.content.String(), "Debug capture: "+debug) || strings.Contains(output.content.String(), "fallback") {
|
||||||
|
t.Fatalf("native completion text: %s", output.content.String())
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
report := decodeCompletionReport(t, format, output.content.Bytes())
|
||||||
|
if len(report) != 4 || report["pipeline"] != "demo" || report["artifact_count"] != float64(1) || report["output_directory"] != directory || report["debug_directory"] != debug {
|
||||||
|
t.Fatalf("native report: %#v", report)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if probe.produced != 1 {
|
||||||
|
t.Fatalf("reporting reran a producer: %d", probe.produced)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNativeRunDisclosesFallbackIndependentlyOfDiagnosticVisibility(t *testing.T) {
|
||||||
|
for _, format := range []commandkit.Format{commandkit.FormatText, commandkit.FormatJSON, commandkit.FormatYAML} {
|
||||||
|
for _, verbose := range []bool{false, true} {
|
||||||
|
probe := &runProbe{correctFirst: true}
|
||||||
|
app, file, input, _ := runFixture(t, probe)
|
||||||
|
args := []string{"pipeline", "run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--set", `/pipelines/demo/mode="permissive"`, "--set", "/pipelines/demo/max_additional_turns=0", "--format", string(format)}
|
||||||
|
if verbose {
|
||||||
|
args = append(args, "--verbose")
|
||||||
|
}
|
||||||
|
var output, diagnostics bytes.Buffer
|
||||||
|
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: &diagnostics})
|
||||||
|
if outcome.Status != 0 || probe.produced != 1 {
|
||||||
|
t.Fatalf("permissive completion: %#v %s", outcome, diagnostics.String())
|
||||||
|
}
|
||||||
|
if verbose && !strings.Contains(diagnostics.String(), "WARN:") || !verbose && diagnostics.Len() != 0 {
|
||||||
|
t.Fatalf("native warning visibility: %s", diagnostics.String())
|
||||||
|
}
|
||||||
|
if format == commandkit.FormatText {
|
||||||
|
if !strings.Contains(output.String(), "Success with fallback:") || !strings.Contains(output.String(), "1 operations used fallback.") {
|
||||||
|
t.Fatalf("fallback text: %s", output.String())
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
report := decodeCompletionReport(t, format, output.Bytes())
|
||||||
|
if report["fallback_count"] != float64(1) || report["artifact_count"] != float64(1) || len(report) != 4 {
|
||||||
|
t.Fatalf("fallback report: %#v", report)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assertCompletedRunFiles(t, file, false)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNativeRunQuietAndDebugUseNativeModeOrdering(t *testing.T) {
|
||||||
|
for _, test := range []struct {
|
||||||
|
flags []string
|
||||||
|
quiet bool
|
||||||
|
progress bool
|
||||||
|
}{
|
||||||
|
{[]string{"--debug", "--quiet"}, true, false},
|
||||||
|
{[]string{"--quiet", "--debug"}, false, true},
|
||||||
|
{[]string{"--debug", "--progress=false"}, false, false},
|
||||||
|
} {
|
||||||
|
probe := &runProbe{}
|
||||||
|
app, file, input, _ := runFixture(t, probe)
|
||||||
|
var output, diagnostics bytes.Buffer
|
||||||
|
args := append([]string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--format", "json"}, test.flags...)
|
||||||
|
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: &diagnostics})
|
||||||
|
if outcome.Status != 0 || test.quiet != (output.Len() == 0) || test.progress != strings.Contains(diagnostics.String(), "Running pipeline") {
|
||||||
|
t.Fatalf("native mode order: %#v output=%s stderr=%s", outcome, output.String(), diagnostics.String())
|
||||||
|
}
|
||||||
|
assertCompletedRunFiles(t, file, true)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNativeRunCancellationKeepsItsCauseAndPublishesNothing(t *testing.T) {
|
||||||
|
probe := &runProbe{}
|
||||||
|
app, file, input, _ := runFixture(t, probe)
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
probe.onGenerate = cancel
|
||||||
|
var output, diagnostics bytes.Buffer
|
||||||
|
outcome := app.Execute(ctx, []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--quiet"}, commandkit.Streams{Out: &output, Err: &diagnostics})
|
||||||
|
if outcome.Status != 1 || !errors.Is(outcome.Cause, context.Canceled) || output.Len() != 0 || diagnostics.Len() == 0 {
|
||||||
|
t.Fatalf("cancellation: %#v stdout=%s stderr=%s", outcome, output.String(), diagnostics.String())
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(filepath.Dir(file), "output-must-not-exist")); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("cancellation published artifacts: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNativeRunPrimaryFailureSurvivesCaptureAndDiagnosticFailures(t *testing.T) {
|
||||||
|
primary := errors.New("provider failure sentinel")
|
||||||
|
delivery := errors.New("diagnostic failure sentinel")
|
||||||
|
probe := &runProbe{providerErr: primary}
|
||||||
|
app, file, input, _ := runFixture(t, probe)
|
||||||
|
debug := filepath.Join(filepath.Dir(file), "debug-must-not-exist", "run-1")
|
||||||
|
probe.onGenerate = func() {
|
||||||
|
if err := os.Mkdir(filepath.Join(debug, "outcome.json"), 0700); err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var output bytes.Buffer
|
||||||
|
outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--llm-transient-retries=0"}, commandkit.Streams{Out: &output, Err: failingResultWriter{cause: delivery}})
|
||||||
|
var captureFailure *os.LinkError
|
||||||
|
if outcome.Status != 1 || !errors.Is(outcome.Cause, primary) || !errors.As(outcome.Cause, &captureFailure) || errors.Is(outcome.Cause, delivery) || !errors.Is(outcome.Presentation.Diagnostics.WriteError, delivery) || output.Len() != 0 || len(probe.requests) != 1 {
|
||||||
|
t.Fatalf("primary/capture/delivery failure: %#v", outcome)
|
||||||
|
}
|
||||||
|
if !strings.Contains(outcome.Explanation.Message, debug) {
|
||||||
|
t.Fatalf("failed run capture location unavailable: %#v", outcome)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNativeRunReportDeliveryFailuresRetainArtifactsAndNativePresentation(t *testing.T) {
|
||||||
|
sentinel := errors.New("result delivery sentinel")
|
||||||
|
for _, partial := range []bool{false, true} {
|
||||||
|
probe := &runProbe{}
|
||||||
|
app, file, input, _ := runFixture(t, probe)
|
||||||
|
var output io.Writer = failingResultWriter{cause: sentinel}
|
||||||
|
prefix := &partialResultWriter{cause: sentinel}
|
||||||
|
if partial {
|
||||||
|
output = prefix
|
||||||
|
}
|
||||||
|
var diagnostics bytes.Buffer
|
||||||
|
outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--format", "json"}, commandkit.Streams{Out: output, Err: &diagnostics})
|
||||||
|
if outcome.Status != 1 || outcome.Cause != nil || !errors.Is(outcome.Presentation.Output.WriteError, sentinel) || outcome.Presentation.Output.State != commandkit.DeliveryIncomplete || probe.produced != 1 || partial && prefix.content.Len() == 0 {
|
||||||
|
t.Fatalf("delivery failure: %#v %s", outcome, diagnostics.String())
|
||||||
|
}
|
||||||
|
assertCompletedRunFiles(t, file, true)
|
||||||
|
for _, root := range []string{"output-must-not-exist", "debug-must-not-exist"} {
|
||||||
|
if !strings.Contains(diagnostics.String(), filepath.Join(filepath.Dir(file), root, "run-1")) {
|
||||||
|
t.Fatalf("missing available path after delivery failure: %s", diagnostics.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNativeRunDiagnosticDeliveryFailureIsNotAnOperationFailure(t *testing.T) {
|
||||||
|
sentinel := errors.New("diagnostic delivery sentinel")
|
||||||
|
probe := &runProbe{correctFirst: true}
|
||||||
|
app, file, input, _ := runFixture(t, probe)
|
||||||
|
var output bytes.Buffer
|
||||||
|
outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--set", `/pipelines/demo/mode="permissive"`, "--set", "/pipelines/demo/max_additional_turns=0", "--format", "json", "--verbose"}, commandkit.Streams{Out: &output, Err: failingResultWriter{cause: sentinel}})
|
||||||
|
if outcome.Status != 1 || outcome.Cause != nil || !errors.Is(outcome.Presentation.Diagnostics.WriteError, sentinel) || outcome.Presentation.Diagnostics.State != commandkit.DeliveryIncomplete {
|
||||||
|
t.Fatalf("diagnostic delivery: %#v", outcome)
|
||||||
|
}
|
||||||
|
if report := decodeCompletionReport(t, commandkit.FormatJSON, output.Bytes()); report["fallback_count"] != float64(1) {
|
||||||
|
t.Fatalf("completed report was replaced by a delivery envelope: %#v", report)
|
||||||
|
}
|
||||||
|
if !strings.Contains(outcome.Explanation.Message, filepath.Join(filepath.Dir(file), "output-must-not-exist", "run-1")) {
|
||||||
|
t.Fatalf("presentation failure lost available artifacts: %#v", outcome)
|
||||||
|
}
|
||||||
|
assertCompletedRunFiles(t, file, false)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNativeRunCaptureFailureAfterPublicationDoesNotReportSuccess(t *testing.T) {
|
||||||
|
probe := &runProbe{}
|
||||||
|
app, file, input, _ := runFixture(t, probe)
|
||||||
|
var once sync.Once
|
||||||
|
debug := filepath.Join(filepath.Dir(file), "debug-must-not-exist", "run-1")
|
||||||
|
probe.onGenerate = func() {
|
||||||
|
once.Do(func() {
|
||||||
|
// The external provider boundary runs after capture allocation but
|
||||||
|
// before terminalization. Make only the terminal file unusable.
|
||||||
|
if err := os.Mkdir(filepath.Join(debug, "outcome.json"), 0700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
var output, diagnostics bytes.Buffer
|
||||||
|
outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug", "--format", "json"}, commandkit.Streams{Out: &output, Err: &diagnostics})
|
||||||
|
if outcome.Status != 1 || outcome.Cause == nil || output.Len() != 0 || probe.produced != 1 {
|
||||||
|
t.Fatalf("post-publication capture failure: %#v stdout=%s stderr=%s", outcome, output.String(), diagnostics.String())
|
||||||
|
}
|
||||||
|
assertCompletedRunFiles(t, file, false)
|
||||||
|
if !strings.Contains(diagnostics.String(), debug) || !strings.Contains(diagnostics.String(), filepath.Join(filepath.Dir(file), "output-must-not-exist", "run-1")) {
|
||||||
|
t.Fatalf("post-publication locations: %s", diagnostics.String())
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(debug, "executions.json")); err != nil {
|
||||||
|
t.Fatalf("existing capture was removed: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNativeRunFailureCategoriesPreserveCausesWithoutSuccessReports(t *testing.T) {
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
configure func(*testing.T, *runProbe, string, string) []string
|
||||||
|
status int
|
||||||
|
cause error
|
||||||
|
}{
|
||||||
|
{"missing configuration", func(t *testing.T, _ *runProbe, _, input string) []string {
|
||||||
|
return []string{"run", "demo", "--input-file", input, "--config", filepath.Join(t.TempDir(), "missing.yml")}
|
||||||
|
}, 1, nil},
|
||||||
|
{"invalid configuration", func(t *testing.T, _ *runProbe, file, input string) []string {
|
||||||
|
return []string{"run", "demo", "--config", file, "--input-file", input, "--set", "/version=5"}
|
||||||
|
}, 2, nil},
|
||||||
|
{"unknown pipeline", func(_ *testing.T, _ *runProbe, file, input string) []string {
|
||||||
|
return []string{"run", "missing", "--config", file, "--input-file", input}
|
||||||
|
}, 2, nil},
|
||||||
|
{"missing input", func(t *testing.T, _ *runProbe, file, _ string) []string {
|
||||||
|
return []string{"run", "demo", "--config", file, "--input-file", filepath.Join(t.TempDir(), "missing.txt")}
|
||||||
|
}, 1, os.ErrNotExist},
|
||||||
|
{"publication failure", func(t *testing.T, _ *runProbe, file, input string) []string {
|
||||||
|
path := filepath.Join(t.TempDir(), "occupied")
|
||||||
|
if err := os.WriteFile(path, []byte("keep"), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--output-dir", path}
|
||||||
|
}, 1, nil},
|
||||||
|
{"provider failure", func(_ *testing.T, probe *runProbe, file, input string) []string {
|
||||||
|
probe.providerErr = io.ErrUnexpectedEOF
|
||||||
|
return []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--llm-transient-retries=0"}
|
||||||
|
}, 1, io.ErrUnexpectedEOF},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
probe := &runProbe{}
|
||||||
|
app, file, input, _ := runFixture(t, probe)
|
||||||
|
var output, diagnostics bytes.Buffer
|
||||||
|
args := append(test.configure(t, probe, file, input), "--quiet", "--format", "json")
|
||||||
|
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: &diagnostics})
|
||||||
|
if outcome.Status != test.status || test.cause != nil && !errors.Is(outcome.Cause, test.cause) || output.Len() != 0 || diagnostics.Len() == 0 {
|
||||||
|
t.Fatalf("failure category: %#v stdout=%s stderr=%s", outcome, output.String(), diagnostics.String())
|
||||||
|
}
|
||||||
|
if test.name == "missing configuration" || test.name == "invalid configuration" {
|
||||||
|
var load *configkit.LoadError
|
||||||
|
if !errors.As(outcome.Cause, &load) {
|
||||||
|
t.Fatalf("lost native load cause: %#v", outcome)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,229 +2,136 @@ package cli
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"encoding/json"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"text/template"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
"go.maximumdirect.net/commandkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestRunResultEncodesRequiredFieldsAndCounts(t *testing.T) {
|
func TestRunResultUsesOneNativeValueAcrossFormats(t *testing.T) {
|
||||||
result, err := newRunResult(testResolvedPipeline(pipeline.DefaultOutputModule), testRunOutput(), "relative-output", "relative-debug")
|
value := runResult{Pipeline: "demo", ArtifactCount: 3, OutputDirectory: "/artifacts/run-one", DebugDirectory: "/debug/run-one", FallbackCount: 2}
|
||||||
if err != nil {
|
for _, format := range []commandkit.Format{commandkit.FormatText, commandkit.FormatJSON, commandkit.FormatYAML} {
|
||||||
t.Fatal(err)
|
var output bytes.Buffer
|
||||||
}
|
if err := commandkit.RenderReport(&output, commandkit.Report{Value: value, TextTemplate: runResultTemplate}, format); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
encoded, err := encodeRunResult(result)
|
}
|
||||||
if err != nil {
|
if format == commandkit.FormatText {
|
||||||
t.Fatal(err)
|
if !strings.Contains(output.String(), "Success with fallback: demo produced 3 artifacts in /artifacts/run-one") || !strings.Contains(output.String(), "2 operations used fallback.") || !strings.Contains(output.String(), "Debug capture: /debug/run-one") {
|
||||||
}
|
t.Fatalf("fallback text: %s", output.String())
|
||||||
if encoded[len(encoded)-1] != '\n' || bytes.Count(encoded, []byte{'\n'}) != 1 {
|
|
||||||
t.Fatalf("encoded result is not one newline-terminated object: %q", encoded)
|
|
||||||
}
|
|
||||||
|
|
||||||
var decoded map[string]any
|
|
||||||
if err := json.Unmarshal(encoded, &decoded); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if got := decoded["schema_version"]; got != runResultSchemaVersion {
|
|
||||||
t.Fatalf("schema_version = %q", got)
|
|
||||||
}
|
|
||||||
if got := decoded["run_id"]; got != "run-123" {
|
|
||||||
t.Fatalf("run_id = %q", got)
|
|
||||||
}
|
|
||||||
if got := decoded["pipeline_id"]; got != "sample" {
|
|
||||||
t.Fatalf("pipeline_id = %q", got)
|
|
||||||
}
|
|
||||||
if got := decoded["validation_status"]; got != "rejected" {
|
|
||||||
t.Fatalf("validation_status = %q", got)
|
|
||||||
}
|
|
||||||
if got := decoded["index_file"]; got != "index.json" {
|
|
||||||
t.Fatalf("index_file = %q", got)
|
|
||||||
}
|
|
||||||
if got := decoded["normalized_output_count"]; got != float64(2) {
|
|
||||||
t.Fatalf("normalized_output_count = %v", got)
|
|
||||||
}
|
|
||||||
if got := decoded["rejected_output_count"]; got != float64(1) {
|
|
||||||
t.Fatalf("rejected_output_count = %v", got)
|
|
||||||
}
|
|
||||||
if got := decoded["warning_group_count"]; got != float64(1) || decoded["warning_occurrence_count"] != float64(1) || decoded["diagnostic_group_count"] != float64(0) || decoded["diagnostic_occurrence_count"] != float64(0) || decoded["diagnostics_truncated"] != false {
|
|
||||||
t.Fatalf("diagnostic counts = %#v", decoded)
|
|
||||||
}
|
|
||||||
if got := decoded["validation_summaries"]; got != nil {
|
|
||||||
t.Fatalf("validation_summaries = %#v, want omitted when empty", got)
|
|
||||||
}
|
|
||||||
if got := decoded["output_directory"]; got != filepath.Join(mustWorkingDirectory(t), "relative-output") {
|
|
||||||
t.Fatalf("output_directory = %q", got)
|
|
||||||
}
|
|
||||||
if got := decoded["debug_directory"]; got != filepath.Join(mustWorkingDirectory(t), "relative-debug") {
|
|
||||||
t.Fatalf("debug_directory = %q", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunResultRejectsInvalidRequiredValues(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
resolved pipeline.ResolvedPipeline
|
|
||||||
output pipeline.RunOutput
|
|
||||||
directory string
|
|
||||||
}{
|
|
||||||
{name: "blank run ID", resolved: testResolvedPipeline(pipeline.DefaultOutputModule), output: testRunOutputWithout(func(output *pipeline.RunOutput) { output.Manifest.RunID = " " }), directory: "output"},
|
|
||||||
{name: "blank resolved pipeline ID", resolved: pipeline.ResolvedPipeline{Output: pipeline.ModuleBinding{Module: pipeline.DefaultOutputModule}}, output: testRunOutput(), directory: "output"},
|
|
||||||
{name: "blank manifest pipeline ID", resolved: testResolvedPipeline(pipeline.DefaultOutputModule), output: testRunOutputWithout(func(output *pipeline.RunOutput) { output.Manifest.PipelineID = "" }), directory: "output"},
|
|
||||||
{name: "mismatched pipeline IDs", resolved: testResolvedPipeline(pipeline.DefaultOutputModule), output: testRunOutputWithout(func(output *pipeline.RunOutput) { output.Manifest.PipelineID = "other" }), directory: "output"},
|
|
||||||
{name: "blank validation status", resolved: testResolvedPipeline(pipeline.DefaultOutputModule), output: testRunOutputWithout(func(output *pipeline.RunOutput) { output.Manifest.ValidationStatus = " " }), directory: "output"},
|
|
||||||
{name: "blank output directory", resolved: testResolvedPipeline(pipeline.DefaultOutputModule), output: testRunOutput(), directory: " "},
|
|
||||||
}
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
if _, err := newRunResult(tt.resolved, tt.output, tt.directory, ""); err == nil {
|
|
||||||
t.Fatal("newRunResult() succeeded")
|
|
||||||
}
|
}
|
||||||
})
|
continue
|
||||||
|
}
|
||||||
|
report := decodeCompletionReport(t, format, output.Bytes())
|
||||||
|
if report["pipeline"] != "demo" || report["output_directory"] != value.OutputDirectory || report["debug_directory"] != value.DebugDirectory || report["fallback_count"] != float64(2) || report["artifact_count"] != float64(3) || len(report) != 5 {
|
||||||
|
t.Fatalf("completion value: %#v", report)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunResultOmitsIndexFileForOtherOutputModules(t *testing.T) {
|
func TestNativeCompletionDiagnosticsRetainDomainClassification(t *testing.T) {
|
||||||
result, err := newRunResult(testResolvedPipeline("test/output"), testRunOutputWithout(func(output *pipeline.RunOutput) {
|
collection := contracts.DiagnosticCollection{}
|
||||||
output.OutputFiles = nil
|
for _, disposition := range []contracts.DiagnosticDisposition{contracts.DiagnosticDispositionWarning, contracts.DiagnosticDispositionAdvisory, contracts.DiagnosticDispositionObservation} {
|
||||||
}), "output", "")
|
category := contracts.DiagnosticCategoryDataQuality
|
||||||
|
if disposition == contracts.DiagnosticDispositionWarning {
|
||||||
|
category = contracts.DiagnosticCategoryFallback
|
||||||
|
} else if disposition == contracts.DiagnosticDispositionObservation {
|
||||||
|
category = contracts.DiagnosticCategoryNormalization
|
||||||
|
}
|
||||||
|
collection.Groups = append(collection.Groups, contracts.DiagnosticGroup{Disposition: disposition, Category: category, ReasonCode: string(disposition), Origin: contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract, StepID: "build", LaneID: "notes", ModuleKey: "producer"}, OccurrenceCount: 1, Samples: []contracts.DiagnosticSample{{Scope: "current source", Message: string(disposition) + " finding"}}})
|
||||||
|
}
|
||||||
|
if err := collection.Validate(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var output, diagnostics bytes.Buffer
|
||||||
|
app, err := commandkit.New(commandkit.Identity{Name: "fixture"}, commandkit.NewCommand(commandkit.CommandSpec[struct{}]{Name: "report", Run: func(invocation commandkit.Invocation, _ struct{}) commandkit.Outcome {
|
||||||
|
reportRunDiagnostics(invocation.Reporter(), collection)
|
||||||
|
return reportRunResult(invocation, runResult{Pipeline: "demo", ArtifactCount: 1, OutputDirectory: "/artifacts/run-one"})
|
||||||
|
}}))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if result.IndexFile != "" {
|
outcome := app.Execute(context.Background(), []string{"report", "--verbose"}, commandkit.Streams{Out: &output, Err: &diagnostics})
|
||||||
t.Fatalf("index_file = %q", result.IndexFile)
|
if outcome.Status != 0 || strings.Contains(output.String(), "finding") || strings.Count(diagnostics.String(), "WARN:") != 1 {
|
||||||
|
t.Fatalf("diagnostic classification: %#v stdout=%s stderr=%s", outcome, output.String(), diagnostics.String())
|
||||||
}
|
}
|
||||||
encoded, err := encodeRunResult(result)
|
for _, group := range collection.Groups {
|
||||||
if err != nil {
|
if !strings.Contains(diagnostics.String(), string(group.Disposition)+" finding") {
|
||||||
t.Fatal(err)
|
t.Fatalf("missing diagnostic: %s", diagnostics.String())
|
||||||
}
|
}
|
||||||
var decoded map[string]any
|
|
||||||
if err := json.Unmarshal(encoded, &decoded); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, ok := decoded["index_file"]; ok {
|
|
||||||
t.Fatalf("encoded non-JSON result contains index_file: %s", encoded)
|
|
||||||
}
|
|
||||||
if _, ok := decoded["debug_directory"]; ok {
|
|
||||||
t.Fatalf("encoded result without debug capture contains debug_directory: %s", encoded)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunResultProjectsOwnedValidationSummaries(t *testing.T) {
|
func TestCompletedRunRenderingFailureRetainsPublicationEvenWhenQuiet(t *testing.T) {
|
||||||
output := testRunOutput()
|
previous := runResultTemplate
|
||||||
output.Manifest.ValidationSummaries = []artifacts.ValidationSummary{{Status: "incomplete", IncompleteValidators: []string{"validator"}, ProducerAttemptCount: 1, TerminalAction: "warn_continue"}}
|
t.Cleanup(func() { runResultTemplate = previous })
|
||||||
result, err := newRunResult(testResolvedPipeline(pipeline.DefaultOutputModule), output, "output", "")
|
// Simulate a broken application-owned text template, not a new renderer seam.
|
||||||
if err != nil {
|
runResultTemplate = template.Must(commandkit.ParseTextTemplate("invalid", "{{.MissingField}}", nil))
|
||||||
t.Fatal(err)
|
for _, quiet := range []bool{false, true} {
|
||||||
}
|
probe := &runProbe{}
|
||||||
output.Manifest.ValidationSummaries[0].IncompleteValidators[0] = "caller mutation"
|
app, file, input, _ := runFixture(t, probe)
|
||||||
if got := result.ValidationSummaries[0].IncompleteValidators; len(got) != 1 || got[0] != "validator" {
|
args := []string{"run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--debug"}
|
||||||
t.Fatalf("result validation summaries = %#v", result.ValidationSummaries)
|
if quiet {
|
||||||
}
|
args = append(args, "--quiet")
|
||||||
encoded, err := encodeRunResult(result)
|
}
|
||||||
if err != nil {
|
var output, diagnostics bytes.Buffer
|
||||||
t.Fatal(err)
|
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: &diagnostics})
|
||||||
}
|
if outcome.Status != 1 || outcome.Cause != nil || outcome.Presentation.Output.RenderError == nil || outcome.Presentation.Output.State != commandkit.DeliveryIncomplete || probe.produced != 1 {
|
||||||
if !bytes.Contains(encoded, []byte(`"validation_summaries":[{"status":"incomplete","incomplete_validators":["validator"],"producer_attempt_count":1,"terminal_action":"warn_continue"}]`)) {
|
t.Fatalf("render failure: %#v %s", outcome, diagnostics.String())
|
||||||
t.Fatalf("encoded result = %s", encoded)
|
}
|
||||||
|
assertCompletedRunFiles(t, file, true)
|
||||||
|
artifacts := filepath.Join(filepath.Dir(file), "output-must-not-exist", "run-1")
|
||||||
|
if !strings.Contains(diagnostics.String(), artifacts) || (quiet && output.Len() != 0) {
|
||||||
|
t.Fatalf("render failure locations/quiet: %s", diagnostics.String())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunResultRequiresOneProductionIndexFile(t *testing.T) {
|
type failingResultWriter struct {
|
||||||
tests := []struct {
|
cause error
|
||||||
name string
|
|
||||||
files []contracts.OutputFile
|
|
||||||
}{
|
|
||||||
{name: "missing", files: nil},
|
|
||||||
{name: "duplicate", files: []contracts.OutputFile{{Name: "index.json"}, {Name: "index.json"}}},
|
|
||||||
}
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
output := testRunOutput()
|
|
||||||
output.OutputFiles = tt.files
|
|
||||||
if _, err := newRunResult(testResolvedPipeline(pipeline.DefaultOutputModule), output, "output", ""); err == nil {
|
|
||||||
t.Fatal("newRunResult() succeeded")
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestWriteRunResultCompletesAndReportsWriterFailure(t *testing.T) {
|
func (writer failingResultWriter) Write([]byte) (int, error) { return 0, writer.cause }
|
||||||
content := []byte("result\n")
|
|
||||||
var target bytes.Buffer
|
|
||||||
if err := writeRunResult(partialResultWriter{writer: &target, limit: 2}, content); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if got := target.String(); got != string(content) {
|
|
||||||
t.Fatalf("written result = %q", got)
|
|
||||||
}
|
|
||||||
|
|
||||||
writerErr := errors.New("result writer failed")
|
|
||||||
if err := writeRunResult(failingResultWriter{err: writerErr}, content); !errors.Is(err, writerErr) {
|
|
||||||
t.Fatalf("writeRunResult() error = %v", err)
|
|
||||||
}
|
|
||||||
if err := writeRunResult(zeroResultWriter{}, content); !errors.Is(err, io.ErrShortWrite) {
|
|
||||||
t.Fatalf("zero-progress error = %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func testResolvedPipeline(outputModule string) pipeline.ResolvedPipeline {
|
|
||||||
return pipeline.ResolvedPipeline{ID: "sample", Output: pipeline.ModuleBinding{Module: outputModule}}
|
|
||||||
}
|
|
||||||
|
|
||||||
func testRunOutput() pipeline.RunOutput {
|
|
||||||
return pipeline.RunOutput{
|
|
||||||
Manifest: artifacts.RunManifest{RunID: "run-123", PipelineID: "sample", ValidationStatus: "rejected"},
|
|
||||||
NormalizeOutputs: []contracts.SerializedOutput{{}, {}},
|
|
||||||
Rejected: []contracts.RejectedOutput{{}},
|
|
||||||
Diagnostics: contracts.DiagnosticCollection{Groups: []contracts.DiagnosticGroup{{
|
|
||||||
Disposition: contracts.DiagnosticDispositionWarning,
|
|
||||||
Category: contracts.DiagnosticCategoryFallback,
|
|
||||||
ReasonCode: "fallback",
|
|
||||||
Origin: contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageNormalize, StepID: "step", LaneID: "lane", ModuleKey: "module"},
|
|
||||||
OccurrenceCount: 1,
|
|
||||||
Samples: []contracts.DiagnosticSample{{Scope: "scope", Message: "message"}},
|
|
||||||
}}},
|
|
||||||
OutputFiles: []contracts.OutputFile{{Name: "index.json"}},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func testRunOutputWithout(change func(*pipeline.RunOutput)) pipeline.RunOutput {
|
|
||||||
output := testRunOutput()
|
|
||||||
change(&output)
|
|
||||||
return output
|
|
||||||
}
|
|
||||||
|
|
||||||
func mustWorkingDirectory(t *testing.T) string {
|
|
||||||
t.Helper()
|
|
||||||
workingDirectory, err := filepath.Abs(".")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return workingDirectory
|
|
||||||
}
|
|
||||||
|
|
||||||
type partialResultWriter struct {
|
type partialResultWriter struct {
|
||||||
writer io.Writer
|
content bytes.Buffer
|
||||||
limit int
|
cause error
|
||||||
}
|
}
|
||||||
|
|
||||||
func (w partialResultWriter) Write(content []byte) (int, error) {
|
func (writer *partialResultWriter) Write(content []byte) (int, error) {
|
||||||
if len(content) > w.limit {
|
if len(content) > 8 {
|
||||||
content = content[:w.limit]
|
content = content[:8]
|
||||||
}
|
}
|
||||||
return w.writer.Write(content)
|
n, _ := writer.content.Write(content)
|
||||||
|
return n, writer.cause
|
||||||
}
|
}
|
||||||
|
|
||||||
type failingResultWriter struct{ err error }
|
var _ io.Writer = (*partialResultWriter)(nil)
|
||||||
|
|
||||||
func (w failingResultWriter) Write([]byte) (int, error) { return 0, w.err }
|
func TestPipelinePreflightClassifiesOnlyIdentifiableInvalidSelections(t *testing.T) {
|
||||||
|
// Real native missing-resource errors retain their category separately from
|
||||||
type zeroResultWriter struct{}
|
// private lookup causes. A joined operational branch must still win.
|
||||||
|
probe := &runProbe{}
|
||||||
func (zeroResultWriter) Write([]byte) (int, error) { return 0, nil }
|
app, file, input, _ := runFixture(t, probe)
|
||||||
|
var diagnostics bytes.Buffer
|
||||||
|
missing := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--llm-profile", "absent"}, commandkit.Streams{Out: io.Discard, Err: &diagnostics}).Cause
|
||||||
|
for _, test := range []struct {
|
||||||
|
cause error
|
||||||
|
status int
|
||||||
|
}{
|
||||||
|
{missing, 2},
|
||||||
|
{errors.Join(missing, context.Canceled), 1},
|
||||||
|
{errors.Join(missing, errors.New("unavailable inspection")), 1},
|
||||||
|
{errors.New("unavailable inspection"), 1},
|
||||||
|
} {
|
||||||
|
outcome := pipelineResourceFailure(test.cause)
|
||||||
|
if outcome.Status != test.status || !errors.Is(outcome.Cause, test.cause) {
|
||||||
|
t.Fatalf("preflight classification: %#v", outcome)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -12,6 +12,21 @@ import (
|
|||||||
"go.maximumdirect.net/promptkit"
|
"go.maximumdirect.net/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// SelectionError identifies an absent configured native resource. Callers can
|
||||||
|
// distinguish this invalid selection from unavailable inspection or cancellation
|
||||||
|
// without inspecting diagnostic text or the kit's private lookup causes.
|
||||||
|
type SelectionError struct{ Cause error }
|
||||||
|
|
||||||
|
func (err *SelectionError) Error() string { return err.Cause.Error() }
|
||||||
|
func (err *SelectionError) Unwrap() error { return err.Cause }
|
||||||
|
|
||||||
|
func resourceFinding(err error) error {
|
||||||
|
if !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) && (errors.Is(err, promptkit.ErrPromptNotFound) || errors.Is(err, promptkit.ErrProfileNotFound)) {
|
||||||
|
return &SelectionError{Cause: err}
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
// RequiresEngine reports selected model work, including nested validators.
|
// RequiresEngine reports selected model work, including nested validators.
|
||||||
// Deterministic pipelines need no engine merely to validate their composition.
|
// Deterministic pipelines need no engine merely to validate their composition.
|
||||||
func RequiresEngine(resolved pipeline.ResolvedPipeline) bool {
|
func RequiresEngine(resolved pipeline.ResolvedPipeline) bool {
|
||||||
@@ -69,7 +84,7 @@ func Validate(ctx context.Context, resolved pipeline.ResolvedPipeline, catalog p
|
|||||||
}
|
}
|
||||||
if profile != "" {
|
if profile != "" {
|
||||||
if _, err := engine.InspectProfile(ctx, profile); err != nil {
|
if _, err := engine.InspectProfile(ctx, profile); err != nil {
|
||||||
findings = append(findings, fmt.Errorf("%s profile %q: %w", scope, profile, err))
|
findings = append(findings, resourceFinding(fmt.Errorf("%s profile %q: %w", scope, profile, err)))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for _, id := range prompts {
|
for _, id := range prompts {
|
||||||
@@ -78,7 +93,7 @@ func Validate(ctx context.Context, resolved pipeline.ResolvedPipeline, catalog p
|
|||||||
}
|
}
|
||||||
prompt, err := engine.InspectPrompt(ctx, id)
|
prompt, err := engine.InspectPrompt(ctx, id)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
findings = append(findings, fmt.Errorf("%s prompt %q: %w", scope, id, err))
|
findings = append(findings, resourceFinding(fmt.Errorf("%s prompt %q: %w", scope, id, err)))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if profile != "" {
|
if profile != "" {
|
||||||
@@ -90,7 +105,7 @@ func Validate(ctx context.Context, resolved pipeline.ResolvedPipeline, catalog p
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if _, err := engine.InspectProfile(ctx, *prompt.DefaultProfileID); err != nil {
|
if _, err := engine.InspectProfile(ctx, *prompt.DefaultProfileID); err != nil {
|
||||||
findings = append(findings, fmt.Errorf("%s prompt %q default profile %q: %w", scope, id, *prompt.DefaultProfileID, err))
|
findings = append(findings, resourceFinding(fmt.Errorf("%s prompt %q default profile %q: %w", scope, id, *prompt.DefaultProfileID, err)))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user