From 41dc1f55171f5c48923317256456a0f21ff284b1 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Fri, 20 Nov 2026 04:48:04 +0000 Subject: [PATCH] Add independent native run report delivery --- docs/roadmap/implementation.md | 18 ++ internal/adapter/cli/delivery.go | 73 ++++++++ internal/adapter/cli/delivery_test.go | 251 ++++++++++++++++++++++++++ internal/adapter/cli/native_run.go | 14 +- internal/adapter/cli/run.go | 49 ----- 5 files changed, 348 insertions(+), 57 deletions(-) create mode 100644 internal/adapter/cli/delivery.go create mode 100644 internal/adapter/cli/delivery_test.go diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 38132d5..36b799a 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -520,6 +520,24 @@ wording or requiring a particular generation summary. ## Stage 6: Add Separate Native Report Delivery +**Status: Complete.** Added presence-aware report paths, typed native report +formats with no default, and native completion hints. Validation precedes model +execution and file creation. Main delivery still uses the invocation reporter; +the separate file renders the same native result without its artifact-only +template, independently of quiet and stdout format. Ordinary create/truncate +writes handle render/write/close errors without making parents. CLI observation +collection and its old report-file writer have been removed; HTTP is unchanged. + +Verified on Go 1.27.4 with focused integration tests, race checks, and vet: +automatic/explicit native report formats, extension independence, replacement, +literal `-` filenames, quiet and redirected subprocess stdout, preflight errors +before provider/file effects, unchanged existing reports on resultless failure, +stdout failure followed by a report attempt, report failure after main success, +native render failures, retained presentation/file errors, and domain status 4 +surviving delivery failure. Subprocess checks invoke the native adapter in a test +executable; final shipped-binary verification follows the legacy-file cleanup. +Full-package compilation remains deferred to Stages 8–9 as previously recorded. + **Primary files:** CLI generation-delivery helpers and focused delivery tests. 1. Add `--report` as a presence-aware application file flag with path completion. diff --git a/internal/adapter/cli/delivery.go b/internal/adapter/cli/delivery.go new file mode 100644 index 0000000..700347c --- /dev/null +++ b/internal/adapter/cli/delivery.go @@ -0,0 +1,73 @@ +package cli + +import ( + "errors" + "fmt" + "io" + "os" + + "go.maximumdirect.net/commandkit" + "go.maximumdirect.net/promptkit" +) + +func reportFlags() []commandkit.Flag[runInput] { + return []commandkit.Flag[runInput]{ + commandkit.ValueFlag(commandkit.FlagMeta{Name: "report", Usage: "Additionally write the native run result to a file.", Completion: commandkit.CompleteFiles()}, (*string)(nil), parseString, + func(input *runInput, value commandkit.Supplied[string]) { input.ReportPath = value }), + commandkit.ValueFlag(commandkit.FlagMeta{Name: "report-format", Usage: "Select the separate report format (omission uses native automatic rendering).", Completion: commandkit.CompleteChoices(commandkit.CompletionChoice{Value: "text"}, commandkit.CompletionChoice{Value: "yaml"}, commandkit.CompletionChoice{Value: "json"})}, (*commandkit.Format)(nil), parseReportFormat, + func(input *runInput, value commandkit.Supplied[commandkit.Format]) { input.ReportFormat = value }), + } +} + +func parseReportFormat(raw string) (commandkit.Format, error) { + format := commandkit.Format(raw) + switch format { + case commandkit.FormatText, commandkit.FormatYAML, commandkit.FormatJSON: + return format, nil + default: + return "", errors.New("report format must be text, yaml, or json") + } +} + +func checkReportInput(path commandkit.Supplied[string]) error { + if path.Set && path.Value == "" { + return errors.New("report path must not be empty") + } + return nil +} + +func deliverRunResult(invocation commandkit.Invocation, input runInput, result *promptkit.RunResult) commandkit.Outcome { + outcome := completedRunOutcome(result) + if result == nil { + return outcome + } + // The invocation reporter records its own presentation failures. A requested + // file remains an independent effect and is attempted even after stdout fails. + _ = invocation.Reporter().Report(commandkit.Report{Value: result, TextTemplate: artifactTextTemplate}) + if input.ReportPath.Set { + if err := writeNativeReport(input.ReportPath.Value, result, input.ReportFormat.Value); err != nil { + if outcome.Status == 0 { + outcome = commandkit.Failure("The report file could not be delivered.") + } else { + outcome.Explanation.Message += " The report file could not be delivered." + } + outcome.Cause = errors.Join(outcome.Cause, err) + outcome.Explanation.Details = append(outcome.Explanation.Details, func(writer io.Writer) error { _, writeErr := fmt.Fprintln(writer, err); return writeErr }) + } + } + _ = invocation.Reporter().Info("Prompt execution completed.") + return outcome +} + +func writeNativeReport(path string, result *promptkit.RunResult, format commandkit.Format) error { + file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) + if err != nil { + return fmt.Errorf("open report %q: %w", path, err) + } + renderErr := commandkit.RenderReport(file, commandkit.Report{Value: result}, format) + closeErr := file.Close() + if err := errors.Join(renderErr, closeErr); err != nil { + return fmt.Errorf("deliver report %q: %w", path, err) + } + return nil +} diff --git a/internal/adapter/cli/delivery_test.go b/internal/adapter/cli/delivery_test.go new file mode 100644 index 0000000..a5c922e --- /dev/null +++ b/internal/adapter/cli/delivery_test.go @@ -0,0 +1,251 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + "sync/atomic" + "testing" + "time" + + "go.maximumdirect.net/commandkit" + "go.maximumdirect.net/promptkit" +) + +func deliveryArgs() []string { + return []string{"run", "support/summary", "--config=app.yml", "--profile=support/writer", "--session-id=session", "--var=mode=", "--var=title=Title", "--input-text=transcript=text", "--input-text=unlisted=literal"} +} + +func TestSeparateReportsUseNativeFormats(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"choices":[{"message":{"content":"artifact"}}]}`) + })) + defer server.Close() + nativePromptFixture(t, server.URL+"/v1") + for _, format := range []commandkit.Format{"", commandkit.FormatText, commandkit.FormatYAML, commandkit.FormatJSON} { + path := "report-" + string(format) + ".different-extension" + writeNativeFile(t, path, "old-report-must-be-replaced-completely") + args := append(deliveryArgs(), "--format=json", "--report="+path) + if format != "" { + args = append(args, "--report-format="+string(format)) + } + outcome, output, diagnostics := executeNative(t, args...) + if outcome.Status != 0 { + t.Fatalf("%q: %+v %q", format, outcome, diagnostics) + } + var result promptkit.RunResult + if err := json.Unmarshal([]byte(output), &result); err != nil { + t.Fatal(err) + } + // Use the serialized native value rather than round-tripping timing + // metadata through RunResult: a positive sub-millisecond duration is + // serialized as zero but would be omitted after that typed round trip. + var nativeValue any + if err := json.Unmarshal([]byte(output), &nativeValue); err != nil { + t.Fatal(err) + } + var expected bytes.Buffer + if err := commandkit.RenderReport(&expected, commandkit.Report{Value: nativeValue}, format); err != nil { + t.Fatal(err) + } + actual, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(actual, expected.Bytes()) || !bytes.Contains(actual, []byte(result.RunID)) || !bytes.Contains(actual, []byte("selected_profile_id")) { + t.Fatalf("report %q differs from native renderer:\n%s\nexpected:\n%s", format, actual, expected.Bytes()) + } + } + outcome, output, diagnostics := executeNative(t, append(deliveryArgs(), "--format=text", "--report=metadata.yaml", "--report-format=json")...) + if outcome.Status != 0 || output != "artifact" { + t.Fatalf("independent formats: %+v %q %q", outcome, output, diagnostics) + } + data, err := os.ReadFile("metadata.yaml") + if err != nil { + t.Fatal(err) + } + var result promptkit.RunResult + if err := json.Unmarshal(data, &result); err != nil { + t.Fatal(err) + } + if string(result.Artifact.Body) != "artifact" || result.RunID == "" { + t.Fatalf("report lost full result: %+v", result) + } + outcome, output, diagnostics = executeNative(t, append(deliveryArgs(), "--quiet", "--report=-", "--report-format=json")...) + if outcome.Status != 0 || output != "" { + t.Fatalf("quiet report: %+v %q %q", outcome, output, diagnostics) + } + if data, err := os.ReadFile("-"); err != nil || !bytes.Contains(data, []byte("run_id")) { + t.Fatalf("literal dash report: %s %v", data, err) + } +} + +func TestReportValidationAndResultlessFailureLeaveFilesUntouched(t *testing.T) { + var calls atomic.Int64 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"choices":[{"message":{"content":"artifact"}}]}`) + })) + defer server.Close() + nativePromptFixture(t, server.URL+"/v1") + for _, flags := range [][]string{{"--report="}, {"--report=untouched", "--report-format=invalid"}, {"--report=untouched", "--report-format="}} { + outcome, _, diagnostics := executeNative(t, append(deliveryArgs(), flags...)...) + if outcome.Status != 2 || diagnostics == "" || calls.Load() != 0 { + t.Fatalf("preflight %v: %+v %q calls=%d", flags, outcome, diagnostics, calls.Load()) + } + } + if _, err := os.Stat("untouched"); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("validation created file: %v", err) + } + writeNativeFile(t, "existing-report", "keep-existing") + outcome, output, diagnostics := executeNative(t, "run", "missing", "--config=app.yml", "--report=existing-report") + if outcome.Status != 1 || output != "" || diagnostics == "" { + t.Fatalf("resultless failure: %+v %q %q", outcome, output, diagnostics) + } + if data, err := os.ReadFile("existing-report"); err != nil || string(data) != "keep-existing" { + t.Fatalf("resultless failure touched report: %q %v", data, err) + } + outcome, output, diagnostics = executeNative(t, append(deliveryArgs(), "--report=missing-parent/report")...) + if outcome.Status != 1 || output != "artifact" || !strings.Contains(diagnostics, "report") { + t.Fatalf("report failure after main output: %+v %q %q", outcome, output, diagnostics) + } + if _, err := os.Stat("missing-parent"); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("created report parent: %v", err) + } +} + +type failedDeliveryWriter struct{ cause error } + +func (writer failedDeliveryWriter) Write([]byte) (int, error) { return 0, writer.cause } + +func TestNativeDeliveryAttemptsReportAfterStdoutFailure(t *testing.T) { + t.Chdir(t.TempDir()) + result := &promptkit.RunResult{RunID: "known-result", Artifact: promptkit.Artifact{Body: []byte("body")}} + stdoutFailure := errors.New("stdout failure") + for _, invalid := range []bool{false, true} { + result.Validation.Status = promptkit.ValidationPassed + if invalid { + result.Validation.Status = promptkit.ValidationFailed + } + for _, path := range []string{"report", "absent/report"} { + app, err := commandkit.New(commandkit.Identity{Name: "delivery"}, commandkit.NewCommand(commandkit.CommandSpec[struct{}]{Name: "run", Run: func(inv commandkit.Invocation, _ struct{}) commandkit.Outcome { + return deliverRunResult(inv, runInput{ReportPath: commandkit.Supplied[string]{Set: true, Value: path}}, result) + }})) + if err != nil { + t.Fatal(err) + } + var diagnostics bytes.Buffer + outcome := app.Execute(context.Background(), []string{"run"}, commandkit.Streams{Out: failedDeliveryWriter{stdoutFailure}, Err: &diagnostics}) + want := 1 + if invalid { + want = 4 + } + if outcome.Status != want || !errors.Is(outcome.Presentation.Output.WriteError, stdoutFailure) { + t.Fatalf("delivery outcome: %+v %q", outcome, diagnostics.String()) + } + if invalid && (outcome.Kind != commandkit.OutcomeDomain || outcome.Domain != "validation_failed") { + t.Fatalf("domain outcome lost: %+v", outcome) + } + if path == "report" { + if data, err := os.ReadFile(path); err != nil || !bytes.Contains(data, []byte("known-result")) { + t.Fatalf("report not attempted: %q %v", data, err) + } + } else { + var fileError *os.PathError + if !errors.As(outcome.Cause, &fileError) || !strings.Contains(diagnostics.String(), "report file") { + t.Fatalf("report error lost: %+v %q", outcome, diagnostics.String()) + } + } + } + } + // Unsupported result data exercises native rendering failure, not a local + // replacement serializer. Both main and file delivery failures survive. + result.Validation.Status = promptkit.ValidationPassed + result.EffectiveModelParams.ExtraParams = map[string]any{"invalid": make(chan int)} + app, err := commandkit.New(commandkit.Identity{Name: "delivery"}, commandkit.NewCommand(commandkit.CommandSpec[struct{}]{Name: "run", Run: func(inv commandkit.Invocation, _ struct{}) commandkit.Outcome { + return deliverRunResult(inv, runInput{ReportPath: commandkit.Supplied[string]{Set: true, Value: "render-failed"}}, result) + }})) + if err != nil { + t.Fatal(err) + } + var output, diagnostics bytes.Buffer + outcome := app.Execute(context.Background(), []string{"run", "--format=json"}, commandkit.Streams{Out: &output, Err: &diagnostics}) + if outcome.Status != 1 || outcome.Presentation.Output.RenderError == nil || outcome.Cause == nil || !strings.Contains(diagnostics.String(), "report file") { + t.Fatalf("render errors lost: %+v %q", outcome, diagnostics.String()) + } + if _, err := os.Stat("render-failed"); err != nil { + t.Fatalf("file effect not attempted: %v", err) + } +} + +func TestCLIProcessOutput(t *testing.T) { + if os.Getenv("SCRIPTORIUM_CLI_CHILD") == "1" { + separator := slices.Index(os.Args, "--") + if separator < 0 { + os.Exit(2) + } + outcome := Execute(context.Background(), commandkit.Identity{Name: "scriptorium"}, os.Args[separator+1:], os.Stdout, os.Stderr) + os.Exit(outcome.Status) + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"choices":[{"message":{"content":"redirected artifact"}}]}`) + })) + defer server.Close() + nativePromptFixture(t, server.URL+"/v1") + executable, err := os.Executable() + if err != nil { + t.Fatal(err) + } + for _, quiet := range []bool{false, true} { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + args := append([]string{"-test.run=^TestCLIProcessOutput$", "--"}, deliveryArgs()...) + args = append(args, "--report=process-report", "--report-format=json") + if quiet { + args = append(args, "--quiet") + } + command := exec.CommandContext(ctx, executable, args...) + command.Env = append(os.Environ(), "SCRIPTORIUM_CLI_CHILD=1") + path := filepath.Join(t.TempDir(), "redirected-output") + file, err := os.Create(path) + if err != nil { + cancel() + t.Fatal(err) + } + command.Stdout = file + var diagnostics bytes.Buffer + command.Stderr = &diagnostics + runErr := command.Run() + closeErr := file.Close() + cancel() + if runErr != nil || closeErr != nil { + t.Fatalf("subprocess: %v %v %q", runErr, closeErr, diagnostics.String()) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + want := "redirected artifact" + if quiet { + want = "" + } + if string(data) != want { + t.Fatalf("redirected quiet=%t: %q", quiet, data) + } + data, err = os.ReadFile("process-report") + if err != nil || !bytes.Contains(data, []byte("run_id")) { + t.Fatalf("independent process report: %s %v", data, err) + } + } +} diff --git a/internal/adapter/cli/native_run.go b/internal/adapter/cli/native_run.go index 9b21672..46effb9 100644 --- a/internal/adapter/cli/native_run.go +++ b/internal/adapter/cli/native_run.go @@ -18,6 +18,8 @@ type runInput struct { Execution ckprompt.ExecutionInput Namespaces promptkit.Namespaces OperationTimeout time.Duration + ReportPath commandkit.Supplied[string] + ReportFormat commandkit.Supplied[commandkit.Format] } var artifactTextTemplate = template.Must(commandkit.ParseTextTemplate("artifact", `{{printf "%s" .Artifact.Body}}`, nil)) @@ -30,6 +32,7 @@ func runDeclarations() []commandkit.Declaration { flags = append(flags, resources.Profile, resources.SessionID) flags = append(flags, inputs.All()...) flags = append(flags, execution.All()...) + flags = append(flags, reportFlags()...) flags = append(flags, commandkit.ValueFlag(commandkit.FlagMeta{Name: "run-timeout", Usage: "Whole-operation timeout as a Go duration."}, (*time.Duration)(nil), time.ParseDuration, func(input *runInput, value commandkit.Supplied[time.Duration]) { input.OperationTimeout = value.Value })) for _, namespace := range []struct { @@ -47,7 +50,8 @@ func runDeclarations() []commandkit.Declaration { Name: "run", Summary: "Run a stored prompt.", Flags: flags, Arguments: []commandkit.Argument[runInput]{commandkit.ValueArgument(commandkit.ArgumentMeta{Name: "PROMPT_ID", Usage: "Stored prompt identifier."}, true, parseString, func(input *runInput, value commandkit.Supplied[string]) { input.PromptID = value.Value })}, - Run: runPrompt, + Run: runPrompt, + CheckInput: func(input runInput) error { return checkReportInput(input.ReportPath) }, }) declarations := []commandkit.Declaration{command} for _, name := range []string{"profile", "session-id", "var", "input-text", "input-file", "endpoint", "model", "temperature", "max-tokens", "top-p", "timeout-seconds", "service-tier", "reasoning-effort", "api-key-env", "transient-retries", "extra-params", "prompt-namespace", "profile-namespace", "backend-namespace"} { @@ -89,11 +93,5 @@ func runPrompt(invocation commandkit.Invocation, input runInput) commandkit.Outc if err != nil { return requestFailure(err, explanation) } - outcome := completedRunOutcome(result) - if result == nil { - return outcome - } - _ = invocation.Reporter().Report(commandkit.Report{Value: result, TextTemplate: artifactTextTemplate}) - _ = invocation.Reporter().Info("Prompt execution completed.") - return outcome + return deliverRunResult(invocation, input, result) } diff --git a/internal/adapter/cli/run.go b/internal/adapter/cli/run.go index 25bdbad..beda55c 100644 --- a/internal/adapter/cli/run.go +++ b/internal/adapter/cli/run.go @@ -2,7 +2,6 @@ package cli import ( "context" - "encoding/json" "errors" "flag" "fmt" @@ -17,7 +16,6 @@ import ( appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config" "gitea.maximumdirect.net/eric/scriptorium/internal/defaults" renderformat "gitea.maximumdirect.net/eric/scriptorium/internal/format" - executionreport "gitea.maximumdirect.net/eric/scriptorium/internal/report" "go.maximumdirect.net/commandkit" "go.maximumdirect.net/configkit" "go.maximumdirect.net/promptkit" @@ -144,53 +142,6 @@ func runCommand(args []string, stdout, stderr io.Writer) int { return executeRun(cfg, req, engine.Run, stdout, stderr) } -func executeRun(cfg *runConfig, req promptkit.RunRequest, run func(context.Context, promptkit.RunRequest) (*promptkit.RunResult, error), stdout, stderr io.Writer) int { - var collector *executionreport.Collector - if cfg.executionReportPath != "" { - collector = executionreport.NewCollector() - req.Observer = collector.Observe - } - req.OperationTimeout = cfg.runTimeout - res, runErr := run(context.Background(), req) - if runErr != nil { - if errors.Is(runErr, promptkit.ErrOperationTimeout) { - fmt.Fprintln(stderr, "run error: execution deadline exceeded") - } else { - fmt.Fprintln(stderr, runErrorMessage(runErr)) - } - } - - outputFailed := false - if runErr == nil { - if err := writeOutput(stdout, cfg.outputPath, res.Artifact.Body); err != nil { - fmt.Fprintf(stderr, "output write error: %v\n", err) - outputFailed = true - } - } - reportFailed := false - if collector != nil { - if value := collector.Snapshot(); value == nil { - fmt.Fprintln(stderr, "execution report not produced: no terminal observation") - } else if err := writeExecutionReport(cfg.executionReportPath, value); err != nil { - fmt.Fprintf(stderr, "execution report write error: %v\n", err) - reportFailed = true - } - } - if runErr != nil || outputFailed || reportFailed { - return ExitRuntimeError - } - - printSummary(stderr, res) - return determineExitCode(nil, res) -} - -func writeExecutionReport(path string, value *executionreport.Execution) error { - data, err := json.Marshal(value) - if err != nil { - return err - } - return os.WriteFile(path, append(data, '\n'), 0644) -} func renderCommand(args []string, stdout, stderr io.Writer) int { cfg, err := parseRenderArgs(args)