From 77f168c66000fce0e7fade342154384b9ee1cc11 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Wed, 18 Nov 2026 19:04:15 +0000 Subject: [PATCH] Migrate CLI execution and terminal reports --- docs/roadmap/implementation.md | 2 + internal/adapter/cli/run.go | 22 ++----- internal/adapter/cli/run_test.go | 89 +++++++++++++++------------- internal/format/prepared_run.go | 1 - internal/format/prepared_run_test.go | 9 +-- internal/report/report.go | 3 +- internal/report/report_test.go | 6 +- 7 files changed, 66 insertions(+), 66 deletions(-) diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index cef259d..673bfe7 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -62,6 +62,8 @@ scaffolding, or conversation and coordination features. ## Stage 2: Migrate CLI execution, rendering, and terminal reports +**Status: Complete.** + 1. In `internal/adapter/cli/run.go` remove `--prompt-version`, the stored value, and associated usage text from `run` and `render`. Remove the version from `buildRunRequestFromConfig` and the run summary. Forward diff --git a/internal/adapter/cli/run.go b/internal/adapter/cli/run.go index 3c77985..bf9a3d4 100644 --- a/internal/adapter/cli/run.go +++ b/internal/adapter/cli/run.go @@ -33,7 +33,6 @@ type runConfig struct { promptkitSources []appconfig.SourceSelection promptID string - promptVersion string profileID string sessionID string inputRaw listFlag @@ -170,16 +169,10 @@ func executeRun(cfg *runConfig, req promptkit.RunRequest, run func(context.Conte collector = executionreport.NewCollector() req.Observer = collector.Observe } - runContext := context.Background() - cancel := func() {} - if cfg.runTimeout > 0 { - runContext, cancel = context.WithTimeout(runContext, cfg.runTimeout) - } - res, runErr := run(runContext, req) - contextErr := runContext.Err() - cancel() + req.OperationTimeout = cfg.runTimeout + res, runErr := run(context.Background(), req) if runErr != nil { - if errors.Is(contextErr, context.DeadlineExceeded) { + if errors.Is(runErr, promptkit.ErrOperationTimeout) { fmt.Fprintln(stderr, "run error: execution deadline exceeded") } else { fmt.Fprintln(stderr, runErrorMessage(runErr)) @@ -439,7 +432,6 @@ func registerExecutionRequestFlags(fs *flag.FlagSet, cfg *runConfig) { var promptkitSources promptKitSourceFlagValue fs.Var(&promptkitSources, "promptkit-source", "selected PromptKit source path (repeatable)") fs.StringVar(&cfg.promptID, "prompt", "", "prompt ID to run") - fs.StringVar(&cfg.promptVersion, "prompt-version", "", "optional prompt definition version") fs.StringVar(&cfg.profileID, "profile", "", "optional execution profile ID; if omitted, prompt default_profile is used") fs.StringVar(&cfg.sessionID, "session-id", "", "optional session ID") fs.Var(&cfg.inputRaw, "input", "input mapping(s): name=path (repeatable, comma-separated)") @@ -630,7 +622,6 @@ func buildRunRequestFromConfig(cfg *runConfig) (promptkit.RunRequest, error) { } return promptkit.RunRequest{ PromptID: cfg.promptID, - PromptVersion: cfg.promptVersion, ProfileID: cfg.profileID, SessionID: cfg.sessionID, Inputs: inputs, @@ -709,9 +700,8 @@ func printSummary(stderr io.Writer, res *promptkit.RunResult) { if res == nil { return } - fmt.Fprintf(stderr, "prompt=%s@%s selected_profile=%s model=%s validation=%s mode=%s validation_errors=%d prompt_hash=%s inputs=%d usage=%d/%d/%d", + fmt.Fprintf(stderr, "prompt=%s selected_profile=%s model=%s validation=%s mode=%s validation_errors=%d prompt_hash=%s inputs=%d usage=%d/%d/%d", res.PromptID, - res.PromptVersion, res.SelectedProfileID, res.ModelName, res.Validation.Status, @@ -741,8 +731,8 @@ func runErrorMessage(err error) string { func printUsage(w io.Writer) { fmt.Fprintln(w, "usage: scriptorium ...") - fmt.Fprintln(w, " run: scriptorium run [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--schema-dir DIR] --prompt ID [--prompt-version VERSION] [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--out path] [--execution-report path] [--timeout 10m] [--run-timeout 10m] [--transient-retries N]") - fmt.Fprintln(w, " render: scriptorium render [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--schema-dir DIR] --prompt ID [--prompt-version VERSION] [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--format text|json] [--out path] [--timeout 10m] [--transient-retries N]") + fmt.Fprintln(w, " run: scriptorium run [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--schema-dir DIR] --prompt ID [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--out path] [--execution-report path] [--timeout 10m] [--run-timeout 10m] [--transient-retries N]") + fmt.Fprintln(w, " render: scriptorium render [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--schema-dir DIR] --prompt ID [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--format text|json] [--out path] [--timeout 10m] [--transient-retries N]") fmt.Fprintf(w, " serve: scriptorium serve [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--addr %s] [--schema-dir DIR] [--artifact-root DIR] [--max-request-bytes N] [--max-artifact-bytes N] [--max-response-bytes N]\n", defaults.HTTPAddrDefault) fmt.Fprintln(w, " inspect prompt: scriptorium inspect prompt --prompt ID [--prompt-version VERSION] [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--schema-dir DIR] [--format text|json] [--out path]") fmt.Fprintln(w, " inspect profile: scriptorium inspect profile --profile ID [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--schema-dir DIR] [--format text|json] [--out path]") diff --git a/internal/adapter/cli/run_test.go b/internal/adapter/cli/run_test.go index 1de4f81..5e1ec7a 100644 --- a/internal/adapter/cli/run_test.go +++ b/internal/adapter/cli/run_test.go @@ -110,9 +110,8 @@ func TestParseRunArgsFlagMapping(t *testing.T) { "--config", configPath, "--promptkit-source", lib.promptDir, "--promptkit-source", sourceFile, - "--prompt", "prompt.a", - "--prompt-version", "2", - "--profile", "profile.a", + "--prompt", "support/prompt.a", + "--profile", "support/profile.a", "--session-id", "session-1", "--input", "a=b", "--llm-base-url", "http://x/v1", @@ -130,8 +129,8 @@ func TestParseRunArgsFlagMapping(t *testing.T) { if !reflect.DeepEqual(cfg.promptkitSources, []appconfig.SourceSelection{{Directory: lib.promptDir}, {File: sourceFile}}) { t.Fatalf("unexpected source selections: %#v", cfg.promptkitSources) } - if cfg.promptID != "prompt.a" || cfg.promptVersion != "2" || cfg.profileID != "profile.a" { - t.Fatalf("unexpected prompt/version/profile ids: %q %q %q", cfg.promptID, cfg.promptVersion, cfg.profileID) + if cfg.promptID != "support/prompt.a" || cfg.profileID != "support/profile.a" { + t.Fatalf("unexpected prompt/profile ids: %q %q", cfg.promptID, cfg.profileID) } if cfg.sessionID != "session-1" || cfg.reasoningEffort != "medium" || !cfg.reasoningEffortSet { t.Fatalf("unexpected session or reasoning configuration: %+v", cfg) @@ -286,7 +285,6 @@ func TestUsageIncludesExecutionAndServeFlags(t *testing.T) { usage := stderr.String() for _, want := range []string{ - "--prompt-version VERSION", "--session-id ID", "--reasoning-effort VALUE", "--artifact-root", @@ -890,22 +888,59 @@ func TestBuildRunRequestPreservesReasoningEffortPresenceAndSessionID(t *testing. } } -func TestBuildRunRequestAllowsOmittedInputsAndMapsPromptVersion(t *testing.T) { +func TestBuildRunRequestAllowsOmittedInputsAndPreservesQualifiedIDs(t *testing.T) { req, err := buildRunRequestFromConfig(&runConfig{ - promptID: "prompt-1", - promptVersion: "2", + promptID: "support/prompt-1", + profileID: "support/profile-1", }) if err != nil { t.Fatalf("expected request without inputs to build, got %v", err) } - if req.PromptVersion != "2" { - t.Fatalf("expected prompt version to be mapped, got %q", req.PromptVersion) + if req.PromptID != "support/prompt-1" || req.ProfileID != "support/profile-1" { + t.Fatalf("qualified IDs changed: %+v", req) } if req.Inputs != nil { t.Fatalf("expected omitted inputs to remain nil, got %#v", req.Inputs) } } +func TestExecutionCommandsRejectPromptVersion(t *testing.T) { + args := []string{"--prompt", "support/answer", "--prompt-version", "1"} + if _, err := parseRunArgs(args); err == nil { + t.Fatal("run accepted removed prompt version flag") + } + if _, err := parseRenderArgs(args); err == nil { + t.Fatal("render accepted removed prompt version flag") + } +} + +func TestRunOperationTimeoutAttribution(t *testing.T) { + for _, tc := range []struct { + name string + err error + deadline bool + }{ + {name: "operation budget", err: fmt.Errorf("wrapped: %w", promptkit.ErrOperationTimeout), deadline: true}, + {name: "provider deadline", err: fmt.Errorf("provider: %w", context.DeadlineExceeded)}, + } { + t.Run(tc.name, func(t *testing.T) { + var stderr bytes.Buffer + code := executeRun(&runConfig{runTimeout: 125 * time.Millisecond}, promptkit.RunRequest{}, func(ctx context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) { + if req.OperationTimeout != 125*time.Millisecond { + t.Errorf("operation timeout = %v", req.OperationTimeout) + } + if _, ok := ctx.Deadline(); ok { + t.Error("adapter added a context deadline") + } + return nil, tc.err + }, io.Discard, &stderr) + if code != ExitRuntimeError || strings.Contains(stderr.String(), "execution deadline exceeded") != tc.deadline { + t.Fatalf("unexpected outcome: code=%d stderr=%q", code, stderr.String()) + } + }) + } +} + func TestRunAndRenderParsingDoesNotRequireCallerSources(t *testing.T) { configPath := writeAppConfigFile(t, "{}\n") if cfg, err := parseRunArgs([]string{"--config", configPath, "--prompt", "p"}); err != nil || len(cfg.promptkitSources) != 0 { @@ -1455,7 +1490,6 @@ func TestPromptkitDefinitionsRenderThroughCLI(t *testing.T) { code, stdout, stderr := runCLICommand(t, renderCommand, []string{ "--config", configPath, "--prompt", "compat.complete", - "--prompt-version", "1.0.0", "--input", "source=" + inputPath, "--var", "topic=testing", "--format", "json", @@ -1468,7 +1502,7 @@ func TestPromptkitDefinitionsRenderThroughCLI(t *testing.T) { if err := json.Unmarshal([]byte(stdout), &payload); err != nil { t.Fatalf("decode rendered fixture: %v\nbody=%s", err, stdout) } - if payload["prompt_version"] != "1.0.0" || payload["selected_profile_id"] != "custom-derived" || payload["selected_backend_id"] != "fixture-custom" { + if _, present := payload["prompt_version"]; present || payload["selected_profile_id"] != "custom-derived" || payload["selected_backend_id"] != "fixture-custom" { t.Fatalf("unexpected selected definition and target: %#v", payload) } if payload["session_id"] != "fixture-testing" { @@ -2337,7 +2371,7 @@ func TestRenderCommandExplicitProfileOverridesPromptDefault(t *testing.T) { } } -func TestRenderCommandUsesDefinitionInputRulesAndPromptVersions(t *testing.T) { +func TestRenderCommandUsesDefinitionInputRules(t *testing.T) { lib := newCLITestLibrary(t) writeProfileFile(t, lib.profileDir, "local", "http://127.0.0.1:1/v1", "model") @@ -2351,25 +2385,6 @@ func TestRenderCommandUsesDefinitionInputRulesAndPromptVersions(t *testing.T) { output: format: text validation_mode: none -`) - writePromptDefinition(t, lib.promptDir, "versioned.yaml", `prompts: - versioned: - "1": - default_profile: local - messages: - - role: user - content: "one" - output: - format: text - validation_mode: none - "2": - default_profile: local - messages: - - role: user - content: "two" - output: - format: text - validation_mode: none `) writePromptDefinition(t, lib.promptDir, "optional.yaml", `prompts: optional: @@ -2418,9 +2433,6 @@ func TestRenderCommandUsesDefinitionInputRulesAndPromptVersions(t *testing.T) { wantCode int wantText string }{ - {name: "sole version selected when omitted", args: []string{"--prompt", "sole"}, wantCode: ExitOK, wantText: "prompt_version: 1"}, - {name: "explicit version selected", args: []string{"--prompt", "versioned", "--prompt-version", "2"}, wantCode: ExitOK, wantText: "prompt_version: 2"}, - {name: "multiple versions require selection", args: []string{"--prompt", "versioned"}, wantCode: ExitRuntimeError, wantText: "version is ambiguous"}, {name: "no declared inputs", args: []string{"--prompt", "sole"}, wantCode: ExitOK, wantText: "prompt: sole"}, {name: "optional input omitted", args: []string{"--prompt", "optional"}, wantCode: ExitOK, wantText: "prompt: optional"}, {name: "required input omitted", args: []string{"--prompt", "required"}, wantCode: ExitRuntimeError, wantText: "required"}, @@ -2563,7 +2575,6 @@ func TestWriteOutputAndSummaryUseSeparateWriters(t *testing.T) { } printSummary(&stderr, &promptkit.RunResult{ PromptID: "p", - PromptVersion: "1", SelectedProfileID: "exec", ModelName: "m", Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic}, @@ -2574,7 +2585,7 @@ func TestWriteOutputAndSummaryUseSeparateWriters(t *testing.T) { if stdout.String() != "artifact-body" { t.Fatalf("expected artifact output on stdout, got %q", stdout.String()) } - if !strings.Contains(stderr.String(), "prompt=p@1") { + if !strings.Contains(stderr.String(), "prompt=p ") || strings.Contains(stderr.String(), "prompt_version") { t.Fatalf("expected summary on stderr, got %q", stderr.String()) } if strings.Contains(stderr.String(), "cached_tokens=") || strings.Contains(stderr.String(), "cache_write_tokens=") { @@ -2590,7 +2601,6 @@ func TestPrintSummaryIncludesCacheUsageWhenPresent(t *testing.T) { printSummary(&stderr, &promptkit.RunResult{ PromptID: "p", - PromptVersion: "1", SelectedProfileID: "exec", ModelName: "m", Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic}, @@ -2621,7 +2631,6 @@ func TestPrintSummaryIncludesBackendWhenPresent(t *testing.T) { var stderr bytes.Buffer printSummary(&stderr, &promptkit.RunResult{ PromptID: "p", - PromptVersion: "1", SelectedProfileID: "exec", SelectedBackendID: "local", ModelName: "m", diff --git a/internal/format/prepared_run.go b/internal/format/prepared_run.go index 8618043..960f4c0 100644 --- a/internal/format/prepared_run.go +++ b/internal/format/prepared_run.go @@ -106,7 +106,6 @@ func (textPreparedRunFormatter) Format(prepared *promptkit.PreparedRun) ([]byte, var b bytes.Buffer fmt.Fprintf(&b, "prompt: %s\n", prepared.PromptID) - fmt.Fprintf(&b, "prompt_version: %s\n", prepared.PromptVersion) fmt.Fprintf(&b, "selected_profile_id: %s\n", prepared.SelectedProfileID) if prepared.SelectedBackendID != "" { fmt.Fprintf(&b, "selected_backend_id: %s\n", prepared.SelectedBackendID) diff --git a/internal/format/prepared_run_test.go b/internal/format/prepared_run_test.go index e5a7725..1374746 100644 --- a/internal/format/prepared_run_test.go +++ b/internal/format/prepared_run_test.go @@ -17,10 +17,12 @@ func TestTextFormatterIncludesPreparedRunDetails(t *testing.T) { t.Fatalf("expected no error, got %v", err) } s := string(out) + if strings.Contains(s, "prompt_version") { + t.Fatalf("unexpected prompt version in text output: %s", s) + } for _, want := range []string{ "prompt: prompt.id", - "prompt_version: v1", "selected_profile_id: local-fast", "selected_backend_id: local", "endpoint: http://llm/v1", @@ -194,8 +196,8 @@ func TestJSONFormatterEmitsValidJSONAndIncludesPreparedRunFields(t *testing.T) { if decoded["prompt_id"] != "prompt.id" { t.Fatalf("expected prompt_id in json output, got %#v", decoded["prompt_id"]) } - if decoded["prompt_version"] != "v1" { - t.Fatalf("expected prompt_version in json output, got %#v", decoded["prompt_version"]) + if _, ok := decoded["prompt_version"]; ok { + t.Fatalf("unexpected prompt_version in json output: %#v", decoded) } if decoded["selected_profile_id"] != "local-fast" { t.Fatalf("expected selected_profile_id in json output, got %#v", decoded["selected_profile_id"]) @@ -344,7 +346,6 @@ func TestFormatPreparedRunByNameUnknownFailsClearly(t *testing.T) { func samplePreparedRun() *promptkit.PreparedRun { return &promptkit.PreparedRun{ PromptID: "prompt.id", - PromptVersion: "v1", PromptHash: "prompt-hash", SelectedProfileID: "local-fast", SelectedBackendID: "local", diff --git a/internal/report/report.go b/internal/report/report.go index ee408b1..10dedf7 100644 --- a/internal/report/report.go +++ b/internal/report/report.go @@ -6,7 +6,6 @@ import "go.maximumdirect.net/promptkit" type Execution struct { RunID string `json:"run_id,omitempty"` PromptID string `json:"prompt_id,omitempty"` - PromptVersion string `json:"prompt_version,omitempty"` ProfileID string `json:"profile_id,omitempty"` BackendID string `json:"backend_id,omitempty"` Model string `json:"model,omitempty"` @@ -53,7 +52,7 @@ func (collector *Collector) Observe(event promptkit.ExecutionEvent) { return TokenCount{Value: value.Value, Known: value.Known, Complete: value.Complete} } collector.report = &Execution{ - RunID: event.RunID, PromptID: event.PromptID, PromptVersion: event.PromptVersion, + RunID: event.RunID, PromptID: event.PromptID, ProfileID: event.ProfileID, BackendID: event.BackendID, Model: event.Model, Outcome: outcome(event.Outcome), Failure: failure(event.Failure), StatusCode: event.StatusCode, RetriesExhausted: event.RetriesExhausted, diff --git a/internal/report/report_test.go b/internal/report/report_test.go index 90f93a9..b1416ce 100644 --- a/internal/report/report_test.go +++ b/internal/report/report_test.go @@ -24,7 +24,7 @@ func TestCollectorProjectsOnlyTerminalAggregates(t *testing.T) { } collector.Observe(promptkit.ExecutionEvent{ Kind: promptkit.ExecutionFinished, RunID: "run-1", PromptID: "prompt-1", - PromptVersion: "1", ProfileID: "profile-1", BackendID: "backend-1", Model: "model-1", + ProfileID: "profile-1", BackendID: "backend-1", Model: "model-1", Outcome: promptkit.ExecutionOutcomeError, Failure: promptkit.ExecutionFailureProviderStatus, StatusCode: 503, RetriesExhausted: true, Attempts: 3, TransientRetries: 2, RepairCalls: 1, Duration: 1500 * time.Millisecond, CapacityWait: 20 * time.Millisecond, @@ -54,7 +54,7 @@ func TestCollectorProjectsOnlyTerminalAggregates(t *testing.T) { if err := json.Unmarshal(encoded, &fields); err != nil { t.Fatal(err) } - for _, key := range []string{"run_id", "prompt_id", "prompt_version", "profile_id", "backend_id", "model", "outcome", "failure", "status_code", "retries_exhausted", "attempts", "transient_retries", "repair_calls", "duration_ms", "capacity_wait_ms", "generation_duration_ms", "retry_delay_ms", "usage"} { + for _, key := range []string{"run_id", "prompt_id", "profile_id", "backend_id", "model", "outcome", "failure", "status_code", "retries_exhausted", "attempts", "transient_retries", "repair_calls", "duration_ms", "capacity_wait_ms", "generation_duration_ms", "retry_delay_ms", "usage"} { if _, ok := fields[key]; !ok { t.Fatalf("report missing %q: %s", key, encoded) } @@ -65,7 +65,7 @@ func TestCollectorProjectsOnlyTerminalAggregates(t *testing.T) { t.Fatalf("incomplete token count %q: %#v", key, item) } } - for _, forbidden := range []string{"session_id", "headers", "endpoint", "api_key", "error", "output", "input"} { + for _, forbidden := range []string{"prompt_version", "session_id", "headers", "endpoint", "api_key", "error", "output", "input"} { if _, present := fields[forbidden]; present { t.Fatalf("report exposed %q: %s", forbidden, encoded) }