Migrate CLI execution and terminal reports

This commit is contained in:
2026-11-18 19:04:15 +00:00
parent e7a8b0db04
commit 77f168c660
7 changed files with 66 additions and 66 deletions

View File

@@ -62,6 +62,8 @@ scaffolding, or conversation and coordination features.
## Stage 2: Migrate CLI execution, rendering, and terminal reports ## Stage 2: Migrate CLI execution, rendering, and terminal reports
**Status: Complete.**
1. In `internal/adapter/cli/run.go` remove `--prompt-version`, the stored 1. In `internal/adapter/cli/run.go` remove `--prompt-version`, the stored
value, and associated usage text from `run` and `render`. Remove the value, and associated usage text from `run` and `render`. Remove the
version from `buildRunRequestFromConfig` and the run summary. Forward version from `buildRunRequestFromConfig` and the run summary. Forward

View File

@@ -33,7 +33,6 @@ type runConfig struct {
promptkitSources []appconfig.SourceSelection promptkitSources []appconfig.SourceSelection
promptID string promptID string
promptVersion string
profileID string profileID string
sessionID string sessionID string
inputRaw listFlag inputRaw listFlag
@@ -170,16 +169,10 @@ func executeRun(cfg *runConfig, req promptkit.RunRequest, run func(context.Conte
collector = executionreport.NewCollector() collector = executionreport.NewCollector()
req.Observer = collector.Observe req.Observer = collector.Observe
} }
runContext := context.Background() req.OperationTimeout = cfg.runTimeout
cancel := func() {} res, runErr := run(context.Background(), req)
if cfg.runTimeout > 0 {
runContext, cancel = context.WithTimeout(runContext, cfg.runTimeout)
}
res, runErr := run(runContext, req)
contextErr := runContext.Err()
cancel()
if runErr != nil { if runErr != nil {
if errors.Is(contextErr, context.DeadlineExceeded) { if errors.Is(runErr, promptkit.ErrOperationTimeout) {
fmt.Fprintln(stderr, "run error: execution deadline exceeded") fmt.Fprintln(stderr, "run error: execution deadline exceeded")
} else { } else {
fmt.Fprintln(stderr, runErrorMessage(runErr)) fmt.Fprintln(stderr, runErrorMessage(runErr))
@@ -439,7 +432,6 @@ func registerExecutionRequestFlags(fs *flag.FlagSet, cfg *runConfig) {
var promptkitSources promptKitSourceFlagValue var promptkitSources promptKitSourceFlagValue
fs.Var(&promptkitSources, "promptkit-source", "selected PromptKit source path (repeatable)") fs.Var(&promptkitSources, "promptkit-source", "selected PromptKit source path (repeatable)")
fs.StringVar(&cfg.promptID, "prompt", "", "prompt ID to run") fs.StringVar(&cfg.promptID, "prompt", "", "prompt ID to run")
fs.StringVar(&cfg.promptVersion, "prompt-version", "", "optional prompt definition version")
fs.StringVar(&cfg.profileID, "profile", "", "optional execution profile ID; if omitted, prompt default_profile is used") fs.StringVar(&cfg.profileID, "profile", "", "optional execution profile ID; if omitted, prompt default_profile is used")
fs.StringVar(&cfg.sessionID, "session-id", "", "optional session ID") fs.StringVar(&cfg.sessionID, "session-id", "", "optional session ID")
fs.Var(&cfg.inputRaw, "input", "input mapping(s): name=path (repeatable, comma-separated)") fs.Var(&cfg.inputRaw, "input", "input mapping(s): name=path (repeatable, comma-separated)")
@@ -630,7 +622,6 @@ func buildRunRequestFromConfig(cfg *runConfig) (promptkit.RunRequest, error) {
} }
return promptkit.RunRequest{ return promptkit.RunRequest{
PromptID: cfg.promptID, PromptID: cfg.promptID,
PromptVersion: cfg.promptVersion,
ProfileID: cfg.profileID, ProfileID: cfg.profileID,
SessionID: cfg.sessionID, SessionID: cfg.sessionID,
Inputs: inputs, Inputs: inputs,
@@ -709,9 +700,8 @@ func printSummary(stderr io.Writer, res *promptkit.RunResult) {
if res == nil { if res == nil {
return return
} }
fmt.Fprintf(stderr, "prompt=%s@%s selected_profile=%s model=%s validation=%s mode=%s validation_errors=%d prompt_hash=%s inputs=%d usage=%d/%d/%d", fmt.Fprintf(stderr, "prompt=%s selected_profile=%s model=%s validation=%s mode=%s validation_errors=%d prompt_hash=%s inputs=%d usage=%d/%d/%d",
res.PromptID, res.PromptID,
res.PromptVersion,
res.SelectedProfileID, res.SelectedProfileID,
res.ModelName, res.ModelName,
res.Validation.Status, res.Validation.Status,
@@ -741,8 +731,8 @@ func runErrorMessage(err error) string {
func printUsage(w io.Writer) { func printUsage(w io.Writer) {
fmt.Fprintln(w, "usage: scriptorium <run|render|serve|inspect|config> ...") fmt.Fprintln(w, "usage: scriptorium <run|render|serve|inspect|config> ...")
fmt.Fprintln(w, " run: scriptorium run [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--schema-dir DIR] --prompt ID [--prompt-version VERSION] [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--out path] [--execution-report path] [--timeout 10m] [--run-timeout 10m] [--transient-retries N]") fmt.Fprintln(w, " run: scriptorium run [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--schema-dir DIR] --prompt ID [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--out path] [--execution-report path] [--timeout 10m] [--run-timeout 10m] [--transient-retries N]")
fmt.Fprintln(w, " render: scriptorium render [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--schema-dir DIR] --prompt ID [--prompt-version VERSION] [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--format text|json] [--out path] [--timeout 10m] [--transient-retries N]") fmt.Fprintln(w, " render: scriptorium render [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--schema-dir DIR] --prompt ID [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--format text|json] [--out path] [--timeout 10m] [--transient-retries N]")
fmt.Fprintf(w, " serve: scriptorium serve [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--addr %s] [--schema-dir DIR] [--artifact-root DIR] [--max-request-bytes N] [--max-artifact-bytes N] [--max-response-bytes N]\n", defaults.HTTPAddrDefault) fmt.Fprintf(w, " serve: scriptorium serve [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--addr %s] [--schema-dir DIR] [--artifact-root DIR] [--max-request-bytes N] [--max-artifact-bytes N] [--max-response-bytes N]\n", defaults.HTTPAddrDefault)
fmt.Fprintln(w, " inspect prompt: scriptorium inspect prompt --prompt ID [--prompt-version VERSION] [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--schema-dir DIR] [--format text|json] [--out path]") fmt.Fprintln(w, " inspect prompt: scriptorium inspect prompt --prompt ID [--prompt-version VERSION] [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--schema-dir DIR] [--format text|json] [--out path]")
fmt.Fprintln(w, " inspect profile: scriptorium inspect profile --profile ID [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--schema-dir DIR] [--format text|json] [--out path]") fmt.Fprintln(w, " inspect profile: scriptorium inspect profile --profile ID [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--schema-dir DIR] [--format text|json] [--out path]")

View File

@@ -110,9 +110,8 @@ func TestParseRunArgsFlagMapping(t *testing.T) {
"--config", configPath, "--config", configPath,
"--promptkit-source", lib.promptDir, "--promptkit-source", lib.promptDir,
"--promptkit-source", sourceFile, "--promptkit-source", sourceFile,
"--prompt", "prompt.a", "--prompt", "support/prompt.a",
"--prompt-version", "2", "--profile", "support/profile.a",
"--profile", "profile.a",
"--session-id", "session-1", "--session-id", "session-1",
"--input", "a=b", "--input", "a=b",
"--llm-base-url", "http://x/v1", "--llm-base-url", "http://x/v1",
@@ -130,8 +129,8 @@ func TestParseRunArgsFlagMapping(t *testing.T) {
if !reflect.DeepEqual(cfg.promptkitSources, []appconfig.SourceSelection{{Directory: lib.promptDir}, {File: sourceFile}}) { if !reflect.DeepEqual(cfg.promptkitSources, []appconfig.SourceSelection{{Directory: lib.promptDir}, {File: sourceFile}}) {
t.Fatalf("unexpected source selections: %#v", cfg.promptkitSources) t.Fatalf("unexpected source selections: %#v", cfg.promptkitSources)
} }
if cfg.promptID != "prompt.a" || cfg.promptVersion != "2" || cfg.profileID != "profile.a" { if cfg.promptID != "support/prompt.a" || cfg.profileID != "support/profile.a" {
t.Fatalf("unexpected prompt/version/profile ids: %q %q %q", cfg.promptID, cfg.promptVersion, cfg.profileID) t.Fatalf("unexpected prompt/profile ids: %q %q", cfg.promptID, cfg.profileID)
} }
if cfg.sessionID != "session-1" || cfg.reasoningEffort != "medium" || !cfg.reasoningEffortSet { if cfg.sessionID != "session-1" || cfg.reasoningEffort != "medium" || !cfg.reasoningEffortSet {
t.Fatalf("unexpected session or reasoning configuration: %+v", cfg) t.Fatalf("unexpected session or reasoning configuration: %+v", cfg)
@@ -286,7 +285,6 @@ func TestUsageIncludesExecutionAndServeFlags(t *testing.T) {
usage := stderr.String() usage := stderr.String()
for _, want := range []string{ for _, want := range []string{
"--prompt-version VERSION",
"--session-id ID", "--session-id ID",
"--reasoning-effort VALUE", "--reasoning-effort VALUE",
"--artifact-root", "--artifact-root",
@@ -890,22 +888,59 @@ func TestBuildRunRequestPreservesReasoningEffortPresenceAndSessionID(t *testing.
} }
} }
func TestBuildRunRequestAllowsOmittedInputsAndMapsPromptVersion(t *testing.T) { func TestBuildRunRequestAllowsOmittedInputsAndPreservesQualifiedIDs(t *testing.T) {
req, err := buildRunRequestFromConfig(&runConfig{ req, err := buildRunRequestFromConfig(&runConfig{
promptID: "prompt-1", promptID: "support/prompt-1",
promptVersion: "2", profileID: "support/profile-1",
}) })
if err != nil { if err != nil {
t.Fatalf("expected request without inputs to build, got %v", err) t.Fatalf("expected request without inputs to build, got %v", err)
} }
if req.PromptVersion != "2" { if req.PromptID != "support/prompt-1" || req.ProfileID != "support/profile-1" {
t.Fatalf("expected prompt version to be mapped, got %q", req.PromptVersion) t.Fatalf("qualified IDs changed: %+v", req)
} }
if req.Inputs != nil { if req.Inputs != nil {
t.Fatalf("expected omitted inputs to remain nil, got %#v", req.Inputs) t.Fatalf("expected omitted inputs to remain nil, got %#v", req.Inputs)
} }
} }
func TestExecutionCommandsRejectPromptVersion(t *testing.T) {
args := []string{"--prompt", "support/answer", "--prompt-version", "1"}
if _, err := parseRunArgs(args); err == nil {
t.Fatal("run accepted removed prompt version flag")
}
if _, err := parseRenderArgs(args); err == nil {
t.Fatal("render accepted removed prompt version flag")
}
}
func TestRunOperationTimeoutAttribution(t *testing.T) {
for _, tc := range []struct {
name string
err error
deadline bool
}{
{name: "operation budget", err: fmt.Errorf("wrapped: %w", promptkit.ErrOperationTimeout), deadline: true},
{name: "provider deadline", err: fmt.Errorf("provider: %w", context.DeadlineExceeded)},
} {
t.Run(tc.name, func(t *testing.T) {
var stderr bytes.Buffer
code := executeRun(&runConfig{runTimeout: 125 * time.Millisecond}, promptkit.RunRequest{}, func(ctx context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) {
if req.OperationTimeout != 125*time.Millisecond {
t.Errorf("operation timeout = %v", req.OperationTimeout)
}
if _, ok := ctx.Deadline(); ok {
t.Error("adapter added a context deadline")
}
return nil, tc.err
}, io.Discard, &stderr)
if code != ExitRuntimeError || strings.Contains(stderr.String(), "execution deadline exceeded") != tc.deadline {
t.Fatalf("unexpected outcome: code=%d stderr=%q", code, stderr.String())
}
})
}
}
func TestRunAndRenderParsingDoesNotRequireCallerSources(t *testing.T) { func TestRunAndRenderParsingDoesNotRequireCallerSources(t *testing.T) {
configPath := writeAppConfigFile(t, "{}\n") configPath := writeAppConfigFile(t, "{}\n")
if cfg, err := parseRunArgs([]string{"--config", configPath, "--prompt", "p"}); err != nil || len(cfg.promptkitSources) != 0 { if cfg, err := parseRunArgs([]string{"--config", configPath, "--prompt", "p"}); err != nil || len(cfg.promptkitSources) != 0 {
@@ -1455,7 +1490,6 @@ func TestPromptkitDefinitionsRenderThroughCLI(t *testing.T) {
code, stdout, stderr := runCLICommand(t, renderCommand, []string{ code, stdout, stderr := runCLICommand(t, renderCommand, []string{
"--config", configPath, "--config", configPath,
"--prompt", "compat.complete", "--prompt", "compat.complete",
"--prompt-version", "1.0.0",
"--input", "source=" + inputPath, "--input", "source=" + inputPath,
"--var", "topic=testing", "--var", "topic=testing",
"--format", "json", "--format", "json",
@@ -1468,7 +1502,7 @@ func TestPromptkitDefinitionsRenderThroughCLI(t *testing.T) {
if err := json.Unmarshal([]byte(stdout), &payload); err != nil { if err := json.Unmarshal([]byte(stdout), &payload); err != nil {
t.Fatalf("decode rendered fixture: %v\nbody=%s", err, stdout) t.Fatalf("decode rendered fixture: %v\nbody=%s", err, stdout)
} }
if payload["prompt_version"] != "1.0.0" || payload["selected_profile_id"] != "custom-derived" || payload["selected_backend_id"] != "fixture-custom" { if _, present := payload["prompt_version"]; present || payload["selected_profile_id"] != "custom-derived" || payload["selected_backend_id"] != "fixture-custom" {
t.Fatalf("unexpected selected definition and target: %#v", payload) t.Fatalf("unexpected selected definition and target: %#v", payload)
} }
if payload["session_id"] != "fixture-testing" { if payload["session_id"] != "fixture-testing" {
@@ -2337,7 +2371,7 @@ func TestRenderCommandExplicitProfileOverridesPromptDefault(t *testing.T) {
} }
} }
func TestRenderCommandUsesDefinitionInputRulesAndPromptVersions(t *testing.T) { func TestRenderCommandUsesDefinitionInputRules(t *testing.T) {
lib := newCLITestLibrary(t) lib := newCLITestLibrary(t)
writeProfileFile(t, lib.profileDir, "local", "http://127.0.0.1:1/v1", "model") writeProfileFile(t, lib.profileDir, "local", "http://127.0.0.1:1/v1", "model")
@@ -2351,25 +2385,6 @@ func TestRenderCommandUsesDefinitionInputRulesAndPromptVersions(t *testing.T) {
output: output:
format: text format: text
validation_mode: none validation_mode: none
`)
writePromptDefinition(t, lib.promptDir, "versioned.yaml", `prompts:
versioned:
"1":
default_profile: local
messages:
- role: user
content: "one"
output:
format: text
validation_mode: none
"2":
default_profile: local
messages:
- role: user
content: "two"
output:
format: text
validation_mode: none
`) `)
writePromptDefinition(t, lib.promptDir, "optional.yaml", `prompts: writePromptDefinition(t, lib.promptDir, "optional.yaml", `prompts:
optional: optional:
@@ -2418,9 +2433,6 @@ func TestRenderCommandUsesDefinitionInputRulesAndPromptVersions(t *testing.T) {
wantCode int wantCode int
wantText string wantText string
}{ }{
{name: "sole version selected when omitted", args: []string{"--prompt", "sole"}, wantCode: ExitOK, wantText: "prompt_version: 1"},
{name: "explicit version selected", args: []string{"--prompt", "versioned", "--prompt-version", "2"}, wantCode: ExitOK, wantText: "prompt_version: 2"},
{name: "multiple versions require selection", args: []string{"--prompt", "versioned"}, wantCode: ExitRuntimeError, wantText: "version is ambiguous"},
{name: "no declared inputs", args: []string{"--prompt", "sole"}, wantCode: ExitOK, wantText: "prompt: sole"}, {name: "no declared inputs", args: []string{"--prompt", "sole"}, wantCode: ExitOK, wantText: "prompt: sole"},
{name: "optional input omitted", args: []string{"--prompt", "optional"}, wantCode: ExitOK, wantText: "prompt: optional"}, {name: "optional input omitted", args: []string{"--prompt", "optional"}, wantCode: ExitOK, wantText: "prompt: optional"},
{name: "required input omitted", args: []string{"--prompt", "required"}, wantCode: ExitRuntimeError, wantText: "required"}, {name: "required input omitted", args: []string{"--prompt", "required"}, wantCode: ExitRuntimeError, wantText: "required"},
@@ -2563,7 +2575,6 @@ func TestWriteOutputAndSummaryUseSeparateWriters(t *testing.T) {
} }
printSummary(&stderr, &promptkit.RunResult{ printSummary(&stderr, &promptkit.RunResult{
PromptID: "p", PromptID: "p",
PromptVersion: "1",
SelectedProfileID: "exec", SelectedProfileID: "exec",
ModelName: "m", ModelName: "m",
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic}, Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic},
@@ -2574,7 +2585,7 @@ func TestWriteOutputAndSummaryUseSeparateWriters(t *testing.T) {
if stdout.String() != "artifact-body" { if stdout.String() != "artifact-body" {
t.Fatalf("expected artifact output on stdout, got %q", stdout.String()) t.Fatalf("expected artifact output on stdout, got %q", stdout.String())
} }
if !strings.Contains(stderr.String(), "prompt=p@1") { if !strings.Contains(stderr.String(), "prompt=p ") || strings.Contains(stderr.String(), "prompt_version") {
t.Fatalf("expected summary on stderr, got %q", stderr.String()) t.Fatalf("expected summary on stderr, got %q", stderr.String())
} }
if strings.Contains(stderr.String(), "cached_tokens=") || strings.Contains(stderr.String(), "cache_write_tokens=") { if strings.Contains(stderr.String(), "cached_tokens=") || strings.Contains(stderr.String(), "cache_write_tokens=") {
@@ -2590,7 +2601,6 @@ func TestPrintSummaryIncludesCacheUsageWhenPresent(t *testing.T) {
printSummary(&stderr, &promptkit.RunResult{ printSummary(&stderr, &promptkit.RunResult{
PromptID: "p", PromptID: "p",
PromptVersion: "1",
SelectedProfileID: "exec", SelectedProfileID: "exec",
ModelName: "m", ModelName: "m",
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic}, Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic},
@@ -2621,7 +2631,6 @@ func TestPrintSummaryIncludesBackendWhenPresent(t *testing.T) {
var stderr bytes.Buffer var stderr bytes.Buffer
printSummary(&stderr, &promptkit.RunResult{ printSummary(&stderr, &promptkit.RunResult{
PromptID: "p", PromptID: "p",
PromptVersion: "1",
SelectedProfileID: "exec", SelectedProfileID: "exec",
SelectedBackendID: "local", SelectedBackendID: "local",
ModelName: "m", ModelName: "m",

View File

@@ -106,7 +106,6 @@ func (textPreparedRunFormatter) Format(prepared *promptkit.PreparedRun) ([]byte,
var b bytes.Buffer var b bytes.Buffer
fmt.Fprintf(&b, "prompt: %s\n", prepared.PromptID) fmt.Fprintf(&b, "prompt: %s\n", prepared.PromptID)
fmt.Fprintf(&b, "prompt_version: %s\n", prepared.PromptVersion)
fmt.Fprintf(&b, "selected_profile_id: %s\n", prepared.SelectedProfileID) fmt.Fprintf(&b, "selected_profile_id: %s\n", prepared.SelectedProfileID)
if prepared.SelectedBackendID != "" { if prepared.SelectedBackendID != "" {
fmt.Fprintf(&b, "selected_backend_id: %s\n", prepared.SelectedBackendID) fmt.Fprintf(&b, "selected_backend_id: %s\n", prepared.SelectedBackendID)

View File

@@ -17,10 +17,12 @@ func TestTextFormatterIncludesPreparedRunDetails(t *testing.T) {
t.Fatalf("expected no error, got %v", err) t.Fatalf("expected no error, got %v", err)
} }
s := string(out) s := string(out)
if strings.Contains(s, "prompt_version") {
t.Fatalf("unexpected prompt version in text output: %s", s)
}
for _, want := range []string{ for _, want := range []string{
"prompt: prompt.id", "prompt: prompt.id",
"prompt_version: v1",
"selected_profile_id: local-fast", "selected_profile_id: local-fast",
"selected_backend_id: local", "selected_backend_id: local",
"endpoint: http://llm/v1", "endpoint: http://llm/v1",
@@ -194,8 +196,8 @@ func TestJSONFormatterEmitsValidJSONAndIncludesPreparedRunFields(t *testing.T) {
if decoded["prompt_id"] != "prompt.id" { if decoded["prompt_id"] != "prompt.id" {
t.Fatalf("expected prompt_id in json output, got %#v", decoded["prompt_id"]) t.Fatalf("expected prompt_id in json output, got %#v", decoded["prompt_id"])
} }
if decoded["prompt_version"] != "v1" { if _, ok := decoded["prompt_version"]; ok {
t.Fatalf("expected prompt_version in json output, got %#v", decoded["prompt_version"]) t.Fatalf("unexpected prompt_version in json output: %#v", decoded)
} }
if decoded["selected_profile_id"] != "local-fast" { if decoded["selected_profile_id"] != "local-fast" {
t.Fatalf("expected selected_profile_id in json output, got %#v", decoded["selected_profile_id"]) t.Fatalf("expected selected_profile_id in json output, got %#v", decoded["selected_profile_id"])
@@ -344,7 +346,6 @@ func TestFormatPreparedRunByNameUnknownFailsClearly(t *testing.T) {
func samplePreparedRun() *promptkit.PreparedRun { func samplePreparedRun() *promptkit.PreparedRun {
return &promptkit.PreparedRun{ return &promptkit.PreparedRun{
PromptID: "prompt.id", PromptID: "prompt.id",
PromptVersion: "v1",
PromptHash: "prompt-hash", PromptHash: "prompt-hash",
SelectedProfileID: "local-fast", SelectedProfileID: "local-fast",
SelectedBackendID: "local", SelectedBackendID: "local",

View File

@@ -6,7 +6,6 @@ import "go.maximumdirect.net/promptkit"
type Execution struct { type Execution struct {
RunID string `json:"run_id,omitempty"` RunID string `json:"run_id,omitempty"`
PromptID string `json:"prompt_id,omitempty"` PromptID string `json:"prompt_id,omitempty"`
PromptVersion string `json:"prompt_version,omitempty"`
ProfileID string `json:"profile_id,omitempty"` ProfileID string `json:"profile_id,omitempty"`
BackendID string `json:"backend_id,omitempty"` BackendID string `json:"backend_id,omitempty"`
Model string `json:"model,omitempty"` Model string `json:"model,omitempty"`
@@ -53,7 +52,7 @@ func (collector *Collector) Observe(event promptkit.ExecutionEvent) {
return TokenCount{Value: value.Value, Known: value.Known, Complete: value.Complete} return TokenCount{Value: value.Value, Known: value.Known, Complete: value.Complete}
} }
collector.report = &Execution{ collector.report = &Execution{
RunID: event.RunID, PromptID: event.PromptID, PromptVersion: event.PromptVersion, RunID: event.RunID, PromptID: event.PromptID,
ProfileID: event.ProfileID, BackendID: event.BackendID, Model: event.Model, ProfileID: event.ProfileID, BackendID: event.BackendID, Model: event.Model,
Outcome: outcome(event.Outcome), Failure: failure(event.Failure), Outcome: outcome(event.Outcome), Failure: failure(event.Failure),
StatusCode: event.StatusCode, RetriesExhausted: event.RetriesExhausted, StatusCode: event.StatusCode, RetriesExhausted: event.RetriesExhausted,

View File

@@ -24,7 +24,7 @@ func TestCollectorProjectsOnlyTerminalAggregates(t *testing.T) {
} }
collector.Observe(promptkit.ExecutionEvent{ collector.Observe(promptkit.ExecutionEvent{
Kind: promptkit.ExecutionFinished, RunID: "run-1", PromptID: "prompt-1", Kind: promptkit.ExecutionFinished, RunID: "run-1", PromptID: "prompt-1",
PromptVersion: "1", ProfileID: "profile-1", BackendID: "backend-1", Model: "model-1", ProfileID: "profile-1", BackendID: "backend-1", Model: "model-1",
Outcome: promptkit.ExecutionOutcomeError, Failure: promptkit.ExecutionFailureProviderStatus, Outcome: promptkit.ExecutionOutcomeError, Failure: promptkit.ExecutionFailureProviderStatus,
StatusCode: 503, RetriesExhausted: true, Attempts: 3, TransientRetries: 2, RepairCalls: 1, StatusCode: 503, RetriesExhausted: true, Attempts: 3, TransientRetries: 2, RepairCalls: 1,
Duration: 1500 * time.Millisecond, CapacityWait: 20 * time.Millisecond, Duration: 1500 * time.Millisecond, CapacityWait: 20 * time.Millisecond,
@@ -54,7 +54,7 @@ func TestCollectorProjectsOnlyTerminalAggregates(t *testing.T) {
if err := json.Unmarshal(encoded, &fields); err != nil { if err := json.Unmarshal(encoded, &fields); err != nil {
t.Fatal(err) t.Fatal(err)
} }
for _, key := range []string{"run_id", "prompt_id", "prompt_version", "profile_id", "backend_id", "model", "outcome", "failure", "status_code", "retries_exhausted", "attempts", "transient_retries", "repair_calls", "duration_ms", "capacity_wait_ms", "generation_duration_ms", "retry_delay_ms", "usage"} { for _, key := range []string{"run_id", "prompt_id", "profile_id", "backend_id", "model", "outcome", "failure", "status_code", "retries_exhausted", "attempts", "transient_retries", "repair_calls", "duration_ms", "capacity_wait_ms", "generation_duration_ms", "retry_delay_ms", "usage"} {
if _, ok := fields[key]; !ok { if _, ok := fields[key]; !ok {
t.Fatalf("report missing %q: %s", key, encoded) t.Fatalf("report missing %q: %s", key, encoded)
} }
@@ -65,7 +65,7 @@ func TestCollectorProjectsOnlyTerminalAggregates(t *testing.T) {
t.Fatalf("incomplete token count %q: %#v", key, item) t.Fatalf("incomplete token count %q: %#v", key, item)
} }
} }
for _, forbidden := range []string{"session_id", "headers", "endpoint", "api_key", "error", "output", "input"} { for _, forbidden := range []string{"prompt_version", "session_id", "headers", "endpoint", "api_key", "error", "output", "input"} {
if _, present := fields[forbidden]; present { if _, present := fields[forbidden]; present {
t.Fatalf("report exposed %q: %s", forbidden, encoded) t.Fatalf("report exposed %q: %s", forbidden, encoded)
} }