From bd7136ceebbdc7498e9a768ce8abbe0cbb00318f Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Fri, 20 Nov 2026 05:08:40 +0000 Subject: [PATCH] Retire the legacy CLI and document native operator workflows --- README.md | 11 +- docs/cli.md | 487 +-- docs/config.md | 382 +-- docs/consumers/api.md | 3 +- docs/consumers/upgrading-cli.md | 95 + docs/consumers/upgrading-promptkit.md | 38 +- docs/development.md | 6 +- docs/integrations/subprocess.md | 103 +- docs/internal/adapters.md | 206 +- docs/internal/overview.md | 23 +- docs/internal/sources.md | 146 +- docs/operations.md | 317 +- docs/policy/architecture.md | 13 +- docs/policy/testing.md | 13 +- docs/roadmap/implementation.md | 19 + docs/roadmap/narratio-feature-request.md | 11 +- examples/collection-gpu.yml | 6 + examples/collections.yml | 5 + .../collections/10-backends.yml | 6 +- .../collections/20-profile.yml | 6 +- .../collections/30-prompt.yml | 6 +- examples/config-fragments/sources.yml | 8 +- examples/config.collections.yml | 13 +- examples/config.discovery.yml | 4 +- examples/config.full.yml | 11 +- examples/config.inheritance.yml | 8 +- examples/config.yml | 11 +- examples/definitions.yml | 8 + examples/inheritance.yml | 6 + examples/overlays/collection.yml | 4 +- examples/overlays/local.yml | 2 - examples/promptkit.yml | 6 + examples/render-markdown-summary.sh | 7 +- examples/render-v0.9-features.sh | 13 +- go.mod | 2 +- internal/adapter/cli/application_config.go | 39 - internal/adapter/cli/config.go | 101 - internal/adapter/cli/config_test.go | 226 -- internal/adapter/cli/discovery_test.go | 31 +- internal/adapter/cli/examples_test.go | 42 + internal/adapter/cli/framework_config.go | 159 - internal/adapter/cli/framework_config_test.go | 391 --- internal/adapter/cli/framework_sources.go | 17 - .../adapter/cli/framework_sources_test.go | 386 --- internal/adapter/cli/inspect.go | 212 -- internal/adapter/cli/run.go | 636 ---- internal/adapter/cli/run_test.go | 2807 ----------------- internal/adapter/dependency_test.go | 2 +- internal/format/inspection.go | 179 -- internal/format/inspection_test.go | 95 - internal/format/prepared_run.go | 210 -- internal/format/prepared_run_test.go | 375 --- 52 files changed, 846 insertions(+), 7067 deletions(-) create mode 100644 docs/consumers/upgrading-cli.md create mode 100644 examples/collection-gpu.yml create mode 100644 examples/collections.yml create mode 100644 examples/definitions.yml create mode 100644 examples/inheritance.yml create mode 100644 examples/promptkit.yml delete mode 100644 internal/adapter/cli/application_config.go delete mode 100644 internal/adapter/cli/config.go delete mode 100644 internal/adapter/cli/config_test.go create mode 100644 internal/adapter/cli/examples_test.go delete mode 100644 internal/adapter/cli/framework_config.go delete mode 100644 internal/adapter/cli/framework_config_test.go delete mode 100644 internal/adapter/cli/framework_sources.go delete mode 100644 internal/adapter/cli/framework_sources_test.go delete mode 100644 internal/adapter/cli/inspect.go delete mode 100644 internal/adapter/cli/run.go delete mode 100644 internal/adapter/cli/run_test.go delete mode 100644 internal/format/inspection.go delete mode 100644 internal/format/inspection_test.go delete mode 100644 internal/format/prepared_run.go delete mode 100644 internal/format/prepared_run_test.go diff --git a/README.md b/README.md index dd2df81..bb012e9 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,9 @@ an HTTP service. It prepares prompt requests, runs them against OpenAI-compatible model endpoints, and returns generated output with validation metadata. +CommandKit supplies native configuration, resource discovery, preview, reporting, +and completion interfaces alongside application generation and HTTP startup. + The application uses [PromptKit v0.23.3](https://forge.maximumdirect.net/base-layer/promptkit/src/commit/804a4d32b23aa2e7973400bb0a322a7016574c9f/) for prompt, profile, schema, preparation, generation, and validation behavior. @@ -16,11 +19,10 @@ behavior, output presentation, and HTTP artifact-containment policy. From the repository root: ```bash -go run ./cmd/scriptorium render \ +go run ./cmd/scriptorium promptkit show prepared generic.markdown_summary \ --config ./examples/config.yml \ - --prompt generic.markdown_summary \ - --input transcript=./examples/fixtures/transcript.md \ - --input glossary=./examples/fixtures/glossary.yml \ + --input-file transcript=./examples/fixtures/transcript.md \ + --input-file glossary=./examples/fixtures/glossary.yml \ --format json ``` @@ -38,6 +40,7 @@ a model. For complete invocation and output behavior, see the - [Operations guide](docs/operations.md) - [Consumer integration overview](docs/consumers/api.md) - [Migration from the former Go package](docs/consumers/migrating-to-promptkit.md) +- [Native CLI migration](docs/consumers/upgrading-cli.md) - [Upgrade Promptkit source selectors and definitions](docs/consumers/upgrading-promptkit.md) - [Subprocess integration](docs/integrations/subprocess.md) - [Architecture policy](docs/policy/architecture.md) diff --git a/docs/cli.md b/docs/cli.md index dfca66a..1f140e9 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -1,377 +1,186 @@ # CLI Reference -This is the canonical contract for invoking Scriptorium. Configuration discovery, -precedence, application source locations, and server settings are defined in the -[configuration reference](config.md). The [HTTP API reference](api.md) owns -service request and response behavior. - -## Shortest Useful Command - -```bash -go run ./cmd/scriptorium render \ - --config ./examples/config.yml \ - --prompt generic.markdown_summary \ - --input transcript=./examples/fixtures/transcript.md \ - --input glossary=./examples/fixtures/glossary.yml -``` - -`render` prepares a request without calling an LLM. +Scriptorium uses CommandKit v0.7.0, ckconfig v0.9.0, and ckprompt v0.10.0. +This document owns application invocation and delivery behavior. The pinned +[upstream CLI contract][native-cli] owns standard family grammar and reports. +[Configuration](config.md) owns discovery, settings, and source composition; +the [HTTP API](api.md) owns the service contract. +Existing callers should read the [CLI migration guide](consumers/upgrading-cli.md). ## Commands -- `scriptorium run`: prepare a prompt, call the configured LLM, and write the - generated artifact. -- `scriptorium render`: prepare a prompt and write prepared-run output. -- `scriptorium serve`: start the HTTP server. -- `scriptorium inspect prompt`: inspect one prompt definition without model execution. -- `scriptorium inspect profile`: inspect one resolved profile without model execution. -- `scriptorium inspect configuration`: validate or inspect Promptkit's captured framework configuration. -- `scriptorium config validate`: validate application configuration without building an engine. -- `scriptorium config show`: show the resolved application configuration. -- `scriptorium config sources`: show the ordered configuration sources. -- `scriptorium config explain`: explain one effective or historical configuration path. - -All command families accept `--config ` and `--config-overlay `; -positional arguments are rejected. `--config-overlay` selects one application -configuration overlay; it is separate from `--profile`, which selects a -Promptkit execution profile. See [configuration composition](config.md#explicit-imports-and-selected-overlays) -for name, root, and absent-main behavior. `run`, `render`, `serve`, `inspect prompt`, -`inspect profile`, and `inspect configuration` accept repeated -`--promptkit-source ` flags and `--schema-dir ` where applicable. -They replace the configured source sequence in flag order. `--prompt-dir` and -`--profile-dir` are no longer supported; use `promptkit_sources` in application -configuration or the repeated source flags. Application `config` commands -remain independent of framework source loading. Explicit application flags, -including empty values, override file settings. Configuration load diagnostics -go to stderr and exit with status 1; successful load warnings are printed once -before command work. - -## `scriptorium run` - -```text -scriptorium run [flags] -``` - -Required flags: - -| Flag | Meaning | -| --- | --- | -| `--prompt ` | Prompt ID to execute. | - -Optional flags: - -| Flag | Meaning | -| --- | --- | -| `--config ` | Application configuration file. | -| `--config-overlay ` | Selected application configuration overlay. | -| `--promptkit-source ` | PromptKit source file or directory; repeat to select ordered sources. | -| `--schema-dir ` | Schema base-directory override. | -| `--profile ` | Execution-profile override. | -| `--session-id ` | Optional direct session identifier. | -| `--input name=path` | Optional input file mapping; repeat or use comma-separated mappings. | -| `--var name=value` | Template-variable mapping; repeat or use comma-separated mappings. | -| `--out ` | Write generated content to this file instead of stdout. | -| `--execution-report ` | Write an opt-in terminal execution report as newline-terminated JSON to a separate file. | -| `--llm-base-url ` | Runtime endpoint override. | -| `--model ` | Runtime model override. | -| `--api-key-env ` | Runtime API-key environment-variable name override. | -| `--temperature ` | Runtime temperature override. | -| `--max-tokens ` | Runtime maximum-token override. | -| `--top-p ` | Runtime top-p override. | -| `--reasoning-effort ` | Runtime reasoning-effort override. | -| `--timeout ` | Runtime timeout override using Go duration syntax. | -| `--run-timeout ` | Optional whole-run engine deadline using Go duration syntax; zero or omission adds no deadline, negative values are invalid. | -| `--transient-retries ` | Nonnegative additional transient attempt count for this request; `0` disables additional attempts. | - -Deprecated aliases: `--prompt-id` for `--prompt`, and `--profile-id` for -`--profile`. - -Omitted numeric runtime flags preserve the selected effective value; explicit -zero values override it. `--timeout 0s` disables the per-attempt generation deadline -only; the caller context and configured transport cap remain active. CLI -durations are converted to whole seconds by truncation toward zero, so any -duration whose absolute value is below one second becomes an explicit -zero-second override. The timeout layers are defined in the -[Promptkit outbound integration contract](https://forge.maximumdirect.net/base-layer/promptkit/src/commit/804a4d32b23aa2e7973400bb0a322a7016574c9f/docs/integrations/openai-compatible-chat.md#timeout-and-cancellation). -With the built-in client, each transient retry gets a fresh generation timeout; -this flag does not limit the complete run. -`--run-timeout` keeps subsecond precision and starts immediately before engine -execution. It bounds preparation, capacity waits, retry waits, generation, and -repairs where they honor context. Parsing, engine construction, and output -writes are outside that interval. A failed run identified by PromptKit as an -operation timeout exits 1 with a safe deadline diagnostic; a completed invalid -output still exits 2. Synchronous work may not stop immediately. -Omitting `--transient-retries` inherits the application setting or Promptkit's -current default of two additional attempts. The flag overrides only this -request; it does not alter the configured engine. Structural repair generations -have separate transient retry budgets. - -There is no raw API-key flag. Use `--api-key-env`. - -Prompt and profile IDs are passed unchanged, including collection-qualified -IDs. The removed `--prompt-version` flag is rejected. `--input` is optional -at the CLI boundary: PromptKit decides whether the selected definition requires -declared inputs or template-referenced values. - -`--session-id` supplies a direct, non-secret session identifier. A nonblank -value replaces a definition-rendered session ID; an omitted or blank value lets -the definition supply one. Promptkit trims direct values and limits them to 256 -Unicode code points. Use stable, non-sensitive identifiers because effective -session IDs may appear in prepared output, run metadata, and provider-facing -requests. - -`--reasoning-effort` is presence-aware: omitting it inherits the selected -profile value, a nonblank value replaces that value, and -`--reasoning-effort=` explicitly clears inherited reasoning. Promptkit treats -nonblank values as provider-specific opaque strings. - -### Execution report file - -`--execution-report` requires a nonempty path. It is available only for `run`; -`-` is an ordinary filename. Parent directories are not created. An existing -file is replaced through ordinary file writing, without rollback or an atomic -replacement guarantee. Generated content still goes to stdout or `--out`. -Equal output/report paths, including existing paths that refer to the same -file, are rejected before execution. - -The file is written after engine execution when Promptkit supplies a terminal -event, including for a failed run or invalid generated output. A failure before -invocation leaves the report path untouched. If no terminal event is supplied, -Scriptorium leaves the path untouched and emits a diagnostic; an existing file -may therefore contain an older report. Artifact output is attempted first on a -successful run, and report writing is attempted even if artifact writing fails. -Any output or report write failure exits 1; already-written output is not -undone. Without a write failure, the actual run keeps its usual exit status. - -The [HTTP API reference](api.md#execution-report) defines the shared report -fields and usage-presence semantics. The file contains that JSON object followed -by a newline. - -The report excludes content, credentials, URLs, headers, and raw errors, but -its identifiers can still be sensitive. The caller owns file retention and -access. - -## `scriptorium render` - -```text -scriptorium render [flags] -``` - -`--prompt ` is required. The following optional flags are supported: -`--config`, `--config-overlay`, repeatable `--promptkit-source`, `--schema-dir`, `--profile`, -`--input`, `--var`, `--out`, `--llm-base-url`, -`--model`, `--api-key-env`, `--temperature`, `--max-tokens`, `--top-p`, -`--reasoning-effort`, `--session-id`, `--timeout`, `--transient-retries`, and `--format text|json`. Their meanings match the corresponding -`run` flags; `--format` selects prepared-run output and otherwise uses -`defaults.render_format`. - -The same deprecated aliases and numeric/timeout behavior as `run` apply. -The same session and reasoning inheritance, replacement, and clearing behavior -also applies. -`render` validates the retry override but does not generate model output or -include a retry field in prepared output. -`render` resolves profiles and schemas as part of preparation but -does not call an LLM. - -To render from independently maintained collection checkouts: - -```bash -go run ./cmd/scriptorium render --config ./examples/config.discovery.yml \ - --prompt support/answer --profile support/fast \ - --input transcript=./examples/fixtures/transcript.md -``` - -The [discovery example](../examples/discovery/framework.yml) also provides -`review/answer`; see [configuration discovery](config.md#collection-checkout-discovery). - -## `scriptorium serve` - ```text +scriptorium run PROMPT_ID [flags] scriptorium serve [flags] +scriptorium config scopes +scriptorium config validate|show|sources|explain|diff [flags] +scriptorium promptkit list prompts|profiles|backends|namespaces|collections [flags] +scriptorium promptkit show prompt|profile|backend ID [flags] +scriptorium promptkit requirements show|validate PROMPT_ID [flags] +scriptorium promptkit show prepared PROMPT_ID [flags] +scriptorium promptkit test KIND [flags] +scriptorium help [COMMAND ...] +scriptorium version +scriptorium completion bash|zsh|fish ``` -Optional flags: +Native help is also available with `--help`. Help, version, scope discovery, +and static completion do not load configuration or contact a provider. +Version/revision metadata is supplied by the build; ordinary builds identify +themselves as `dev` with no revision. Release build mechanics belong to the +[release procedure](release.md). + +Commands that consume configuration accept `--config PATH` and `--overlay NAME`. +An explicitly empty config value remains explicit; overlay names follow +[ConfigKit selection](config.md#explicit-imports-and-selected-overlays). +There are no source/schema convenience flags or legacy command aliases. +Native shared `--format`, `--quiet`, `--verbose`, `--debug`, and progress +controls retain their upstream meanings. Diagnostics use stderr, not stdout. +Verbose exposes emitted Info and Warning messages without changing result bytes. + +## Generation + +`run` requires exactly one positional prompt ID. It has no duplicate prompt +flag or direct backend-selection flag; choose a profile for backend routing. +It uses these native bindings, with the preferred aliases below. Canonical +spellings prepend `llm-` (for example, `--llm-profile`). + +| Preferred flags | Meaning | +| --- | --- | +| `--profile ID`, `--session-id ID` | Profile selection and optional non-secret session context. | +| `--var KEY=VALUE` | Repeatable caller-variable mappings. | +| `--input-text NAME=TEXT`, `--input NAME=TEXT` | Repeatable literal input mappings; `--llm-input` is also a literal-text alias. | +| `--input-file NAME=PATH` | Repeatable file references. | +| `--endpoint URL`, `--model NAME` | Native endpoint/model overrides. | +| `--temperature N`, `--max-tokens N`, `--top-p N` | Presence-aware numeric overrides, including zero. | +| `--timeout-seconds N` | Integer per-attempt generation timeout; zero disables that deadline, not caller cancellation or the transport cap. | +| `--reasoning-effort VALUE`, `--service-tier VALUE` | Native provider controls; an explicit empty reasoning value is preserved. | +| `--api-key-env NAME` | Credential environment-variable name, never raw key bytes. | +| `--extra-params JSON` | Native JSON-object provider parameters. | +| `--transient-retries N` | Request retry override; omission leaves the engine default. | +| `--prompt-namespace NAME`, `--profile-namespace NAME`, `--backend-namespace NAME` | Independent native namespace views. Qualified resource IDs remain exact; stored references are not rewritten. | + +Mappings split at the first `=`; commas and subsequent equals signs are literal. +Repeat flags for multiple mappings. The last text/file occurrence for the same +input wins in command-line order. PromptKit validates selected definitions, +inputs, namespaces, provider settings, and credential requirements; Scriptorium +does not add an execution-readiness preflight or configurable default namespace. +Native string/pointer override semantics remain upstream-owned. + +Application-owned generation controls: | Flag | Meaning | | --- | --- | -| `--config ` | Application configuration file. | -| `--config-overlay ` | Selected application configuration overlay. | -| `--addr ` | HTTP listen-address override. | -| `--promptkit-source ` | PromptKit source file or directory; repeat to select ordered sources. | -| `--schema-dir ` | Schema base-directory override. | -| `--artifact-root ` | Root for HTTP `file` input references. | -| `--max-request-bytes ` | Maximum encoded HTTP request-body bytes; `0` disables the limit. | -| `--max-artifact-bytes ` | Maximum HTTP file-input artifact bytes; `0` disables the limit. | -| `--max-response-bytes ` | Maximum encoded HTTP response bytes; `0` disables the limit. | +| `--run-timeout DURATION` | Go duration passed unchanged to PromptKit's operation budget. Positive values cover engine preparation/execution, including cooperative waits and repairs. Zero, omission, and negative values add no budget. Parsing, configuration/engine setup, and delivery are outside it. | +| `--report PATH` | Additionally write the returned native result to a separate file. | +| `--report-format text\|yaml\|json` | Explicit separate-report format; omission leaves native automatic rendering selected. | -`serve` accepts no runtime model override flags. HTTP request fields, response -schemas, and error codes are defined in the [HTTP API reference](api.md). +The operation budget is independent of the integer per-attempt timeout. +Earlier parent cancellation/deadlines still apply. Cancellation is cooperative; +synchronous work that ignores context cannot be forcibly interrupted. -## `scriptorium inspect prompt` +### Results And Separate Reports -```text -scriptorium inspect prompt --prompt ID - [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--schema-dir DIR] [--format text|json] [--out PATH] -``` +The main report goes through the invocation reporter to stdout. Automatic/text +output is exactly `Artifact.Body`, with no added newline, header, escaping, or +summary. JSON/YAML output is the full native PromptKit `RunResult`, without a +Scriptorium envelope. In native structured serialization, artifact bytes use +the upstream representation (JSON base64), not an application text projection. -`--prompt` is required. Inspection uses normal configuration discovery and the -selected Promptkit sources, defaults to text regardless of -`defaults.render_format`, and writes to stdout unless `--out` is supplied. +Save main output with shell redirection or the subprocess stdout destination. +There is no `--output` or `--out` flag. Omit `--quiet` when capturing the main +result: native quiet suppresses it even when stdout is redirected. -Both definition-inspection commands first construct the same engine used for -execution from the complete selected source sequence. Construction resolves -and validates all effective definitions, including unused ones, and captures -referenced message-file contents and static template syntax. Invalid unused -definitions or unreadable message resources can therefore prevent inspection. -No prompt collection is mandatory, but selected prompts are not ignored. +`--report` renders the same result through CommandKit without the artifact-only +text template. Its format is independent of stdout's `--format` and the filename +extension. Omission passes an unset format to the upstream renderer; it currently +renders structured values as YAML in automatic/text mode. Choose an explicit +format when required rather than relying on that upstream default. -Inspection does not prepare an execution request, load request-time schemas or -artifacts, resolve credential values, reserve backend capacity, or call a model. -Prompt inspection does not select an execution profile or render request-specific -templates. -Prompt output includes ID, hash, optional default profile and input metadata, -and the output contract. Absent optional values are omitted; explicitly empty -or zero values are retained. Captured message text is not projected. +The requested file is written even in quiet mode. Omission creates no separate +report. An explicitly empty path is a usage error before generation; `-` is an +ordinary filename. New files use mode 0644 subject to umask, existing files use +normal OS replacement behavior, and parent directories are not created. +Render, write, and close failures are reported. -## `scriptorium inspect profile` +Only a returned result is eligible for delivery. A resultless failure leaves an +existing report untouched; it does not synthesize an invocation-error result. +Main delivery is attempted first, then the requested report even if main +delivery fails. Writes are nontransactional; bytes already written remain. +Callers own redirected paths, retention, and keeping main and report destinations +separate. A shell may truncate its redirected file before the process starts. -```text -scriptorium inspect profile --profile ID - [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--schema-dir DIR] [--format text|json] [--out PATH] -``` +## Application Configuration Inspection -`--profile` is required. After the [full-source engine construction described -above](#scriptorium-inspect-prompt), inspection selects one resolved profile -with inheritance. `profile_model_params` contains only fields present on that -profile; it does not fill backend defaults. `api_key_required` is a separate -optional field. Explicit zero and empty values remain visible. Use `render` -for request-completed model settings. +The native `config` family defaults to `app`; select `--scope promptkit` for +captured framework configuration or use native scope discovery/all-scope modes. +`app` uses the actual completed ConfigKit result and does not construct an engine. -## Application configuration inspection +Native `--set POINTER=JSON` is enabled only for `app`. Records go directly to +ConfigKit: duplicate and ancestor/descendant-overlapping paths are rejected, +not resolved by flag order. No application-specific configuration flags or +custom `--against-*` counterparts are installed. -```text -scriptorium config validate [flags] -scriptorium config show [flags] -scriptorium config sources [flags] -scriptorium config explain --path JSON_POINTER [flags] -``` +Native diff uses independent `--against-config`, `--against-overlay`, and +supported `--against-set` selections. Omitted right-side values do not inherit +the left side. See the [upstream contract][native-cli] for focused paths, +metadata discovery, source IDs, findings, explanations, and detailed comparisons. -Each accepts `--config`, `--config-overlay`, `--schema-dir`, `--addr`, -`--artifact-root`, `--max-request-bytes`, -`--max-artifact-bytes`, and `--max-response-bytes`. These are application -configuration selectors and overrides; request, credential, backend-editing, -output-format, and output-file flags are not accepted. Positional arguments -are rejected. +## Framework Configuration Inspection -`validate` writes no success output; status 0 means the application configuration -loaded and passed its own validation. It does not establish that selected -Promptkit sources can construct an engine, credentials are present, or a -provider is available. `show` writes ConfigKit's resolved YAML snapshot. `sources` -writes one newline-terminated JSON array of source records in load order. Each -record has `id`, `label`, and `layer`, plus `path` when available. Application -main-file and import records use the `main` layer; explicit CLI -overrides appear as synthetic sources. Defaults-only input without overrides -produces `[]` for `sources`. +The `promptkit` config scope loads application settings selected by `--config` +and `--overlay`, constructs an engine, and exposes its captured native view. +It does not accept temporary overrides; native capability checks also apply +when an all-scope operation requests them. -`explain` requires an explicitly supplied `--path` in JSON Pointer syntax. -The empty string selects the root; `/server/addr` selects a field; `~1` and -`~0` escape `/` and `~` inside mapping keys. Numeric tokens are interpreted -according to the containing mapping or sequence. Dotted paths, file paths, -and URL fragments are not accepted. ConfigKit writes a human-readable account -of the effective value and its chronological default, file/import, overlay, -CLI, normalization, and removed-value history. This is not a JSON schema or a -byte-for-byte stable prose format. Unknown and malformed paths exit 1 without -successful explanation output. +Framework effective/explanation views retain native classified-field redaction, +including history. Environment-reference names and ordinary values remain +visible. This is not arbitrary secret detection and does not redact all prompt, +prepared, application, or generated content. Protect reports accordingly. -Diagnostics and warnings go to stderr, never into successful stdout. Parse, -load, output, and diagnostic-write failures exit 1. Ordinary output writes may -leave partial data on failure. Application inspection can contain paths and -configured values; protect its output accordingly. Raw API-key values are not -accepted by the application configuration schema. +## Discovery, Preview, And Explicit Smoke Testing -## Framework configuration inspection +Inventories retain native defaults and `--namespace`/`--all-namespaces` filters. +Named inspections expose native prompt messages and profile/backend values; +prepared preview exposes rendered content, not a summary projection. -```text -scriptorium inspect configuration validate [flags] -scriptorium inspect configuration show [flags] -scriptorium inspect configuration sources [flags] -scriptorium inspect configuration explain --path JSON_POINTER [flags] -``` +Requirements discovery works before input artifacts exist and without schema +loading, credential resolution, message rendering, or provider access. It still +loads configuration and captured message resources. Reports preserve exact names, +metadata, variable presence (including empty values), completeness, and issue +codes. Partial discovery may succeed with status zero; completeness is not +binding satisfaction or execution readiness. Concrete validation and preview +keep their native preparation-time prerequisites and make no generation call. -Each operation accepts `--config`, `--config-overlay`, repeatable -`--promptkit-source`, and `--schema-dir`. These flags select the application -settings and framework sources; positional arguments, `--format`, and `--out` -are not accepted. Unlike `config validate`, framework `validate` constructs -the Promptkit engine and validates all effective definitions, including -unused ones. It writes nothing on success and does not prepare a request, -resolve a credential, or contact a provider. +Smoke-test kinds are `reachability`, `authentication`, `text_inference`, +`structured_output`, or `all`, with native aliases as documented upstream. +These explicit commands may contact providers and incur cost. Their presence +does not add a startup/provider probe. Read the native findings: unsuccessful or +unsupported checks are not automatically invocation failures. -`show` writes Promptkit's native effective YAML, an inspection snapshot rather -than a promised reloadable file. `sources` writes a -newline-terminated JSON array in captured source order. Each source record has -`id`, `label`, `layer`, `kind`, and `underlay_index`, plus `path` when available -and `imported_from` for imported sources. That nested object has `source_id`, -`line`, and `column`. Values and output structure are a projection of the -captured Promptkit view, not Scriptorium's application-source records. +## Server And Statuses -`explain` requires an explicitly supplied `--path` in ConfigKit JSON Pointer -syntax; `--path ''` selects the root. For example, -`/collections/support/profiles/0/model` addresses the first support profile's -model in the captured list. Escape `/` as `~1` and `~` as `~0`; dots in a key -remain literal. Malformed and unknown pointers exit 1 without a successful -report. Explanation prose is native and is not a stable machine-readable -format. +`serve` accepts configuration selection and presence-aware HTTP controls: +`--addr`, `--artifact-root`, `--max-request-bytes`, `--max-artifact-bytes`, +and `--max-response-bytes`. It has no generation request controls. Values and +defaults belong to [configuration](config.md). -Framework `show` and `explain` use Promptkit's native redaction for its -classified credential-shaped provider fields, including historical values. -Environment-variable reference names and ordinary provider parameters remain -visible. Arbitrary prompt or provider content is not classified as secret by -name inspection, so treat these reports as sensitive. This redaction policy is -limited to framework configuration reports; it does not add redaction to -`inspect profile` or prepared-run output. Credentials are not resolved by any -of these inspection operations. Promptkit owns detailed field and source -semantics; see its tagged [format reference](https://forge.maximumdirect.net/base-layer/promptkit/src/commit/804a4d32b23aa2e7973400bb0a322a7016574c9f/docs/formats.md). - -Warnings and errors go to stderr. Successful `validate` writes nothing to -stdout; successful `show`, `sources`, and `explain` write their reports there. -Parse, load, path, and output failures exit 1. A native writer error may leave -partial output. - -## Input And Variable Syntax - -`--input name=path` maps an input name to a local file; `--var name=value` -maps a template variable to a string. Both flags can be repeated or contain -comma-separated mappings. Values may contain `=` after the first separator. -Empty names and values are rejected. - -CLI inputs are file references. HTTP inline inputs are defined by the -[HTTP API reference](api.md). - -## Output And Exit Behavior - -- `run` writes generated content to stdout, or to `--out` when supplied, and - writes a concise summary to stderr. The summary includes `backend=` when - Promptkit selected a backend; endpoint-only profiles omit it. -- `render` writes prepared-run output to stdout, or to `--out` when supplied, - without a success summary. Text output includes `selected_backend_id` after - `selected_profile_id` when Promptkit selected one; endpoint-only profiles - omit it. -- `serve` writes startup and server errors to stderr. - -Exit statuses: +Interrupt/SIGTERM cancels the invocation context. The server closes its listener +and active connections; canceled shutdown returns operational status, with no +graceful-drain deadline or guarantee that context-ignoring work has finished. | Status | Meaning | | --- | --- | -| `0` | Success. | -| `1` | Parse, configuration, loading, rendering, generation, output-write, or other runtime error. | -| `2` | `run` generated and wrote output, but validation failed. | +| 0 | Native success, subject to successful presentation/delivery. | +| 1 | Operational failure, including lookup/preparation/generation/capacity, cancellation/timeouts, unclassified load errors, and delivery failure for an otherwise successful run. | +| 2 | Native usage/declaration-input errors, or configuration errors classified as invalid input by native diagnostic evidence. | +| 3 | Native `config diff --check` found differences. | +| 4 | Completed generation returned `ValidationFailed`; artifact and native result remain deliverable. | -A backend admission rejection is a runtime error and prints `run error: model -backend capacity is exhausted`. The HTTP capacity response is defined in the -[HTTP API reference](api.md). +Native finalization handles presentation errors: success/usage become operational; +an already nonzero domain outcome retains its classification/status. A report-file +failure preserves completed-validation status 4 while adding its explanation and +cause. An operational failure to perform validation is status 1, not status 4. +JSON selection does not turn failures into a separate JSON error envelope. -## Workflows And Examples - -The [maintained render script](../examples/render-markdown-summary.sh) is a -copyable render workflow. The [HTTP request example](../examples/http-run.json) -is for a running `serve` process. +[native-cli]: https://forge.maximumdirect.net/base-layer/commandkit/src/commit/d330d7637f2e7507eca931a899397222c82bc892/docs/cli.md diff --git a/docs/config.md b/docs/config.md index 3ec56a1..fd6abf9 100644 --- a/docs/config.md +++ b/docs/config.md @@ -1,87 +1,126 @@ # Configuration Reference -This is the canonical reference for Scriptorium application settings. Prompt, -profile, backend, schema, execution-setting, built-in profile, and framework -credential semantics are defined by the -[PromptKit v0.23.3 format reference](https://forge.maximumdirect.net/base-layer/promptkit/src/commit/804a4d32b23aa2e7973400bb0a322a7016574c9f/docs/formats.md). -For command syntax, see the [CLI reference](cli.md); for HTTP request shapes and -outcomes, see the [HTTP API reference](api.md). For migration examples, see -[Upgrading Promptkit sources and definitions](consumers/upgrading-promptkit.md). +This document owns Scriptorium's application settings, discovery, and source +selection. [CLI syntax](cli.md) and the [HTTP API](api.md) have separate owners. +Framework definitions and execution semantics belong to the pinned +[PromptKit v0.23.3 formats][formats]; ConfigKit v1.2.1 owns +[native composition and inspection][configkit]. ## Discovery And Precedence -ConfigKit assembles the selected main source, imports, and overlay, resolves -inheritance, and decodes the result with Scriptorium's typed defaults. It then -applies explicitly supplied application CLI values, records application -normalization, and validates the result. Source, composition, inheritance, and -decoding failures happen before CLI assignment and cannot be repaired by it; -an override can correct a decoded value before application validation. +An explicit `--config` selects that file, including an explicitly empty value +(which fails rather than falling back). With no selection, ConfigKit searches +these native presets in order, selecting one main file: -When `--config` is omitted, Scriptorium searches -`/usr/local/etc/scriptorium/config.yml` and then `/etc/scriptorium/config.yml`. -ConfigKit performs this ordered search for each load. If neither exists, it -uses built-in defaults; an error in a selected candidate does not cause a -fallback. An explicit `--config` path must exist and decode successfully, -including when an empty path is explicitly supplied. +| Preset | Directories in order | Filename groups | +| --- | --- | --- | +| WorkingDirectory | Current working directory | Named only | +| UserConfigDirectory | Go user-config directory / `scriptorium`, then the user-config directory | Named then Generic in the application directory; Named only in the base | +| HomeDirectory | Go home directory / `.scriptorium`, then the home directory | Named then Generic in the application directory; Named only in the base | +| UnixSystemDirectories | `/usr/local/etc/scriptorium`, `/etc/scriptorium`, then `/etc` | Named then Generic in application directories; Named only in `/etc` | -The maintained [minimal configuration](../examples/config.yml), -[complete configuration](../examples/config.full.yml), -[composed configuration](../examples/config.composed.yml), -[collection configuration](../examples/config.collections.yml), and -[framework inheritance configuration](../examples/config.inheritance.yml) are -copyable examples. +Named means `scriptorium.yml`, then `scriptorium.yaml`. Generic means +`config.yml`, `config.yaml`, `main.yml`, then `main.yaml`. +ConfigKit uses native Go directory resolution (including its user-config/home +environment behavior) and checks every filename in one directory before the next. +Candidates are not configuration layers. Optional exhaustion uses application +defaults; a selected-file acquisition/decoding error is terminal. -## Application Configuration File +ConfigKit assembles the main, enabled imports, and selected overlay, applies its +inheritance/composition rules, creates typed defaults and strictly decodes, applies +explicit override records, then runs application normalization and validation. +An override can correct a decoded value before validation, but cannot repair +source/composition/decoding errors. The completed result is authoritative: +runtime settings conversion does not merge, default, or normalize again. -Application configuration is decoded by ConfigKit v1.2.1 as strict YAML. -Unknown fields, empty files, multiple YAML documents, and YAML aliases/anchors -are rejected. -An explicit empty string is a value and may clear a file-provided optional -string. A scalar `null` is not an empty string and is rejected for non-nullable -fields; nullable fields such as `transient_retries` retain their documented -meaning. Raw API-key fields and unknown application fields are rejected. +## Application Settings -Only explicitly supplied application CLI flags override decoded settings. Paths -in the application document remain relative to the process working directory, not to the file's -directory. Load diagnostics are printed safely on stderr; invalid application -configuration exits with status 1. - -### Promptkit source selectors - -`promptkit_sources` is an ordered sequence of explicit Promptkit source -selectors. Each item must set exactly one of `file` or `directory`: +Complete copyable configurations are maintained under +[examples](../examples/config.yml), including [full settings](../examples/config.full.yml), +[application imports](../examples/config.composed.yml), +[framework fragment composition](../examples/config.collections.yml), +[collection checkout discovery](../examples/config.discovery.yml), and +[framework inheritance](../examples/config.inheritance.yml). ```yaml -promptkit_sources: - - directory: ./examples/prompts - - directory: ./examples/profiles - - file: ./examples/framework/backends.yml -schema_dir: ./examples/schemas +promptkit: + configuration: ./examples/promptkit.yml + schema_dir: ./examples/schemas ``` -An empty or null final resolved `promptkit_sources` list selects no -application-supplied framework sources; PromptKit's bundled OpenRouter underlay -still applies. A later sequence item overlays an earlier -one. Files within a selected directory are composed together, so conflicting -definitions in that group fail instead of using filename order as precedence. -An explicit file may use ConfigKit imports. The complete framework format and -source rules belong to the [PromptKit v0.23.3 format reference](https://forge.maximumdirect.net/base-layer/promptkit/src/commit/804a4d32b23aa2e7973400bb0a322a7016574c9f/docs/formats.md). +The `promptkit` section contains acquisition settings, not embedded framework +definitions. Profiles, prompts, backends, and collections belong in the separate +framework document. The retired `promptkit_sources`, top-level `schema_dir`, +and `defaults.render_format` fields have no compatibility conversion and are +rejected by strict decoding, as are unknown/raw credential fields. -Selector paths are relative to the process working directory, not to the -application configuration file. Application validation checks the selector -shape but does not open or validate Promptkit sources. `config validate` -therefore does not prove that selected sources can build an engine; use -`inspect configuration validate` for that separate check. Repeated -`--promptkit-source` flags replace the entire configured selector sequence -and preserve flag order. Without an authored collection policy, a selected -overlay replaces `promptkit_sources` as a whole; a declared sequence policy -may instead combine the lists. +| Field | Default | Meaning | +| --- | --- | --- | +| `promptkit.configuration` | empty | One external framework file; omission/empty selects no external main. | +| `promptkit.schema_dir` | `.` | Schema base directory passed to PromptKit. | +| `transient_retries` | upstream default | Optional nonnegative engine retry count. Omitted/null delegates upstream; zero disables additional transient attempts. | +| `server.addr` | `:8080` | HTTP listen address. | +| `server.artifact_root` | empty | Root enabling HTTP file-artifact references. | +| `server.max_request_bytes` | `16777216` | Encoded HTTP request limit; zero is unlimited. | +| `server.max_artifact_bytes` | `16777216` | HTTP file-artifact limit; zero is unlimited. | +| `server.max_response_bytes` | `16777216` | Encoded HTTP response limit; zero is unlimited. | -### Collection checkout discovery +Size fields are nonnegative signed 64-bit values. [HTTP behavior](api.md) defines +their enforcement; [operations](operations.md) covers safe deployment choices. -Select a framework discovery file using the existing file selector, as in -[config.discovery.yml](../examples/config.discovery.yml). Its native imports -select both supported collection-entry spellings: +ConfigKit strictly decodes application YAML: empty files, unknown fields, +multiple documents, aliases/anchors, and incompatible/null values fail under its +schema rules. Explicit empty strings and numeric zeros remain values. The +nullable retry field preserves null. Application paths are trimmed/cleaned and +the server address is trimmed within ConfigKit's resolver, with native +normalization evidence. Paths remain relative to process cwd, not the application +file's directory. No general environment-variable override layer is installed. + +## Explicit Imports And Selected Overlays + +Application imports use native root `config.imports` and are enabled during +loading. ConfigKit owns paths/directories/globs, requiredness, destination/item +placement, inheritance, sequence policy, and source-root confinement; consult its +[pinned contract][configkit]. Scriptorium does not widen the selected file's +canonical source root or implement another traversal/merge policy. + +`--overlay NAME` passes the name unchanged to ConfigKit. At the selected release, +empty and exact lowercase `default` select none; other names use letters, +digits, underscores, or hyphens and select `overlays/NAME.yml` beneath the main +source root. A missing selected overlay fails. It is not a filesystem-path flag, +inherited stack, or fallback-root mechanism. An active overlay after absent-main +search exhaustion has no independent filesystem root in this application. + +Only `serve` has named application override flags, for its HTTP settings. +Native application-scope `--set` records are passed directly to ConfigKit. +Nonconflicting records are supported; duplicate and ancestor/descendant paths +are rejected rather than resolved by order. Configuration commands have no +application-defined source/schema/HTTP selectors or comparison counterparts. +Native comparison selections and supported overrides are independent on each side. + +## Framework Source Mapping + +A nonempty resolved `promptkit.configuration` becomes an import-enabled +`configkit.File` main input to PromptKit. An empty value leaves its main absent. +The native bundled OpenRouter underlay remains enabled; no application source +list, directory expansion, or local precedence interpreter is installed. + +Imports in that external document compose framework definitions using PromptKit's +schema and ConfigKit's native rules. The [maintained entry file](../examples/promptkit.yml) +imports the application's maintained prompt/profile fragments and backend +definitions. Peer conflicts are errors, not filename-order precedence. +Native whole-record replacement and inheritance remain framework-owned. +Application settings do not embed or re-decode framework definitions. + +PromptKit captures message-file resources with their defining source context +when constructing the engine. Schema/artifact reads, credential resolution, and +provider work retain native preparation/runtime lifecycles. The server keeps one +engine until restarted; later CLI invocations perform fresh loads. + +### Collection Checkout Discovery + +The [discovery configuration](../examples/config.discovery.yml) selects +[one framework entry](../examples/discovery/framework.yml) with two optional imports: ```yaml config: @@ -90,195 +129,30 @@ config: - glob: collections/*/collection.yaml ``` -Both globs are optional, so either spelling may have no matches. Paths are -relative to the framework file; this example's `*` selects one checkout -level and is not a general layout restriction. Each entry declares its own -collection namespace and imports only its intended resource fragments. The -checkout directory name does not determine the namespace. New checkouts are -available on the next invocation without editing this file; a running server -does not reload them. See the [maintained collection examples](../examples/discovery/collections/). +Each independently maintained checkout declares its namespace and explicitly +imports its own definitions relative to its entry file. Checkout names do not +determine resource IDs. The [support example](../examples/discovery/collections/support-repo/collection.yml) +imports multiple profiles/prompts and a source-relative message; the +[review example](../examples/discovery/collections/review-repo/collection.yaml) +uses the other spelling. Arbitrary checkout YAML is not loaded implicitly. -Entries imported by one discovery file are composed together as peers. -Conflicting definitions of the same qualified resource ID fail to load; -filename, checkout, and glob order do not set precedence. To override a -discovered record, select a separate framework file later in -`promptkit_sources` and supply the complete replacement record. +Either glob may have no matches. New checkouts become visible on a later +invocation, not through server hot reload. Entries in one discovery document are +peers: conflicting qualified IDs fail instead of relying on glob order. -### Explicit imports and selected overlays +## Inspection And Credentials -Application files declare imports under root `config.imports`. Native `path`, -`dir`, and `glob` selectors, filters, recursion, destinations, and item -placement are available as described in the [pinned ConfigKit contract](https://forge.maximumdirect.net/base-layer/configkit/src/commit/51ef3144158db1a09fabdbd1c7ae59908c84923b/docs/config.md). -Imports are optional by default; set `required: true` when absence must fail. -The old `optional` field is not supported. Directory selection has no default -filename filter; author a pattern when only YAML files are intended. ConfigKit -composes peer sources strictly, rather than using filename order to resolve -conflicts. Imports are resolved relative to their declaring file within the -native source root. Scriptorium does not widen that root. +The native `app` scope exposes the actual application result without acquiring +framework definitions or credentials. The `promptkit` scope exposes the captured +framework result selected by the application config/overlay. They are not merged +into one synthetic document. See [CLI inspection](cli.md#application-configuration-inspection) +for operations and independent comparison. -Root `config.sequences` can declare an identity key and merge mode for a -collection. Scriptorium does not register or infer an identity for -`promptkit_sources`: absent an authored policy, unkeyed lists replace across -precedence boundaries. The [collection example](../examples/config.collections.yml) -declares `file` identity with `upsert`, combines imported file selectors, and -lets an overlay add another. CLI list overrides still replace the whole list. +PromptKit owns profiles, namespaces, provider controls, repairs, retries, and +credential resolution. Store credential environment-variable names, not raw +values; supply the selected environment to the process. Native framework +redaction is limited to classified fields and history, not arbitrary sensitive +content. Protect application, prompt, prepared, and generated reports as data. -At application precedence boundaries, omission preserves lower-precedence -selectors. Under ordinary list replacement, `[]` clears the selection and -`null` resolves to no selectors. Under authored `append` or `upsert`, `[]` -contributes no records and `null` is invalid. To clear a collection with an -inherited policy, declare `mode: replace` at -`destination: [promptkit_sources]` in the higher-precedence source and supply -`promptkit_sources: []`. See -[ConfigKit's collection policy contract](https://forge.maximumdirect.net/base-layer/configkit/src/commit/51ef3144158db1a09fabdbd1c7ae59908c84923b/docs/config.md#yaml-declarations-and-policy-lifetime) -for policy lifetime and combination rules. - -These application root `config` controls do not change the separate -PromptKit v0.23.3 framework format: its selected framework files use the -[Promptkit contract](https://forge.maximumdirect.net/base-layer/promptkit/src/commit/804a4d32b23aa2e7973400bb0a322a7016574c9f/docs/formats.md) -and their own native ConfigKit `config` controls. - -`--config-overlay NAME` selects one file at `overlays/NAME.yml` below that -root. The name must use letters, digits, `_`, or `-`; it is not a path. Empty -and exact lowercase `default` select no overlay. There is no fallback, -case folding, inherited stack, or automatic selection; a missing selected -overlay fails. Mappings merge; sequence contributions and nulls at collection -destinations follow the effective collection policy described above. Other -explicit nulls replace lower values subject to the application schema. -Defaults-only loading can enable imports, but an absent main has no import -declarations. This CLI supplies no independent overlay root, so an active -overlay cannot be located without a selected main context. That is not a general -ConfigKit restriction. - -The [composed example](../examples/config.composed.yml) imports the application -[source-selection fragment](../examples/config-fragments/sources.yml) and has -a selectable [local overlay](../examples/overlays/local.yml). That fragment -selects the [framework backend file](../examples/framework/backends.yml) through -`promptkit_sources`; backend definitions are not imported into the application -document. From the repository root: - -```bash -go run ./cmd/scriptorium render \ - --config ./examples/config.composed.yml --config-overlay local \ - --prompt generic.markdown_summary \ - --input transcript=./examples/fixtures/transcript.md \ - --input glossary=./examples/fixtures/glossary.yml -``` - -Inspect the collection overlay and render without contacting a provider: - -```bash -go run ./cmd/scriptorium config show --config ./examples/config.collections.yml --config-overlay collection -go run ./cmd/scriptorium render --config ./examples/config.collections.yml --config-overlay collection \ - --prompt generic.markdown_summary \ - --input transcript=./examples/fixtures/transcript.md \ - --input glossary=./examples/fixtures/glossary.yml -``` - -Import and overlay file locations are constrained by ConfigKit; application -path values such as `promptkit_sources`, `schema_dir`, and `server.artifact_root` -remain relative to the process working directory even when authored in a -fragment or overlay. These source constraints are separate from Promptkit -message-resource paths and HTTP artifact containment. - -| Field | Default | Meaning | -| --- | --- | --- | -| `promptkit_sources` | empty sequence | Ordered file/directory sources for Promptkit backends, profiles, and prompts. | -| `schema_dir` | `.` | Promptkit schema source directory for relative schema paths. | -| `server.addr` | `:8080` | Address used by `serve`. | -| `server.artifact_root` | unset | Root that enables HTTP `file` input references. | -| `server.max_request_bytes` | `16777216` | Maximum encoded HTTP request-body bytes; `0` disables the limit. | -| `server.max_artifact_bytes` | `16777216` | Maximum HTTP file-input artifact bytes; `0` disables the limit. | -| `server.max_response_bytes` | `16777216` | Maximum encoded HTTP response bytes; `0` disables the limit. | -| `defaults.render_format` | `text` | Default prepared-run output format: `text` or `json`. | -| `transient_retries` | Promptkit default | Optional nonnegative count of additional built-in transient generation attempts. Omitted or `null` delegates to Promptkit; `0` disables additional attempts. | - -The size fields must be zero or greater. The [HTTP API](api.md) defines how -each limit is enforced and reported. `server.artifact_root` configures an HTTP -deployment boundary; see [operations](operations.md) for deployment handling. - -The `config validate`, `config show`, `config sources`, and `config explain` -commands inspect this application configuration without creating a Promptkit -engine. `show` is a resolved inspection snapshot, not necessarily a reloadable -file. Application validation does not check framework sources, backend -connectivity, or credential availability. Use `inspect configuration` for -Promptkit's separately captured framework configuration; its four operations -are documented in the [CLI reference](cli.md#framework-configuration-inspection). -`config explain --path /server/addr` uses JSON Pointer syntax to show the -selected field's current value and history from defaults, files, overlays, -explicit CLI overrides, and normalization. `--path ''` selects the root. -History includes replaced and removed values; `show` presents the final -resolved snapshot. The application schema rejects raw credential fields, but -do not treat inspection as a general-purpose filter for sensitive strings. - -## Framework Source Mapping - -Scriptorium assembles selected Promptkit source files and directories in the -order described above, then constructs the Promptkit engine. Framework -definitions and custom backends belong in those selected source files, not in -the Scriptorium application configuration. Message `content_file` paths use -the defining source's resource context and are captured when the engine is -constructed; schema and request-artifact loading remain preparation-time -operations. Scriptorium does not redefine or independently parse framework -file formats. - -- Prompt selection by ID, message templates, inputs, output contracts, and - session IDs are Promptkit contracts. -- Profile fields, numeric ranges, execution defaults, overlay precedence, - built-in profiles, and credential rules are Promptkit contracts. -- Schema path behavior and generated-content validation are Promptkit - contracts. - -See the -[tagged Promptkit format reference](https://forge.maximumdirect.net/base-layer/promptkit/src/commit/804a4d32b23aa2e7973400bb0a322a7016574c9f/docs/formats.md) -for all of those definitions. The files under -[`examples/prompts`](../examples/prompts/), -[`examples/profiles`](../examples/profiles/), and -[`examples/schemas`](../examples/schemas/) are maintained Scriptorium -application inputs using that tagged format. - -PromptKit supplies a default-on OpenRouter bundle as the lowest-priority -underlay. Choose an actual bundled profile such as `auto`, or a complete -application-owned profile; old catalog IDs are not aliases. Caller definitions -can replace bundled records by normal composition. Bundle availability is not -a readiness check or a promise of a fixed provider model. - -## Credentials And Outbound Behavior - -Scriptorium maps `--api-key-env` and HTTP `model.api_key_env` into Promptkit -request overrides. Keep secret values in environment variables and store only -their names in configuration or framework source files. Do not place raw keys -in configuration, prompts, profiles, CLI arguments, examples, or HTTP -payloads. - -An optional `api_key_env` whose environment value is absent or empty can result -in an unauthenticated provider request. A profile that declares credentials -required still fails as an invalid request when no credential source is -selected, and fails with `ErrAPIKeyEnvMissing` when its selected environment -source is absent or empty. Scriptorium never reads or emits the environment -value itself. - -Unset optional provider controls are omitted from compatible provider requests. -A positive Promptkit `repair_attempts` budget can add provider calls, latency, -token use, and cost; see the tagged format reference for its permitted values -and validation-mode requirements. - -Promptkit's built-in client makes up to two additional attempts for eligible -transient HTTP statuses by default. The application `transient_retries` setting -changes that budget for its constructed engine; an explicit `run` request -override takes precedence. This can increase latency, request count, and cost. -Each initial or repair generation has its own transient budget; an injected -client retains its own retry behavior. The tagged integration contract owns -eligibility and timing. - -Promptkit's -[OpenAI-compatible integration contract](https://forge.maximumdirect.net/base-layer/promptkit/src/commit/804a4d32b23aa2e7973400bb0a322a7016574c9f/docs/integrations/openai-compatible-chat.md) -defines outbound authentication, provider request mapping, transport limits, -and timeout layering. - -## Related References - -- [CLI reference](cli.md) -- [HTTP API reference](api.md) -- [Operations guide](operations.md) -- [Promptkit framework formats](https://forge.maximumdirect.net/base-layer/promptkit/src/commit/804a4d32b23aa2e7973400bb0a322a7016574c9f/docs/formats.md) +[formats]: https://forge.maximumdirect.net/base-layer/promptkit/src/commit/804a4d32b23aa2e7973400bb0a322a7016574c9f/docs/formats.md +[configkit]: https://forge.maximumdirect.net/base-layer/configkit/src/commit/51ef3144158db1a09fabdbd1c7ae59908c84923b/docs/config.md diff --git a/docs/consumers/api.md b/docs/consumers/api.md index f74c9c2..89035c2 100644 --- a/docs/consumers/api.md +++ b/docs/consumers/api.md @@ -11,6 +11,7 @@ and the HTTP service according to the boundary your application needs. - CLI subprocess: [subprocess integration](../integrations/subprocess.md) - HTTP service: [HTTP API reference](../api.md) - Application configuration: [configuration reference](../config.md) +- Existing CLI callers: [native CLI migration](upgrading-cli.md) Go applications that need an in-process prompt framework should import `go.maximumdirect.net/promptkit` directly. The tagged @@ -27,7 +28,7 @@ Consumers are responsible for: - supplying required inputs and template variables; - supplying credentials through the chosen interface; - protecting rendered prompts and generated artifacts as potentially - sensitive; + sensitive, including native inspection and full result reports; - deciding whether validation-failed output is usable; and - retrying only when another model call is acceptable. diff --git a/docs/consumers/upgrading-cli.md b/docs/consumers/upgrading-cli.md new file mode 100644 index 0000000..b87a23d --- /dev/null +++ b/docs/consumers/upgrading-cli.md @@ -0,0 +1,95 @@ +# Migrate To The Native CLI + +This guide covers Scriptorium's breaking CommandKit-native CLI cutover. It does +not select a release version or change historical release notes. The +[CLI reference](../cli.md) owns current syntax, streams, and statuses; +[configuration](../config.md) owns source discovery and settings. +The earlier [PromptKit upgrade](upgrading-promptkit.md) covers framework formats, +not a compatibility layer for this interface. + +## Replace Commands And Selections + +| Former interface | Current interface | +| --- | --- | +| `run --prompt ID` / `--prompt-id` | `run ID`; exactly one positional prompt ID. | +| `render --prompt ID` | `promptkit show prepared ID`. | +| `inspect prompt --prompt ID` | `promptkit show prompt ID`. | +| `inspect profile --profile ID` | `promptkit show profile ID`. | +| `inspect configuration ...` | Native `config ... --scope promptkit`. | +| Application `config ...` | Native `config ... --scope app` (the default scope), including diff. | +| `--config-overlay` | `--overlay NAME`; a native overlay name, not an arbitrary path. | +| `--input NAME=PATH` | `--input-file NAME=PATH`; `--input` now means literal text. | +| `--llm-base-url` | `--endpoint` (canonical `--llm-endpoint`). | +| Duration-valued `--timeout` | Integer `--timeout-seconds` for each generation attempt. | +| `--out` / output-file flags | Caller stdout redirection. | +| `--execution-report` | `--report PATH` with native result rendering. | + +Prefer native short aliases such as `--profile`, `--var`, `--input-text`, and +`--input-file`; canonical `--llm-*` spellings remain available. +There is no duplicate prompt flag, direct backend-ID override, or retained +`--profile-id` alias on generation. + +Repeat mappings rather than comma-separating them. The first equals sign splits +each mapping; commas and later equals signs are literal. Input text/file +occurrences share native last-occurrence precedence. + +## Migrate Application Configuration + +Replace the source-selector list and top-level schema directory with +`promptkit.configuration` and `promptkit.schema_dir`. +The [maintained example](../../examples/config.yml) selects a separate framework +entry file that imports prompt/profile/backend definitions through native +ConfigKit composition. No source/schema CLI flags or compatibility conversion +remain. Do not embed framework definitions inside the application section. + +Discovery now uses native cwd, user-config, home, then Unix-system presets and +their filename conventions. It is no longer limited to the two former +`config.yml` paths. Set an explicit config path when migrating unattended jobs. +Overlays remain native name selections. Application-scope native `--set` records +retain conflict rejection; order does not make duplicate paths authoritative. +There are no custom source/schema/HTTP selectors under `config`. + +## Capture Artifacts And Reports + +For an artifact file plus full native report, from the repository root: + +```sh +scriptorium run generic.markdown_summary --config examples/config.yml \ + --input-file transcript=examples/fixtures/transcript.md \ + --input-file glossary=examples/fixtures/glossary.yml \ + --report run-result.json --report-format json > summary.md +``` + +This generation example contacts the configured provider. Use native prepared +preview instead when only checking inputs/settings. + +Main automatic/text output is the artifact alone, with no added newline. +Main JSON/YAML is the native result; artifact bytes use native structured +serialization. Separate reports contain that full native result, not the old +observer-derived metadata shape. `--report-format` independently chooses +text/YAML/JSON; omission leaves the format unset for native automatic rendering +(currently YAML for structured values). Extensions do not select formats. + +Omit quiet to capture main output, including redirected stdout. Quiet still +allows an explicitly requested report file. Reports are available only when a +result is returned; resultless failures do not replace an existing report. +Partial delivery can leave one or both destinations written. Keep paths separate +and check status/diagnostics before treating files as fresh. + +## Adjust Status And Sensitivity Handling + +Completed generated-content validation failure is now domain status 4, not the +old status 2. Native usage/invalid-configuration input is 2, diff-check +differences are 3, and operational failures are 1. Presentation/file failures +can change success into failure while preserving an already nonzero domain +classification. The [CLI status table](../cli.md#server-and-statuses) owns details. + +Native prompt inspection includes messages; prepared reports include rendered +content; full run reports include generated content and metadata. Framework +classified-field redaction does not make arbitrary report data safe to expose. + +Requirements discovery is separate from validation/preparation: it can run +before artifacts exist, and a partial report may succeed. Read its native +completeness/issue fields. Smoke tests are explicit provider operations, not +startup checks. See [subprocess guidance](../integrations/subprocess.md) for +stream capture, cancellation, and caller responsibilities. diff --git a/docs/consumers/upgrading-promptkit.md b/docs/consumers/upgrading-promptkit.md index 31bd8a6..5268d10 100644 --- a/docs/consumers/upgrading-promptkit.md +++ b/docs/consumers/upgrading-promptkit.md @@ -1,33 +1,23 @@ # Upgrade PromptKit Sources And Definitions -This guide covers the CLI and framework configuration cutover to +This guide covers the earlier framework configuration cutover to [PromptKit v0.23.3](https://forge.maximumdirect.net/base-layer/promptkit/src/commit/804a4d32b23aa2e7973400bb0a322a7016574c9f/docs/consumers/configuration-migration.md) and ConfigKit v1.2.1. See the [framework format reference](https://forge.maximumdirect.net/base-layer/promptkit/src/commit/804a4d32b23aa2e7973400bb0a322a7016574c9f/docs/formats.md) -for the full upstream contract. +for the full upstream contract. The subsequent [native CLI migration](upgrading-cli.md) +owns command, reporting, application-discovery, and source-setting changes. Removing version selectors and changing framework definition shapes are intentional incompatible changes under the [release policy](../policy/release.md#versioning-and-compatibility). ## Application configuration -Existing application fields that already select the desired sources retain -their meaning. For example, this application configuration works both before -and after the cutover: - -```yaml -promptkit_sources: - - directory: ./examples/prompts - - directory: ./examples/profiles - - file: ./examples/framework/backends.yml -schema_dir: ./examples/schemas -``` - -The selected framework files must still be rewritten as described below. -Scriptorium uses one native application schema: discovery, imports, selected -overlays, source order, explicit overrides, and working-directory-relative -selector paths remain application configuration behavior. It loads framework -sources separately with PromptKit's schema. No second application schema or -legacy definition reader is provided. See the [configuration reference](../config.md). +Framework definition migration and application acquisition are separate changes. +Rewrite older framework files as below, then select their entry file using the +current [application configuration](../config.md#application-settings). +The old source-selector list and top-level schema root are not current aliases; +follow the [CLI migration guide](upgrading-cli.md#migrate-application-configuration). +Scriptorium does not provide a legacy definition reader or embedded-definition +extraction/reload path. ## Rewrite framework definitions @@ -82,9 +72,9 @@ Convert framework source controls from `configkit:` to `config:` and add header, if present, is `version: 2`. Use sibling inheritance such as `extends: {id: base}`. -Across ordered `promptkit_sources` entries, a later framework underlay replaces -the whole record for the same ID. Supply the complete intended replacement, -or create a distinct inherited record. Checkouts imported by the same +The single external framework entry uses native ConfigKit import composition; +there is no application-managed source-list precedence. Native record replacement +and inheritance remain upstream-owned. Checkouts imported by the same [discovery file](../config.md#collection-checkout-discovery) are peers: conflicting definitions of the same ID fail instead of overriding one another. @@ -106,5 +96,5 @@ records can replace it through ordinary composition. For multiple independently maintained repositories, see the [collection-discovery example](../../examples/config.discovery.yml). Its entry files declare their own namespaces; checkout names do not become IDs. -The [CLI reference](../cli.md) demonstrates qualified rendering and the +The [CLI reference](../cli.md) defines qualified selection and prepared preview; the [operations guide](../operations.md) describes when updates take effect. diff --git a/docs/development.md b/docs/development.md index fad41f5..7f041b1 100644 --- a/docs/development.md +++ b/docs/development.md @@ -5,6 +5,8 @@ application that consumes the public [PromptKit v0.23.3 package](https://forge.maximumdirect.net/base-layer/promptkit/src/commit/804a4d32b23aa2e7973400bb0a322a7016574c9f/docs/consumers/pkg-promptkit.md); framework implementation work belongs in Promptkit. Application configuration uses ConfigKit v1.2.1; keep Scriptorium's schema and source selection local. +Operator interfaces use CommandKit v0.7.0, ckconfig v0.9.0, and ckprompt v0.10.0; +standard command families and presentation remain upstream-owned. PromptKit uses the same ConfigKit release to load framework sources under its own separate schema and format contract. @@ -46,8 +48,8 @@ Before recommending, planning, or implementing work: | CLI commands, flags, output, or exit behavior | [CLI contract](cli.md) and [adapter internals](internal/adapters.md) | | HTTP routes, DTOs, limits, status mapping, or artifact policy | [HTTP API contract](api.md), [adapter internals](internal/adapters.md), and [source internals](internal/sources.md) | | Application configuration or precedence | [Configuration contract](config.md), [adapter internals](internal/adapters.md), [source internals](internal/sources.md), and `internal/config` definition/load tests | -| Prepared-run presentation | [CLI contract](cli.md), [adapter internals](internal/adapters.md), and `internal/format` | -| Promptkit source selection or captured framework inspection | [Configuration contract](config.md#promptkit-source-selectors), [framework configuration CLI contract](cli.md#framework-configuration-inspection), and [source internals](internal/sources.md) | +| Prepared-run presentation | [CLI contract](cli.md), [adapter internals](internal/adapters.md), and the [native CLI contract](https://forge.maximumdirect.net/base-layer/commandkit/src/commit/d330d7637f2e7507eca931a899397222c82bc892/docs/cli.md) | +| Promptkit source selection or captured framework inspection | [Configuration contract](config.md#framework-source-mapping), [framework configuration CLI contract](cli.md#framework-configuration-inspection), and [source internals](internal/sources.md) | | Prompt, profile, schema, generation, or validation semantics | [PromptKit v0.23.3 framework formats](https://forge.maximumdirect.net/base-layer/promptkit/src/commit/804a4d32b23aa2e7973400bb0a322a7016574c9f/docs/formats.md) and the [Promptkit consumer guide](https://forge.maximumdirect.net/base-layer/promptkit/src/commit/804a4d32b23aa2e7973400bb0a322a7016574c9f/docs/consumers/pkg-promptkit.md) | | OpenAI-compatible outbound behavior or timeout layering | [PromptKit v0.23.3 integration contract](https://forge.maximumdirect.net/base-layer/promptkit/src/commit/804a4d32b23aa2e7973400bb0a322a7016574c9f/docs/integrations/openai-compatible-chat.md) | | Subprocess behavior | [Subprocess integration](integrations/subprocess.md) and [CLI contract](cli.md) | diff --git a/docs/integrations/subprocess.md b/docs/integrations/subprocess.md index 7783c26..6ab9760 100644 --- a/docs/integrations/subprocess.md +++ b/docs/integrations/subprocess.md @@ -1,64 +1,67 @@ # Subprocess Integration -This document covers process-boundary behavior for callers that invoke -Scriptorium as a child process. Command syntax, flags, output, and exit codes -are defined by the [CLI reference](../cli.md). Interface selection belongs in -the [consumer integration overview](../consumers/api.md). +Use the executable when a local process boundary is appropriate. +The [CLI contract](../cli.md) owns exact command/flag/status syntax; the +[consumer overview](../consumers/api.md) compares subprocess and HTTP use. -## Process Contract +## Artifact-First Execution And Inspection -Use `scriptorium render` when the caller needs prepared output without a model -call, and `scriptorium run` for generation. Pass an explicit `--config` or -make the configuration search paths available to the child process; configuration -discovery, fields, profile selection, and credential mechanisms are defined in -the [configuration reference](../config.md). +Select a stored prompt and capture its generated artifact from stdout. Use native +requirements discovery before generated input files exist, concrete validation +once bindings exist, and prepared preview to render without a model call. +The [CLI migration guide](../consumers/upgrading-cli.md) explains replacements +for older command and file-output conventions. -Pass required API-key environment variables through the child environment. Do -not place raw API keys in arguments. Keep the environment limited to the values -needed for the selected profile. +Pass an explicit application configuration or deliberately arrange the child's +[configuration discovery](../config.md). Supply only the environment needed for +selected credentials; never put raw keys in arguments. -For application-only preflight, invoke `scriptorium config validate`; it exits -0 with empty stdout when the application configuration loads, but does not -prove that Promptkit sources or credentials are usable. `config show` emits -resolved YAML and `config sources` emits newline-terminated JSON provenance. -All three use the same source selection and explicit overrides as execution -commands and require separate stderr capture for diagnostics. Their output can -include source paths, identifiers, credential reference names, and ordinary -backend parameters; treat it as potentially sensitive. +Discovery completeness is distinct from binding satisfaction and execution +readiness. Partial reports can succeed. Consumers own their binding graph and +policy for incomplete findings; Scriptorium supplies no workflow state. -## Streams And Output Ownership +## Streams, Reports, And Availability -Capture stdout and stderr separately. Stdout contains the requested artifact or -prepared output unless the caller selects an output file; stderr contains -summaries, diagnostics, and server messages. The exact destinations and status -meanings are part of the [CLI reference](../cli.md), not a stable stderr data -protocol. +Capture stdout and stderr separately. Diagnostics are not a machine-readable +error protocol, even when structured result output is selected. Prefer exit +classification and native report fields; do not parse diagnostic prose. -When using `--out`, the caller owns the output path, its permissions, and -cleanup. Treat rendered prompts, generated artifacts, stdout, and stderr as -potentially sensitive. +The default generated result is artifact content. Native structured results and +optional separate reports carry the full returned result, including content and +metadata. They are not the former metadata-only CLI observation report. Native +prompt/prepared inspection also carries content. Treat all captured streams and +reports as potentially sensitive. -For `run`, an optional [execution report file](../cli.md) contains terminal -metadata separate from generated stdout or `--out`. The caller owns that -path's permissions and retention. Output and report files are not a -transaction: one may be written when the other fails, and a preexisting report -may remain when the invocation has no terminal event. Check the exit status -and diagnostics rather than assuming a report path contains this run's data. +To save main output, supply a file as the subprocess stdout destination or use +shell redirection. Omit quiet when that main result is needed: quiet suppresses +it even for redirected stdout. A separately requested report remains an +application file effect independent of quiet and main format selection. + +A completed content-validation failure can still deliver an artifact/result; +consumers decide whether it is usable. An operational failure with no result +produces diagnostics and a nonzero status, not a synthesized result/error +envelope. An existing separate report may then describe an older invocation. + +## Partial Delivery And Caller Responsibility + +Main output is attempted before a requested report, and report delivery is +attempted even after main presentation failure. Either destination may contain +partial data after an error. File delivery is not transactional and has no +rollback or atomic-replacement promise. + +Callers own redirected file creation/truncation, permissions, retention, and +keeping destinations separate. A shell may truncate its file before the process +starts, unlike a separate report that is opened only after a result exists. +Check the finalized exit status and diagnostic stream; do not infer success or +freshness from a file's existence. ## Cancellation And Recovery -A CLI invocation performs one synchronous request and creates no durable run -state. A supervising process that needs cancellation must terminate the child -process according to its own process-management policy. A later invocation is a -new request and can make another model call; there is no resume or checkpoint -protocol. +Interrupt/SIGTERM cancels the child's invocation context. Engine cancellation is +cooperative. The [whole-operation budget](../cli.md#generation) is separate from +per-attempt provider timeouts and excludes process/configuration setup and +delivery. Supervisors still own forced process termination when required. -For `run`, callers may set the [whole-run deadline](../cli.md) to bound the -engine invocation. It is separate from the per-attempt generation timeout and -does not include child startup or output writing. A typed operation deadline -failure exits 1. -Supervisors still own process termination when they need to stop the child -outside that engine interval. - -For deployment, filesystem permissions, and sensitive-artifact handling, see -the [operations guide](../operations.md). +A new invocation is a new request that may incur another provider call/cost. +There is no checkpoint, archive, conversation, or resume protocol. Deployment and +sensitive-data handling belong to [operations](../operations.md). diff --git a/docs/internal/adapters.md b/docs/internal/adapters.md index 261e2f1..c34b348 100644 --- a/docs/internal/adapters.md +++ b/docs/internal/adapters.md @@ -1,157 +1,89 @@ # Adapter Internals -## Purpose +Adapters map executable inputs and effects through public upstream values. +The [CLI](../cli.md), [HTTP API](../api.md), and [architecture policy](../policy/architecture.md) +own their external contracts and boundaries. -Scriptorium adapters translate executable inputs into Promptkit public requests -and translate Promptkit results or errors back to CLI or HTTP behavior. They -own IO and presentation mechanics, not framework decisions. +## CLI Composition And Acquisition -External contracts are canonical in the [CLI reference](../cli.md) and -[HTTP API reference](../api.md). Promptkit's public engine contract is -described by its tagged -[Go consumer guide](https://forge.maximumdirect.net/base-layer/promptkit/src/commit/804a4d32b23aa2e7973400bb0a322a7016574c9f/docs/consumers/pkg-promptkit.md). +`app.go` composes application run/serve declarations with native configuration, +PromptKit, completion, help, and version support. CommandKit owns parsing, +declaration-input checks, reporting selection, presentation, and finalization. -## Components And Collaborators +`selection.go` maps explicit native config/overlay choices and server-only HTTP +bindings into `internal/config.Load`. It preserves supplied empty/zero values. +Native application `--set` records remain upstream records with native conflict +handling. No application selectors are installed under configuration commands. -- `cmd/scriptorium` passes process arguments and streams to - `internal/adapter/cli`. -- `internal/adapter/cli` resolves settings through `internal/config`, - constructs `promptkit.Engine`, maps CLI values to `promptkit.RunRequest`, - exposes application and captured framework configuration inspection, and - owns output files, summaries, and exit codes. -- `internal/config` delegates application YAML decoding, default resolution, - overrides, and validation ordering to ConfigKit v1.2.1. The CLI supplies only - explicitly visited application flags as overrides and renders safe structured - load diagnostics on stderr. PromptKit uses the same ConfigKit release to - load framework sources under its own definition and format contract. Both - domains use ConfigKit's native diagnostic writer. -- `internal/adapter/http` strictly decodes request DTOs, maps them to Promptkit - public values, calls its adapter-owned `Runner` interface, and maps results - and errors to HTTP DTOs. -- `internal/format` renders `promptkit.PreparedRun` values as deterministic text - or JSON. -- `internal/report` maps Promptkit's terminal execution event to a small - application-owned value without invoking the engine or writing output. +`native_config.go` adapts the actual application result through +`ckconfig.FromConfigKit` and framework inspection through `ckprompt.ConfigScope`. +Native views own effective values, paths, metadata, sources, findings, explanations, +redaction, and comparison. Application inspection does not construct an engine. +Comparison loads each side from that side's selection; there is no config/engine +cache or effective-document reconstruction. -## Wiring Flows +`acquisitionState` contains application diagnostic evidence only, never loaded +configuration or engines. Callback decorators forward it through native +`Acquired` channels. An application LoadError is transported separately while +the framework adapter runs, then its original typed cause is restored before +native outcome classification. This prevents app findings from becoming +framework findings. Native framework evidence remains unchanged; comparison +application messages identify left/right context. Callback marks prevent replay +when operations share invocation state. -### CLI +`native_prompt.go` installs all twelve ckprompt callbacks, nil request builders, +and empty-default native namespace views. Inventories, named resources, +requirements, validation, preparation, and smoke testing retain their upstream +lifecycles and reports. -`run` calls `promptkit.Engine.Run`; `render` calls -`promptkit.Engine.Prepare`. Both share request mapping for exact prompt ID -and profile selection, optional file inputs, variables, and presence-aware -execution overrides. Omitted framework settings remain zero values so Promptkit -resolves its own defaults and definition-required inputs. -The application retry setting enters shared engine construction, while a -run/render flag enters only the mapped request. The server uses the same engine -setting and retains one backend admission state. Render prepares without -generation. +## Generation And Delivery -`inspect prompt` maps the selected ID to -`promptkit.Engine.InspectPrompt`, then formats Scriptorium-owned inspection -data. `inspect profile` maps an explicit profile ID to `Engine.InspectProfile` -and formats Scriptorium-owned resolved profile data without backend completion. +`native_run.go` binds the required operand and reusable resource/input/execution +flags into a native RunRequest. Ordinary typed namespace flags feed +`Engine.WithNamespaces`; stored references are not rewritten. One `Engine.Run` +call receives the invocation context and unchanged operation duration. PromptKit +owns preparation, credentials, admission, retries, repairs, and content validation. -Both paths construct the same full-source engine as execution before inspecting -the chosen definition; selected prompts are not filtered out. They do not -perform request preparation or generation. The [CLI definition-inspection -contract](../cli.md#scriptorium-inspect-prompt) owns the loading and runtime -boundaries, including the distinction between an optional prompt collection -and validating any prompts that were selected. +`outcomes.go` maps typed failures and native configuration evidence to native +outcomes, preserving causes while keeping provider payloads out of terminal +explanations. A returned completed validation failure remains a domain outcome; +an unexpected nil successful result is an integration failure. -`inspect configuration` builds the same engine from the selected native -Promptkit source sequence, reads its captured configuration view, and delegates -effective output and explanation to Promptkit's native writers. The adapter -projects captured source records into its small JSON DTO. Its validation -operation constructs the engine without preparing a request. Framework source -semantics and report redaction are owned by Promptkit; the [CLI -contract](../cli.md#framework-configuration-inspection) documents the exposed -operations and their deliberately limited redaction scope. +`delivery.go` submits the actual RunResult to the invocation reporter with a +reusable artifact-byte text template. The optional file renders that same value +without the template through CommandKit's writer API, outside reporter callbacks +to avoid reentrancy. File failures add explanations/causes while reporter +presentation failures remain native. See [delivery semantics](../cli.md#results-and-separate-reports). -The `config` family loads the same application ConfigKit result once and uses -its validation, effective-output, source, and explanation views directly. It -does not construct a Promptkit engine. The CLI maps source records into a small -JSON DTO for `sources`; `show` and `explain` use ConfigKit's own writers. +## Server And HTTP -`serve` constructs Scriptorium's restricted HTTP artifact reader, injects it -with `promptkit.WithArtifactReader`, passes the engine through the HTTP -adapter's consumer-owned `Runner` interface, and starts the server. +`server.go` loads application settings, injects the restricted artifact reader +into one engine, installs the existing HTTP handler/limits, and opens a listener. +It retains the header timeout and engine retry defaults. Context cancellation +calls Server.Close; callback coordination finishes connection/listener closing +before return, without waiting for arbitrary context-ignoring application work. -The command paths assemble the engine from application-selected framework -source files and directories plus the schema directory and retry setting. -Individual files become import-enabled ConfigKit inputs; directory selections -are expanded through Promptkit's directory-input helper. Scriptorium supplies -those ordered underlays through Promptkit's public configuration API rather -than parsing or mapping framework definitions itself. The server constructs -one engine, which retains the framework's backend registry and associated -capacity state across requests. +The HTTP handler strictly decodes DTOs and applies transport policy before +mapping to PromptKit requests. Its Runner interface is a consumer-side external +boundary using native values. Request controls do not mutate shared engine +defaults; shared backend capacity accounting remains in that engine. -### HTTP - -The handler enforces transport limits and strict JSON decoding before mapping -DTOs into `promptkit.RunRequest`, `promptkit.ArtifactRef`, and -`promptkit.ExecutionTargetOverride`. On success it reads Promptkit artifact, -validation, model, usage, and metadata values directly. -The optional HTTP retry count maps to `RunRequest.TransientRetries` without -mutating the shared engine. Negative values are rejected before invocation; -Promptkit controls retries and backend admission for concurrent requests. -CLI and HTTP operation budgets map to PromptKit's `OperationTimeout`; HTTP -passes the caller context through unchanged. The public sentinels determine -timeout classification. - -Failure mapping uses `errors.Is` against Promptkit's public sentinels and the -HTTP reader's Scriptorium-owned containment and size errors. Wrapped reader -errors preserve their identity through Promptkit's artifact-load boundary. - -The report collector is per invocation. It ignores nonterminal events and -copies terminal aggregates once, including usage presence and completeness. -It retains no history or shared state; adapters decide whether to attach it and -how to present its snapshot. -CLI `run` attaches one only when a report path is selected, writes generated -output first on success, and then writes the terminal snapshot when present. -Its report path remains untouched for pre-invocation failures or absent -terminal observations; report and artifact writes are independent. -HTTP `POST /v1/runs` attaches a separate collector to each invocation only when -`include_execution_report` is true. Its optional top-level `execution` field -appears in success or error envelopes only when Promptkit emits a terminal -event. Pre-invocation adapter errors and absent terminal events leave it out. -The complete serialized success or report-bearing error response is subject to -the configured response-size limit; overflow returns the compact 413 error -without a report. Field meanings are owned by the [API contract](../api.md#execution-report). - -## Package-Local Guarantees - -- Adapters contain no copied framework types or orchestration. -- Configuration is resolved before Promptkit engine construction. -- Explicit numeric overrides preserve presence, including zero. -- HTTP DTO and error mapping remains stable and transport-owned. -- Resolved secrets are not serialized or printed. -- No adapter creates durable run state. +HTTP optional reports use one per-request `internal/report.Collector`, attached +only for opted-in requests. It copies terminal aggregates, not event history, +and is never the CLI result report. Success/error attachment and encoded-size +handling retain the [HTTP contract](../api.md#execution-report). Reader sentinel +identities survive PromptKit wrapping for status mapping. ## Verification -Inspect: +Native composition/configuration/prompt/run/delivery/lifecycle tests protect the +application's integration choices using real libraries and controlled local +providers. Checkout-discovery tests cover fresh invocations and entry-file +spellings. Entrypoint tests protect identity/status mapping; HTTP reader/DTO/ +report/capacity regressions and the architecture guard remain their own owners. +Do not duplicate upstream parsers, analyzers, serializers, or exhaustive format +matrices to change an adapter. -- `internal/adapter/cli/run_test.go` -- `internal/adapter/http/handler_test.go` -- `internal/adapter/http/artifact_reader_test.go` -- `internal/format/prepared_run_test.go` -- `internal/adapter/dependency_test.go` - -The adapter tests protect parsing, configuration mapping, output, status -mapping, restricted artifacts, and representative real Promptkit-engine -workflows. The dependency test protects the repository boundary. - -## Change Recipes - -For a CLI or HTTP change: - -1. identify the Scriptorium-owned external contract; -2. map through Promptkit public values without copying framework semantics; -3. add or update the narrow application-owned test; -4. update the canonical Scriptorium contract; and -5. coordinate and tag Promptkit first if a required public capability is - genuinely absent. - -Update [source internals](sources.md) when application source locations or HTTP -artifact containment changes. +When changing behavior, update its canonical external contract and focused +integration protection. Required shared capabilities belong upstream before +tagged adoption, not in a resurrected application framework facade. diff --git a/docs/internal/overview.md b/docs/internal/overview.md index 03c20ad..6380a54 100644 --- a/docs/internal/overview.md +++ b/docs/internal/overview.md @@ -1,19 +1,18 @@ # Internal Component Overview -This is the complete inventory of Scriptorium's implemented Go components. -The [architecture policy](../policy/architecture.md) owns normative boundaries; +This inventory describes implemented Go components. The +[architecture policy](../policy/architecture.md) owns normative boundaries; public behavior belongs in the linked contracts. | Component | Implemented responsibility | References | | --- | --- | --- | -| `cmd/scriptorium` | Process entrypoint that delegates arguments and streams to the CLI adapter. | [CLI contract](../cli.md), [adapter internals](adapters.md) | -| `internal/adapter/cli` | Parses execution, application-configuration, and captured framework-configuration inspection commands; resolves settings; constructs Promptkit engines for execution, definition inspection, and framework inspection; maps requests; and owns process output and exit behavior. | [CLI contract](../cli.md), [adapter internals](adapters.md), [source internals](sources.md) | -| `internal/adapter/http` | Owns routes, DTOs, strict decoding, limits, Promptkit request/result mapping, public error mapping, and restricted HTTP artifact reading. | [HTTP API](../api.md), [adapter internals](adapters.md), [source internals](sources.md) | -| `internal/config` | Selects application sources through ConfigKit v1.2.1, supplies the application definition and explicit overrides, and converts resolved settings for runtime use. | [configuration contract](../config.md), [adapter internals](adapters.md), [source internals](sources.md) | -| `internal/defaults` | Holds Scriptorium-owned application and HTTP defaults. | [configuration contract](../config.md) | -| `internal/format` | Formats Promptkit prepared-run and prompt/profile inspection values for CLI text or JSON output; framework-configuration output uses Promptkit's native captured view and writers. | [CLI contract](../cli.md), [adapter internals](adapters.md) | -| `internal/report` | Projects one terminal Promptkit observation into Scriptorium-owned execution metadata. | [adapter internals](adapters.md) | +| `cmd/scriptorium` | Build identity, signal-aware context, native status-to-process exit. | [CLI](../cli.md), [adapters](adapters.md) | +| `internal/adapter/cli` | CommandKit composition, ckconfig/ckprompt acquisition, application loading, native request mapping, run/report effects, and HTTP process lifecycle. | [CLI](../cli.md), [adapters](adapters.md), [sources](sources.md) | +| `internal/adapter/http` | Routes, strict DTO decoding, transport limits, public request/result/error mapping, and restricted artifact reader. | [HTTP API](../api.md), [adapters](adapters.md), [sources](sources.md) | +| `internal/config` | ConfigKit application schema, discovery/selection, defaults/resolution/validation callbacks, and completed-value conversion. | [Configuration](../config.md), [sources](sources.md) | +| `internal/defaults` | Application and HTTP default constants. | [Configuration](../config.md) | +| `internal/report` | HTTP terminal-observation metadata projection. It is not used for CLI native result reports. | [HTTP execution report](../api.md#execution-report), [adapters](adapters.md) | -Framework implementation packages are provided by -[PromptKit v0.23.3](https://forge.maximumdirect.net/base-layer/promptkit/src/commit/804a4d32b23aa2e7973400bb0a322a7016574c9f/docs/consumers/pkg-promptkit.md) -and are not part of this repository. +CommandKit, ckconfig, ckprompt, ConfigKit, and PromptKit are released upstream +dependencies, not local framework packages. There is no custom CLI formatter +package or compatibility dispatch surface. diff --git a/docs/internal/sources.md b/docs/internal/sources.md index 8962e2c..f00f76f 100644 --- a/docs/internal/sources.md +++ b/docs/internal/sources.md @@ -1,117 +1,55 @@ # Source Internals -## Purpose +This document covers application acquisition and the restricted HTTP artifact +reader. [Configuration](../config.md) owns field/discovery/composition contracts; +the [PromptKit formats](https://forge.maximumdirect.net/base-layer/promptkit/src/commit/804a4d32b23aa2e7973400bb0a322a7016574c9f/docs/formats.md) +own framework resources and execution semantics. -This document covers Scriptorium-owned source locations and the restricted HTTP -artifact reader. Prompt, profile, schema, and ordinary artifact semantics are -owned by the tagged -[PromptKit v0.23.3 format reference](https://forge.maximumdirect.net/base-layer/promptkit/src/commit/804a4d32b23aa2e7973400bb0a322a7016574c9f/docs/formats.md). +## Application Boundary And Engine Construction -## Application Source Locations +`internal/config.Load` selects an explicit File or the ordered native SearchConfig +presets and performs one ConfigKit load with imports, overlay selection, and +explicit overrides. It returns the actual Result with native inspection evidence. +Definition callbacks supply application defaults, record actual path/address +normalizations through ConfigKit, and validate retry/HTTP-size domains. Settings +conversion consumes completed values without another merge or normalization pass. -`internal/config` resolves the ordered `promptkit_sources` sequence and -`schema_dir` from application configuration and explicit CLI overrides. It -owns selector-shape validation, not framework file parsing. During engine -construction, `internal/adapter/cli` turns file selections into import-enabled -ConfigKit inputs and expands directory selections with -`promptkit.DirectoryInputs`, then passes the ordered groups as Promptkit -configuration underlays, one per selector. Directory inputs retain the -helper's strict-composition grouping; no selected source becomes an implicit -framework main. The shared Promptkit engine receives the selected schema -directory and retry setting, and the default OpenRouter bundle remains enabled. +The typed application PromptKit section selects acquisition only. In +`engine.go`, a nonempty resolved configuration path becomes an import-enabled +main Input; an empty path leaves Main nil. The schema base and retry default are +passed directly, retaining the bundled underlay. There is no source-list parser, +directory classification/expansion, or one-underlay-per-selector algorithm. -Promptkit uses ConfigKit to load and compose those framework inputs according -to Promptkit's own schema and format. Scriptorium does not parse or validate -Promptkit definitions. Promptkit owns definition selection and validation, -profile built-ins and overlays, schema resolution, ordinary file artifacts, -captured framework configuration inspection, and related error identities. +Framework imports resolve within the native source context. Message resources +are captured from their defining contexts at engine construction. Schema and +request-artifact acquisition retain their preparation-time behavior. +Independent collection checkouts are discovered by native file import globs, +not application traversal. New invocations reload normally; a server retains +its single engine until restart. -The [configuration reference](../config.md) owns Scriptorium's source-location -fields and precedence. Maintained files under `examples/` are application -inputs that use Promptkit's tagged formats. - -## Application Configuration Boundary - -`internal/config` contains the application-owned ConfigKit-native `Document` -definition. Its fresh defaults come from Scriptorium's existing default -constants; its resolver records only actual leaf-level trimming, path cleaning, and -render-format normalization. The validator reports selector-shape, retry, -size-limit, and format errors without mutating the document or exposing -rejected values. Application path values keep working-directory-relative -meaning. ConfigKit v1.2.1 owns strict application YAML decoding and presence -semantics. Framework configuration is separately loaded by PromptKit using -the same ConfigKit release under PromptKit's own schema and source contract. - -The `Load` boundary selects an explicit file or constructs a native ConfigKit -search over the ordered implicit paths. Discovery exhaustion uses defaults; -a selected-file error stops loading. One ConfigKit load enables imports, -applies explicitly supplied application overrides after decoding and before -resolution and validation, and returns the document and its frozen inspection -records. `Settings` copies selectors in their resolved order into runtime -settings; it does not read files or apply another merge. ConfigKit uses the selected file's -canonical directory as the source root; the application does not widen it. -Application path values retain process-working-directory-relative meaning. - -Runtime conversion consumes the resolved application values through the -existing `Settings` path. Promptkit receives selected framework sources only -after application configuration has loaded and validated; its source merge, -definition validation, and captured inspection view remain framework-owned. - -Every configuration-loading command uses this boundary. The CLI maps only -explicitly visited application flags to ConfigKit overrides and prints -structured load diagnostics before engine construction. Application and -framework load errors and warnings use their respective native writers. The old application -YAML decoder and merge path have been removed. -Application `config` inspection reuses the result's frozen effective, -provenance, and history views without constructing an engine or reloading -sources. It does not resolve credentials. `inspect configuration` separately -constructs an engine and exposes Promptkit's captured framework view; only its -framework `show` and `explain` reports receive Promptkit's native, -credential-shaped redaction. Neither inspection family is a general-purpose -filter for arbitrary sensitive values. +Application-only inspection uses its completed result without framework +acquisition. Framework inspection exposes the native captured view separately. +Diagnostic decoration in the [CLI adapter](adapters.md) carries application +messages without combining evidence trees or effective documents. There is no +per-invocation engine/config cache or cross-invocation snapshot guarantee. ## Restricted HTTP Artifact Reader -`internal/adapter/http` implements `promptkit.ArtifactReader` for HTTP -requests. The `serve` path injects it with -`promptkit.WithArtifactReader`, replacing Promptkit's ordinary reader for -inbound HTTP inputs. +The HTTP adapter implements `promptkit.ArtifactReader`; server construction +injects it through the public option. Ordinary CLI artifact loading remains +upstream-owned. -The reader: +The reader accepts inline content without a root. File references need a root; +relative paths resolve below it and absolute paths must be lexically contained. +Traversal outside is rejected and the configured file byte limit applies, with +zero unlimited. Reads preserve native artifact metadata and honor cancellation. -- accepts inline references without an artifact root; -- denies file references when no root is configured; -- resolves relative paths below the configured root; -- accepts absolute paths only when they are lexically within that root; -- rejects lexical traversal outside the root; -- applies the configured file byte limit, with zero meaning unlimited; -- preserves content type, body, size, hash, name, and URI metadata; and -- honors context cancellation. +Containment does not resolve symlinks. The OS may follow a symlink inside the +root to an outside target; this is not a filesystem sandbox or a defense against +privileged concurrent filesystem modification. Reader errors retain identity +through upstream artifact-load wrapping, enabling the existing HTTP mapping. -Containment is lexical and does not resolve symlinks. The operating system -follows symlinks after the check. The [HTTP API](../api.md) owns observable -request outcomes, and [operations](../operations.md) owns safe deployment -permissions and root selection. - -Reader errors remain identifiable after Promptkit wraps them as artifact-load -failures, allowing the HTTP adapter to preserve Scriptorium status and error -codes. - -## Verification And Change Recipe - -Inspect: - -- `internal/config/document_test.go` -- `internal/config/load_test.go` -- `internal/adapter/cli/run_test.go` -- `internal/adapter/http/artifact_reader_test.go` -- `internal/adapter/http/handler_test.go` - -When changing an application source location or HTTP artifact policy: - -1. preserve strict configuration precedence and the Promptkit public boundary; -2. keep containment and size policy in Scriptorium; -3. update focused configuration, reader, and handler tests; -4. update the [configuration](../config.md), [HTTP](../api.md), and - [operations](../operations.md) contracts as applicable; and -5. do not duplicate Promptkit loaders, formats, or ordinary artifact behavior. +The [HTTP API](../api.md) owns outcomes and [operations](../operations.md) owns +safe root/permission selection. Configuration, CLI acquisition/engine, and HTTP +reader/handler tests own their respective boundaries; upstream owns exhaustive +source traversal, format parsing, ordinary artifacts, and inspection semantics. diff --git a/docs/operations.md b/docs/operations.md index 2623d66..ed33677 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -1,243 +1,120 @@ # Operations Guide -## Scope And References +This runbook covers deployment, normal operation, capacity, sensitive data, and +recovery. Exact [CLI syntax/statuses](cli.md), [configuration](config.md), and +[HTTP wire behavior](api.md) remain in their canonical references. +See the [CLI migration guide](consumers/upgrading-cli.md) for changed jobs and +the [framework upgrade guide](consumers/upgrading-promptkit.md) for definitions. -This runbook covers deployment, normal operation, capacity planning, and safe -recovery for Scriptorium. It does not redefine invocation syntax, configuration -fields, or HTTP wire behavior. +## Deploy Sources And Process -- [CLI reference](cli.md): commands, output destinations, and exit codes. -- [Configuration reference](config.md): application settings, source - locations, defaults, and credential mapping. -- [Promptkit source upgrade guide](consumers/upgrading-promptkit.md): concise - source-selector and definition migration examples. -- [PromptKit v0.23.3 framework formats](https://forge.maximumdirect.net/base-layer/promptkit/src/commit/804a4d32b23aa2e7973400bb0a322a7016574c9f/docs/formats.md): - prompt, profile, schema, execution-setting, and framework credential - contracts. -- [HTTP API reference](api.md): route, request/response schema, status codes, - limits, and HTTP artifact access. -- [Consumer integration overview](consumers/api.md): caller responsibilities. +Provide readable application configuration, its selected framework entry file, +and explicitly imported definitions/resources. Set cwd deliberately: application +path values are cwd-relative, while framework imports and message resources keep +their native source contexts. ConfigKit confines source imports/overlays to the +selected main's canonical root; Scriptorium does not widen it. -## Operational Model And State +Engine construction validates selected effective definitions and captures message +resources/template syntax, including definitions not used by a particular request. +Schema/artifact reads, credential resolution, and model work retain their native +preparation/runtime lifecycles. A server constructs one engine at startup and +does not hot-reload changed files; restart to capture updates. Separate CLI +invocations load afresh rather than sharing an immutable snapshot. -Scriptorium handles one prompt request for each CLI invocation or HTTP request. -It has no durable run store, archive, checkpoint, cache, or resume mechanism. -A failed or interrupted request is recovered by correcting its inputs, -configuration, or environment and submitting a new request. +For independent collections, install checkouts beneath the layout selected by +the [discovery entry](../examples/discovery/framework.yml). Each checkout declares +its namespace and explicit fragment imports; arbitrary YAML is not a framework +source. The maintained [discovery example](../examples/config.discovery.yml) +supports both entry-file spellings and multiple definitions. New checkouts or +updates become visible to later invocations or a restarted server. -Generated artifacts, rendered prompts, model output, and run metadata are -caller-owned data. Retention, encryption, backup, and deletion are deployment -responsibilities. -Optional [CLI execution reports](cli.md) also belong to the caller. They can -contain sensitive identifiers even though they omit prompt and response bodies. -Keep report paths separate from generated output and apply the same access and -retention controls. Output and report writes are not transactional; an older -report may remain when no terminal observation was produced. -HTTP callers may also request a [terminal execution report](api.md#execution-report). -It shares the same identifier sensitivity and should receive response access -and retention controls. An opted-in report increases the encoded response size; -an oversized success or report-bearing error becomes the compact 413 response -without a report. +Supply only needed credential environment variables through a supervisor or +secret-management mechanism. Confirm presence without printing their values. +Raw keys do not belong in arguments or checked-in configuration. -## Deploy The Filesystem And Process +## Normal Operator Workflow -Provide the process with readable selected Promptkit source files/directories -and schema sources that follow the tagged framework formats. At engine -construction, PromptKit v0.23.3 validates every selected effective definition, -including unused definitions, and captures referenced message-file contents -and static template syntax. `serve` constructs one engine at startup; it does -not reload changed configuration or message content. Restart the process to -validate and capture a new configuration. Schema and request-artifact reads, -credential resolution, and provider work remain request-preparation or runtime -concerns. For an HTTP deployment that accepts file artifacts, use a dedicated, -narrow artifact directory rather than a general-purpose or sensitive filesystem -tree. +1. Validate application configuration through the native application scope. + This does not acquire framework sources or prove provider readiness. +2. Inspect framework configuration/resources and discover requirements for the + selected prompt/profile/namespaces. Discovery can precede generated input + artifacts; completeness does not establish satisfied bindings. +3. Once artifacts exist, validate the concrete request or inspect prepared + content. These are provider-free preparation checks with native prerequisites. +4. Invoke a smoke test only when its provider effects are intended. Findings + can be unsupported or unsuccessful while the command itself succeeds. + Smoke tests are not an automatic startup or release prerequisite. +5. Generate with the same selections and bindings, then inspect the finalized + status and any returned validation result before accepting the output. -If application configuration uses explicit imports or a selected overlay, -deploy those files beneath the selected configuration file's canonical -directory. ConfigKit keeps imported and overlay files within that root; -Scriptorium does not widen it. Application path values inside those files still -resolve from the process working directory, so set that directory deliberately -in the service manager. This configuration-source boundary is distinct from -the HTTP artifact root and does not constrain prompt or schema directories. +Use the [maintained preview script](../examples/render-markdown-summary.sh) +for a provider-free example, and the [CLI contract](cli.md) for commands. +A complete discovery report can still list required inputs that are not bound; +a partial report can still be useful. Consumers own their policy for incomplete +findings and changed resources between invocations. -For independently maintained collections, place checkouts beneath the -directory selected by your framework discovery file. The maintained -[discovery example](../examples/config.discovery.yml) selects optional -`collection.yml` and `collection.yaml` entry files; each checkout owns its -namespace and explicit fragment imports. Update a checkout independently and -invoke the CLI again, or restart a running server to capture the update. Do -not treat arbitrary YAML in a checkout as a framework source. When upgrading -definitions, review exact profile IDs and replace retired catalog references -as described in the [migration guide](consumers/upgrading-promptkit.md). +## Sensitive Data And Retention -Run Scriptorium under an identity that can: +Artifacts, prompt messages, prepared content, native run reports, metadata, +diagnostics, and redirected streams are caller-owned potentially sensitive data. +The separate CLI report contains the full returned result, not metadata alone. +Control access, retention, encryption, backup, and deletion accordingly. +Framework effective/explanation views apply native classified-field redaction, +including history, but do not detect arbitrary secrets. Other content-bearing +reports are not automatically made safe for publication. -- read only the prompt, profile, schema, and allowed input-artifact paths it - needs; -- read the required credential environment variables without writing them to - files or logs; and -- write only caller-selected output locations when CLI output files are used. +Keep main output and report destinations separate. Delivery is nontransactional: +one destination or partial bytes can remain after failure. An existing report +can remain on a resultless error; check status and report identity, not just file +existence. A shell can truncate a redirected destination before the child starts. +Omit quiet if main stdout must be captured; the separate report effect remains +independent. Exact delivery behavior belongs to [CLI results](cli.md#results-and-separate-reports). -Do not make the HTTP artifact directory writable by untrusted users. The HTTP -artifact containment behavior is lexical and the operating system follows -symlinks; account for that when choosing ownership and mount boundaries. See -the [HTTP API reference](api.md) for the externally observable behavior. +HTTP's optional [execution report](api.md#execution-report) remains a distinct +terminal-observation projection. Protect its identifiers and any response data. +It increases encoded response size and remains subject to the configured limit. -## Supply Credentials And Protect Runtime Data +## HTTP Deployment And Shutdown -Set secret values in the process environment and configure only their -environment-variable names. Do not put raw keys in configuration, prompt or -profile files, process arguments, HTTP payloads, captured command lines, or -debug dumps. +The service has no built-in authentication/authorization. Deploy on a trusted +network or behind appropriate authenticated proxy/gateway controls. Use a narrow, +dedicated artifact root with controlled ownership and permissions when accepting +file inputs; this is lexical containment, not a symlink sandbox. -Treat stdout, stderr, prepared-run output, generated artifacts, and HTTP -responses as potentially sensitive. Send service logs to a controlled collector -and apply the same retention and access rules as for model input and output. +Use a supervisor for cwd, environment, restart policy, and stream capture. +Startup information uses native Info reporting, not stdout banners. +Interrupt/SIGTERM cancels the invocation and closes the listener and active +connections. Canceled shutdown is nonzero operational termination. There is no +graceful-drain deadline or promise that context-ignoring work has finished; +supervisors own forced termination when necessary. -## Run A Normal Workflow +## Capacity, Limits, And Recovery -Before changing production inputs, profiles, or schemas: +One server engine shares backend admission/queue state across concurrent requests. +That state is transient operational coordination, not a durable workflow or +queue of resumable runs. HTTP capacity exhaustion retains its +[API outcome](api.md); callers decide whether another model call is safe. -1. use `config validate`, `config show`, or `config sources` to check - Scriptorium's application configuration; these do not open Promptkit sources - or construct an engine; -2. use `inspect configuration validate` to construct the engine and validate - all selected framework definitions; this does not prepare a request, check - credentials, load request-time schemas/artifacts, or contact a provider; -3. confirm the deployed configuration selects the intended sources and - environment-variable names; -4. use [`render`](cli.md) with the same request inputs and variables to confirm - preparation without a model call; -5. use [`run`](cli.md) for generation; and -6. retain or discard validation-failed output according to the caller's - policy. +Plan for input/output sizes, schema complexity, provider latency, retries, and +repair costs. Native transient attempts and repair generations can add calls and +waiting; engine defaults and per-request overrides belong to +[configuration](config.md) and the [CLI](cli.md)/[HTTP](api.md) contracts. +Whole-operation budgets are distinct from per-attempt generation timeouts. +They remain cooperative and do not include configuration/setup or delivery. -Application configuration inspection can include paths and configured values. -Framework `inspect configuration show` and `explain` use Promptkit's native -redaction for classified credential-shaped provider fields, including -historical values, while leaving environment-reference names and ordinary -provider parameters visible. Neither report identifies arbitrary sensitive -content by value or name. Protect inspection output accordingly; Scriptorium -stores environment-variable names, not the resolved credential bytes. +Before increasing HTTP byte limits, measure representative requests/artifacts/ +responses, confirm memory/network/provider capacity, retain authentication and +upstream request controls, and test a representative workload. Do not widen the +artifact root just to make an arbitrary host path accessible. -The [maintained render script](../examples/render-markdown-summary.sh) is a -copyable preflight example. The CLI reference owns its complete invocation and -exit semantics. +For failures, capture diagnostics without publishing sensitive data. Verify +selected configuration, resource identity, readable inputs, profile/credentials, +and schema deployment. Use native prepared preview to isolate pre-generation +problems. A completed content-validation failure is distinct from an operational +failure to validate; consumers choose whether failed content is usable. +Correct the cause, then issue a new request only when another provider call and +its cost are acceptable. -## Expose The HTTP Service - -The HTTP service has no built-in authentication or authorization. Place it on a -trusted network or behind an authenticated reverse proxy, API gateway, or -equivalent access control. Restrict who can reach it and who can read the -artifact root. - -Use a service manager or supervisor appropriate to the deployment to manage -process lifetime, restart policy, log capture, and environment injection. The -[HTTP API reference](api.md) owns client request shapes, status behavior, and -artifact-access outcomes. - -## Plan Capacity And Limits - -Capacity is primarily determined by concurrent model calls, input and output -sizes, schema complexity, provider latency, and network behavior. Size limits -protect request bodies, HTTP file artifacts, and encoded responses; configure -them through the [configuration reference](config.md) and rely on the -[HTTP API reference](api.md) for their response effects. - -Configured backend concurrency and queue capacity are enforced per constructed -Promptkit engine. A `serve` process constructs one engine for its handler, so -concurrent HTTP requests share that transient admission state. Scriptorium does -not retain workflow state: capacity is neither durable nor a queue of resumable -runs. When admission is exhausted, HTTP returns `503 capacity_exceeded` without -retry timing; callers choose any retry policy that is safe for another model -call. - -Promptkit's built-in client now allows two additional attempts for eligible -transient provider statuses by default. Account for added calls, latency, and -cost when planning capacity. Each repair generation has its own transient -budget, separate from the structural repair count. Configure the engine budget -with [`transient_retries`](config.md); `run` and `render` accept a request -override described in the [CLI contract](cli.md), and HTTP callers can select -one through the [API contract](api.md). Omission inherits the engine -setting and then Promptkit's default, while explicit zero disables additional -attempts. Existing generation timeouts apply -per attempt, while the caller context covers the complete operation. Provider -`Retry-After` timing can exceed Promptkit's ordinary backoff cap; consult the -[outbound integration contract](https://forge.maximumdirect.net/base-layer/promptkit/src/commit/804a4d32b23aa2e7973400bb0a322a7016574c9f/docs/integrations/openai-compatible-chat.md) -for details. - -CLI `run` can set a [whole-run deadline](cli.md) when total execution time must -be bounded. The existing generation timeout applies afresh to each attempt and -does not cap retry waits or the complete invocation. The whole-run interval -covers the engine call, including capacity waiting and repairs as supported by -context cancellation; it excludes parsing, engine setup, and output writes. -HTTP callers can use [`run_timeout_seconds`](api.md) to set the engine's -operation budget for the same interval. The API distinguishes a typed expired -execution deadline from caller cancellation and from a provider-attempt -timeout. A canceled capacity waiter releases its Promptkit admission state. - -Before increasing a limit: - -1. measure representative input, generated-output, and optional raw-output - sizes; -2. confirm memory, network, and upstream-provider capacity; -3. retain an upstream request-size and authentication boundary; and -4. test the intended workload in a non-production environment. - -For large local inputs, prefer a controlled file-artifact directory over -placing arbitrary paths on the service host. Avoid disabling a limit unless an -equivalent trusted control exists elsewhere. - -## Diagnose And Recover - -### Preparation Or Configuration Failure - -Capture the CLI diagnostic or HTTP error response, then verify the selected -configuration, prompt ID, profile selection, source readability, and input -mapping. Use `render` with the same request when it is unclear whether failure -occurs before model execution. Consult the [CLI reference](cli.md), the -[configuration reference](config.md), and the [HTTP API reference](api.md) for -the exact interface contract. - -### Credential Or Provider Failure - -Confirm that the process environment contains the configured credential name -without printing the secret. Check endpoint reachability and provider health -from the process network. If preparation succeeds but generation fails, inspect -the selected model settings in prepared output and the service's controlled -logs. Correct the deployment or provider issue, then submit a new request. - -### Artifact Or Permission Failure - -Verify that the process can read the intended local input. For HTTP file -artifacts, verify the deployment's artifact root, ownership, path layout, and -file size. Do not widen filesystem permissions or the allowed root merely to -make an arbitrary path work; move or copy the required artifact into the -controlled location instead. - -### Validation Failure - -A generated-content validation failure is distinct from a runtime failure. -CLI `run` reports the validation result and error count in its success summary; -it does not print the individual validation messages. For HTTP, inspect the -validation object in the response according to the [HTTP API reference](api.md). - -Use rendered input and generated output to determine whether prompt instructions, -the selected model, or the schema needs correction. If schema loading or -compilation itself fails, correct the source deployment or schema document -before rerunning. - -### HTTP Limit Or Request Failure - -Compare the request, artifact, or expected response size with the deployed -configuration, and validate the request against the [HTTP API reference](api.md). -Reduce the payload, use an appropriate controlled artifact source, omit -unneeded raw output, or adjust the deployment limit after capacity review. - -## Cleanup And Reruns - -Because no run state is retained, cleanup concerns caller-owned output files, -logs, and artifacts only. Remove or rotate them using the deployment's normal -retention policy. After a correction, rerun the request from the beginning; -there is no safe resume point. +There is no durable run store, archive, checkpoint, or resume mechanism. +Cleanup concerns caller-owned files/logs only. Rotate/delete them under the +deployment's normal retention policy; recovery starts from a new request. diff --git a/docs/policy/architecture.md b/docs/policy/architecture.md index c40824a..3431346 100644 --- a/docs/policy/architecture.md +++ b/docs/policy/architecture.md @@ -39,8 +39,8 @@ Do not promote incidental implementation choices into architectural invariants. ## Scriptorium's Purpose And System Shape -Scriptorium is an executable application with CLI `run`, `render`, -`inspect`, and `config` command families and an HTTP service started by +Scriptorium is an executable application with CLI `run`, native `promptkit` +and `config` families, and an HTTP service started by `serve`. It offers simple, request-independent execution of predefined prompts, not a reusable Scriptorium root Go API or a workflow framework. The [internal overview](../internal/overview.md) owns the concrete package @@ -71,9 +71,12 @@ externally observable application behavior; PromptKit owns framework formats. As decided in [ADR 0006](../adr/0006-adopt-commandkit-native-operator-interfaces.md), Scriptorium adopts CommandKit's native `ckconfig` and `ckprompt` interfaces to -neutrally expose upstream features and semantics. The current CLI still uses -custom presentation; the [adoption roadmap](../roadmap/commandkit-adoption.md) -tracks its replacement. This decision complements ADRs 0004 and 0005. +neutrally expose upstream features and semantics. CommandKit owns shared command, +parsing, presentation, and outcome mechanics; application commands and effects +remain local. Standard configuration inspection uses native adapters without +custom effective documents or evidence reconciliation. This decision complements +ADRs 0004 and 0005; the [adoption roadmap](../roadmap/commandkit-adoption.md) +records the implementation work. The CLI exposes Promptkit's captured framework configuration inspection. Its `show` and `explain` reports use Promptkit's native redaction for classified diff --git a/docs/policy/testing.md b/docs/policy/testing.md index c18ebfa..7727baa 100644 --- a/docs/policy/testing.md +++ b/docs/policy/testing.md @@ -2,7 +2,7 @@ **Base Policy Version:** 1.3 -**Last Updated**: 2026-11-17 +**Last Updated**: 2026-11-19 **Upstream Source:** https://gitea.maximumdirect.net/eric/policy/src/commit/593c9b52273c5c879785ae9e995d21026df467b4/docs/policy/testing.md @@ -178,12 +178,15 @@ Give each layer its distinct responsibility and keep expectation updates localiz diagnostics. ConfigKit owns exhaustive YAML grammar, source traversal, import and overlay composition, provenance, and inspection renderers. - CLI tests own argument mapping, streams, output, exit behavior, and - representative run, render, inspect, serve, and config workflows. + representative native run, PromptKit resource/requirements/preview/smoke, + serve, and configuration comparison workflows. - HTTP tests own DTO decoding, transport limits, status and error mapping, restricted artifact reading, and representative server workflows. -- Formatting tests own prepared-run and definition-inspection text and JSON - presentation. Architecture guards own dependency direction and removed - framework-package boundaries. +- CommandKit and its integrations own standard parsing and presentation. + Scriptorium's integration tests protect native adoption, byte-preserving main + output, separate report effects, outcomes, and acquisition boundaries without + reproducing the upstream formatting matrix. Architecture guards own dependency + direction and removed framework-package boundaries. - Promptkit owns framework parsing, orchestration, validation, profiles, and model-client behavior. Scriptorium's definition and execution fixtures protect adoption through its public adapter boundary without duplicating diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 6ee6165..631e9bb 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -643,6 +643,25 @@ version output. Test-local fallback deadlines prevent regressions from hanging. ## Stage 8: Retire Superseded Machinery And Update Canonical Documentation +**Status: Complete.** Removed legacy parsing/dispatch, custom inspection/source +DTOs and writers, retired CLI tests, and the unused formatter package. Retained +architecture boundaries, native integration protection, checkout-discovery +coverage, and the unchanged HTTP/report implementation and regression suites. +Maintained examples now select external framework entries with native imports; +both collection-entry spellings and multi-definition discovery remain present. +Canonical CLI/configuration/process/operations/internal/development/policy owners +and navigation reflect the cutover, with a focused CLI migration guide distinct +from the earlier framework-format guide. Accepted ADRs and release-note history +are unchanged; deferred HTTP redesign and dynamic completion remain deferred. + +Verified on Go 1.27.4: full tests and race suite, vet, ordinary build, module +tidiness, formatting/diff checks, all maintained application configs in both +native scopes, application overlays, maintained preview scripts, a structured +schema preview, and local Markdown links/anchors (42 files). Added maintained +example loading/preparation protection and adapted fresh-checkout discovery to +native preview. Entrypoint package tests now run and pass. Linked identity, +release-target builds, and the final assembled acceptance audit remain Stage 9. + **Primary files:** old CLI parser/dispatch/inspection files and tests, `internal/format`, documentation and maintained examples listed below. diff --git a/docs/roadmap/narratio-feature-request.md b/docs/roadmap/narratio-feature-request.md index 3cbd381..67174fe 100644 --- a/docs/roadmap/narratio-feature-request.md +++ b/docs/roadmap/narratio-feature-request.md @@ -1,8 +1,9 @@ # Prompt input requirements for Narratio -Status: accepted direction, pending implementation through the CommandKit -migration. This proposal records Narratio's consumer needs and their mapping -to the native interface; it does not describe shipped Scriptorium behavior. +Status: the native interface and focused consumer scenarios are implemented. +The [implementation plan](implementation.md) tracks final assembled verification. +This proposal records Narratio's consumer needs and their mapping to the native +interface; the [CLI contract](../cli.md) owns current behavior. The [CommandKit notepad](commandkit-notepad.md) records the migration decisions. ## Consumer need @@ -27,8 +28,8 @@ Default and explicit text output use the native YAML presentation. The [upstream CLI contract][requirements-cli] owns syntax and report details. Use the same application configuration and framework source selection as -execution, including `--config`, `--overlay`, and applicable application -selectors. The preferred native aliases `--profile`, `--var`, `--session-id`, and +execution, including `--config` and `--overlay`. There are no application-defined +source/schema selectors. The preferred native aliases `--profile`, `--var`, `--session-id`, and prompt/profile namespace flags supply the relevant discovery context. Preserve upstream selection semantics, including the prompt's default profile when an explicit profile is omitted. Qualified IDs identify installed resources; diff --git a/examples/collection-gpu.yml b/examples/collection-gpu.yml new file mode 100644 index 0000000..629e086 --- /dev/null +++ b/examples/collection-gpu.yml @@ -0,0 +1,6 @@ +config: + imports: + - path: collections.yml + required: true + - path: profiles/local-gpu.yml + required: true diff --git a/examples/collections.yml b/examples/collections.yml new file mode 100644 index 0000000..08bc483 --- /dev/null +++ b/examples/collections.yml @@ -0,0 +1,5 @@ +config: + imports: + - dir: config-fragments/collections + pattern: '*.yml' + required: true diff --git a/examples/config-fragments/collections/10-backends.yml b/examples/config-fragments/collections/10-backends.yml index 2c91b29..e71baec 100644 --- a/examples/config-fragments/collections/10-backends.yml +++ b/examples/config-fragments/collections/10-backends.yml @@ -1,2 +1,4 @@ -promptkit_sources: - - file: ./examples/framework/backends.yml +config: + imports: + - path: ../../framework/backends.yml + required: true diff --git a/examples/config-fragments/collections/20-profile.yml b/examples/config-fragments/collections/20-profile.yml index 287ced6..1ab7da6 100644 --- a/examples/config-fragments/collections/20-profile.yml +++ b/examples/config-fragments/collections/20-profile.yml @@ -1,2 +1,4 @@ -promptkit_sources: - - file: ./examples/profiles/local-fast.yaml +config: + imports: + - path: ../../profiles/local-fast.yaml + required: true diff --git a/examples/config-fragments/collections/30-prompt.yml b/examples/config-fragments/collections/30-prompt.yml index a0f089b..a815f09 100644 --- a/examples/config-fragments/collections/30-prompt.yml +++ b/examples/config-fragments/collections/30-prompt.yml @@ -1,2 +1,4 @@ -promptkit_sources: - - file: ./examples/prompts/generic.markdown_summary.yaml +config: + imports: + - path: ../../prompts/generic.markdown_summary.yaml + required: true diff --git a/examples/config-fragments/sources.yml b/examples/config-fragments/sources.yml index ce06586..9dbea30 100644 --- a/examples/config-fragments/sources.yml +++ b/examples/config-fragments/sources.yml @@ -1,5 +1,3 @@ -promptkit_sources: - - directory: ./examples/prompts - - directory: ./examples/profiles - - file: ./examples/framework/backends.yml -schema_dir: ./examples/schemas +promptkit: + configuration: ./examples/promptkit.yml + schema_dir: ./examples/schemas diff --git a/examples/config.collections.yml b/examples/config.collections.yml index c125932..5b99189 100644 --- a/examples/config.collections.yml +++ b/examples/config.collections.yml @@ -1,10 +1,3 @@ -config: - sequences: - - destination: [promptkit_sources] - key: file - mode: upsert - imports: - - dir: config-fragments/collections - pattern: '*.yml' - required: true -schema_dir: ./examples/schemas +promptkit: + configuration: ./examples/collections.yml + schema_dir: ./examples/schemas diff --git a/examples/config.discovery.yml b/examples/config.discovery.yml index 8aa2994..98aa2fa 100644 --- a/examples/config.discovery.yml +++ b/examples/config.discovery.yml @@ -1,2 +1,2 @@ -promptkit_sources: - - file: ./examples/discovery/framework.yml +promptkit: + configuration: ./examples/discovery/framework.yml diff --git a/examples/config.full.yml b/examples/config.full.yml index 57769c7..4d97fee 100644 --- a/examples/config.full.yml +++ b/examples/config.full.yml @@ -1,8 +1,6 @@ -promptkit_sources: - - directory: ./examples/prompts - - directory: ./examples/profiles - - file: ./examples/framework/backends.yml -schema_dir: ./examples/schemas +promptkit: + configuration: ./examples/promptkit.yml + schema_dir: ./examples/schemas server: addr: 127.0.0.1:8080 @@ -10,6 +8,3 @@ server: max_request_bytes: 16777216 max_artifact_bytes: 16777216 max_response_bytes: 16777216 - -defaults: - render_format: text diff --git a/examples/config.inheritance.yml b/examples/config.inheritance.yml index d246a07..8c21754 100644 --- a/examples/config.inheritance.yml +++ b/examples/config.inheritance.yml @@ -1,6 +1,4 @@ # Application paths are relative to the repository invocation directory. -promptkit_sources: - - directory: ./examples/prompts - - directory: ./examples/profiles - - file: ./examples/framework/inheritance.yml -schema_dir: ./examples/schemas +promptkit: + configuration: ./examples/inheritance.yml + schema_dir: ./examples/schemas diff --git a/examples/config.yml b/examples/config.yml index dccb6fc..5457ea4 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -1,12 +1,7 @@ -promptkit_sources: - - directory: ./examples/prompts - - directory: ./examples/profiles - - file: ./examples/framework/backends.yml -schema_dir: ./examples/schemas +promptkit: + configuration: ./examples/promptkit.yml + schema_dir: ./examples/schemas server: addr: :8080 artifact_root: . - -defaults: - render_format: text diff --git a/examples/definitions.yml b/examples/definitions.yml new file mode 100644 index 0000000..f2e4aae --- /dev/null +++ b/examples/definitions.yml @@ -0,0 +1,8 @@ +config: + imports: + - dir: prompts + pattern: '*.yaml' + required: true + - dir: profiles + pattern: '*.y*ml' + required: true diff --git a/examples/inheritance.yml b/examples/inheritance.yml new file mode 100644 index 0000000..f443d8b --- /dev/null +++ b/examples/inheritance.yml @@ -0,0 +1,6 @@ +config: + imports: + - path: definitions.yml + required: true + - path: framework/inheritance.yml + required: true diff --git a/examples/overlays/collection.yml b/examples/overlays/collection.yml index 52c1ea2..72af312 100644 --- a/examples/overlays/collection.yml +++ b/examples/overlays/collection.yml @@ -1,2 +1,2 @@ -promptkit_sources: - - file: ./examples/profiles/local-gpu.yml +promptkit: + configuration: ./examples/collection-gpu.yml diff --git a/examples/overlays/local.yml b/examples/overlays/local.yml index 3c20ccb..1a22148 100644 --- a/examples/overlays/local.yml +++ b/examples/overlays/local.yml @@ -1,4 +1,2 @@ -defaults: - render_format: json server: addr: 127.0.0.1:8080 diff --git a/examples/promptkit.yml b/examples/promptkit.yml new file mode 100644 index 0000000..7552cba --- /dev/null +++ b/examples/promptkit.yml @@ -0,0 +1,6 @@ +config: + imports: + - path: definitions.yml + required: true + - path: framework/backends.yml + required: true diff --git a/examples/render-markdown-summary.sh b/examples/render-markdown-summary.sh index 12b6ef0..1892fc5 100755 --- a/examples/render-markdown-summary.sh +++ b/examples/render-markdown-summary.sh @@ -5,9 +5,8 @@ repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$repo_root" -go run ./cmd/scriptorium render \ +go run ./cmd/scriptorium promptkit show prepared generic.markdown_summary \ --config ./examples/config.yml \ - --prompt generic.markdown_summary \ - --input transcript=./examples/fixtures/transcript.md \ - --input glossary=./examples/fixtures/glossary.yml \ + --input-file transcript=./examples/fixtures/transcript.md \ + --input-file glossary=./examples/fixtures/glossary.yml \ --format text diff --git a/examples/render-v0.9-features.sh b/examples/render-v0.9-features.sh index 6038629..25dfbbb 100755 --- a/examples/render-v0.9-features.sh +++ b/examples/render-v0.9-features.sh @@ -4,21 +4,18 @@ set -euo pipefail repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$repo_root" -go run ./cmd/scriptorium render \ +go run ./cmd/scriptorium promptkit show prepared generic.markdown_summary \ --config ./examples/config.full.yml \ - --prompt generic.markdown_summary \ --profile local-gpu \ --session-id example-session \ --reasoning-effort= \ - --input transcript=./examples/fixtures/transcript.md \ - --input glossary=./examples/fixtures/glossary.yml + --input-file transcript=./examples/fixtures/transcript.md \ + --input-file glossary=./examples/fixtures/glossary.yml -go run ./cmd/scriptorium inspect prompt \ +go run ./cmd/scriptorium promptkit show prompt generic.markdown_summary \ --config ./examples/config.full.yml \ - --prompt generic.markdown_summary \ --format json -go run ./cmd/scriptorium inspect profile \ +go run ./cmd/scriptorium promptkit show profile local-gpu \ --config ./examples/config.full.yml \ - --profile local-gpu \ --format json diff --git a/go.mod b/go.mod index 4ba7b11..17adddf 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( go.maximumdirect.net/commandkit v0.7.0 go.maximumdirect.net/configkit v1.2.1 go.maximumdirect.net/promptkit v0.23.3 + go.yaml.in/yaml/v4 v4.0.0 ) require ( @@ -15,6 +16,5 @@ require ( github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 // indirect github.com/spf13/cobra v1.10.2 // indirect github.com/spf13/pflag v1.1.0 // indirect - go.yaml.in/yaml/v4 v4.0.0 // indirect golang.org/x/text v0.44.0 // indirect ) diff --git a/internal/adapter/cli/application_config.go b/internal/adapter/cli/application_config.go deleted file mode 100644 index ed82519..0000000 --- a/internal/adapter/cli/application_config.go +++ /dev/null @@ -1,39 +0,0 @@ -package cli - -import ( - "errors" - "fmt" - "io" - - appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config" - "go.maximumdirect.net/configkit" -) - -func loadApplicationSettings(configPath, overlay string, explicit bool, overrides []configkit.Override) (appconfig.AppSettings, []configkit.Diagnostic, error) { - result, err := appconfig.Load(appconfig.LoadOptions{ - Path: configPath, Overlay: overlay, Explicit: explicit, Overrides: overrides, - }) - if err != nil { - return appconfig.AppSettings{}, nil, err - } - return appconfig.Settings(result.Config), result.Diagnostics(), nil -} - -func writeApplicationError(output io.Writer, label string, err error) { - var loadError *configkit.LoadError - if errors.As(err, &loadError) { - _, _ = fmt.Fprintf(output, "%s error:\n", label) - if writeErr := configkit.WriteDiagnostics(output, loadError.Diagnostics()); writeErr != nil { - _, _ = fmt.Fprintf(output, "%s diagnostic write error: %v\n", label, writeErr) - } - return - } - _, _ = fmt.Fprintf(output, "%s error: %v\n", label, err) -} - -func writeApplicationWarnings(output io.Writer, diagnostics []configkit.Diagnostic) error { - if len(diagnostics) == 0 { - return nil - } - return configkit.WriteDiagnostics(output, diagnostics) -} diff --git a/internal/adapter/cli/config.go b/internal/adapter/cli/config.go deleted file mode 100644 index 9a60229..0000000 --- a/internal/adapter/cli/config.go +++ /dev/null @@ -1,101 +0,0 @@ -package cli - -import ( - "encoding/json" - "flag" - "fmt" - "io" - - appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config" - "go.maximumdirect.net/configkit" -) - -type configurationSource struct { - ID string `json:"id"` - Label string `json:"label"` - Layer configkit.Layer `json:"layer"` - Path string `json:"path,omitempty"` -} - -func configCommand(args []string, stdout, stderr io.Writer) int { - if len(args) == 0 { - fmt.Fprintln(stderr, "config parse error: configuration mode is required") - return ExitRuntimeError - } - mode := args[0] - if mode != "validate" && mode != "show" && mode != "sources" && mode != "explain" { - fmt.Fprintln(stderr, "config parse error: unknown configuration mode") - return ExitRuntimeError - } - - fs := flag.NewFlagSet("config "+mode, flag.ContinueOnError) - fs.SetOutput(io.Discard) - var path, overlay, pointer string - registerConfigSelectionFlags(fs, &path, &overlay) - registerConfigurationOverrides(fs) - if mode == "explain" { - fs.StringVar(&pointer, "path", "", "configuration path as a JSON Pointer") - } - if err := fs.Parse(args[1:]); err != nil { - writeApplicationError(stderr, "config parse", err) - return ExitRuntimeError - } - if fs.NArg() != 0 { - fmt.Fprintln(stderr, "config parse error: positional arguments are not accepted") - return ExitRuntimeError - } - var explanationPath configkit.Path - if mode == "explain" { - if !flagWasSet(fs, "path") { - fmt.Fprintln(stderr, "config parse error: --path is required") - return ExitRuntimeError - } - var err error - explanationPath, err = configkit.ParsePath(pointer) - if err != nil { - fmt.Fprintln(stderr, "config parse error: --path must be a JSON Pointer") - return ExitRuntimeError - } - } - - result, err := appconfig.Load(appconfig.LoadOptions{ - Path: path, Explicit: flagWasSet(fs, "config"), Overlay: overlay, - Overrides: applicationOverrides(fs), - }) - if err != nil { - writeApplicationError(stderr, "config", err) - return ExitRuntimeError - } - if err := writeApplicationWarnings(stderr, result.Diagnostics()); err != nil { - return ExitRuntimeError - } - switch mode { - case "validate": - return ExitOK - case "show": - err = result.WriteEffective(stdout) - case "sources": - sources := make([]configurationSource, 0, len(result.Sources())) - for _, source := range result.Sources() { - sources = append(sources, configurationSource{ - ID: source.ID, Label: source.Label, Layer: source.Layer, Path: source.Path, - }) - } - err = json.NewEncoder(stdout).Encode(sources) - case "explain": - err = result.WriteExplanation(stdout, explanationPath) - } - if err != nil { - fmt.Fprintf(stderr, "config output error: %v\n", err) - return ExitRuntimeError - } - return ExitOK -} - -func registerConfigurationOverrides(fs *flag.FlagSet) { - fs.String("addr", "", "HTTP listen address override") - fs.String("artifact-root", "", "HTTP artifact root override") - fs.Int64("max-request-bytes", 0, "HTTP request byte limit override") - fs.Int64("max-artifact-bytes", 0, "HTTP artifact byte limit override") - fs.Int64("max-response-bytes", 0, "HTTP response byte limit override") -} diff --git a/internal/adapter/cli/config_test.go b/internal/adapter/cli/config_test.go deleted file mode 100644 index 5fed75f..0000000 --- a/internal/adapter/cli/config_test.go +++ /dev/null @@ -1,226 +0,0 @@ -package cli - -import ( - "bytes" - "encoding/json" - "errors" - "io" - "os" - "path/filepath" - "reflect" - "strings" - "testing" - - appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config" -) - -func TestConfigCommandsInspectApplicationWithoutEngine(t *testing.T) { - path := writeAppConfigFile(t, "{}\n") - for _, mode := range []string{"validate", "show", "sources"} { - code, stdout, stderr := runCLICommand(t, configCommand, []string{mode, "--config", path}) - if code != ExitOK || stderr != "" { - t.Fatalf("%s failed: code=%d stdout=%q stderr=%q", mode, code, stdout, stderr) - } - switch mode { - case "validate": - if stdout != "" { - t.Fatalf("validation wrote stdout: %q", stdout) - } - case "show": - if !strings.Contains(stdout, "render_format: text") || !strings.Contains(stdout, "max_request_bytes: 16777216") { - t.Fatalf("effective defaults missing: %q", stdout) - } - case "sources": - var sources []configurationSource - if err := json.Unmarshal([]byte(stdout), &sources); err != nil || len(sources) != 1 || sources[0].Layer != "main" { - t.Fatalf("unexpected source records: %q (%v)", stdout, err) - } - } - } -} - -func TestCollectionExampleLoadsAndRenders(t *testing.T) { - t.Chdir(filepath.Join("..", "..", "..")) - want := []appconfig.SourceSelection{ - {File: "examples/framework/backends.yml"}, - {File: "examples/profiles/local-fast.yaml"}, - {File: "examples/prompts/generic.markdown_summary.yaml"}, - } - for _, overlay := range []string{"", "collection"} { - result, err := appconfig.Load(appconfig.LoadOptions{Path: "examples/config.collections.yml", Explicit: true, Overlay: overlay}) - if err != nil { - t.Fatal(err) - } - expected := append([]appconfig.SourceSelection{}, want...) - if overlay != "" { - expected = append(expected, appconfig.SourceSelection{File: "examples/profiles/local-gpu.yml"}) - } - if !reflect.DeepEqual(result.Config.PromptKitSources, expected) { - t.Fatalf("overlay %q selectors = %+v, want %+v", overlay, result.Config.PromptKitSources, expected) - } - args := []string{"--config", "examples/config.collections.yml", "--prompt", "generic.markdown_summary", "--input", "transcript=examples/fixtures/transcript.md", "--input", "glossary=examples/fixtures/glossary.yml"} - if overlay != "" { - args = append(args, "--config-overlay", overlay) - } - code, stdout, stderr := runCLICommand(t, renderCommand, args) - if code != ExitOK || !strings.Contains(stdout, "generic.markdown_summary") || !strings.Contains(stdout, "local-fast") { - t.Fatalf("collection render failed: code=%d stdout=%q stderr=%q", code, stdout, stderr) - } - } -} - -func TestConfigCommandsExposeComposedSourcesAndResolvedValues(t *testing.T) { - dir := t.TempDir() - if err := os.MkdirAll(filepath.Join(dir, "overlays"), 0o755); err != nil { - t.Fatal(err) - } - base := filepath.Join(dir, "config.yml") - for path, body := range map[string]string{ - base: "config:\n imports:\n - path: fragment.yml\n required: true\npromptkit_sources:\n - directory: ' ./before/../prompts '\nschema_dir: ./schemas\n", - filepath.Join(dir, "fragment.yml"): "server:\n addr: :8081\n", - filepath.Join(dir, "overlays", "local.yml"): "promptkit_sources:\n - directory: ./overlay-prompts\nserver:\n addr: :9090\n", - } { - if err := os.WriteFile(path, []byte(body), 0o600); err != nil { - t.Fatal(err) - } - } - args := []string{"--config", base, "--config-overlay", "local", "--schema-dir=", "--addr", ":7070"} - code, stdout, stderr := runCLICommand(t, configCommand, append([]string{"show"}, args...)) - if code != ExitOK || stderr != "" || !strings.Contains(stdout, "directory: overlay-prompts") || !strings.Contains(stdout, "schema_dir: ''") || !strings.Contains(stdout, "addr: :7070") || strings.Contains(stdout, "before/../prompts") { - t.Fatalf("resolved effective output changed: code=%d out=%q err=%q", code, stdout, stderr) - } - code, stdout, stderr = runCLICommand(t, configCommand, append([]string{"sources"}, args...)) - if code != ExitOK || stderr != "" || !strings.HasSuffix(stdout, "\n") { - t.Fatalf("source command failed: code=%d out=%q err=%q", code, stdout, stderr) - } - var sources []configurationSource - if err := json.Unmarshal([]byte(stdout), &sources); err != nil || len(sources) != 5 { - t.Fatalf("unexpected composed source records: %+v (%v)", sources, err) - } - for i, layer := range []string{"main", "main", "overlay", "cli", "cli"} { - if string(sources[i].Layer) != layer { - t.Fatalf("source %d layer = %q, want %q", i, sources[i].Layer, layer) - } - } -} - -func TestApplicationInspectionDoesNotAcquireFrameworkSources(t *testing.T) { - dir := t.TempDir() - frameworkPath := filepath.Join(dir, "invalid-framework.yml") - if err := os.WriteFile(frameworkPath, []byte("not: [valid\n"), 0o600); err != nil { - t.Fatal(err) - } - for _, frameworkSource := range []struct { - name string - path string - }{ - {name: "invalid definition", path: frameworkPath}, - {name: "missing path", path: filepath.Join(dir, "missing-framework.yml")}, - } { - t.Run(frameworkSource.name, func(t *testing.T) { - path := writeAppConfigFile(t, "promptkit_sources:\n - file: "+frameworkSource.path+"\n") - for _, args := range [][]string{ - {"validate"}, - {"show"}, - {"sources"}, - {"explain", "--path", "/promptkit_sources/0/file"}, - } { - code, stdout, stderr := runCLICommand(t, configCommand, append(args, "--config", path)) - if code != ExitOK || stderr != "" { - t.Fatalf("application inspection acquired configured framework source: args=%q code=%d out=%q err=%q", args, code, stdout, stderr) - } - } - }) - } -} - -func TestConfigCommandsRejectRetiredFrameworkFields(t *testing.T) { - for _, field := range []string{"prompt_dir", "profile_dir", "backends"} { - t.Run(field, func(t *testing.T) { - path := writeAppConfigFile(t, field+": {}\n") - code, stdout, stderr := runCLICommand(t, configCommand, []string{"validate", "--config", path}) - if code != ExitRuntimeError || stdout != "" || stderr == "" { - t.Fatalf("retired application field %q was accepted: code=%d out=%q err=%q", field, code, stdout, stderr) - } - }) - } -} - -type errorOutputWriter struct{} - -func (errorOutputWriter) Write([]byte) (int, error) { return 0, errors.New("output unavailable") } - -func TestConfigCommandFailuresDoNotProduceSuccessfulOutput(t *testing.T) { - path := writeAppConfigFile(t, "server:\n max_request_bytes: -1\n") - for _, mode := range []string{"validate", "show", "sources"} { - code, stdout, stderr := runCLICommand(t, configCommand, []string{mode, "--config", path}) - if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "validate.error") { - t.Fatalf("%s misclassified invalid config: code=%d out=%q err=%q", mode, code, stdout, stderr) - } - } - path = writeAppConfigFile(t, "{}\n") - var stderr bytes.Buffer - if code := configCommand([]string{"show", "--config", path}, errorOutputWriter{}, &stderr); code != ExitRuntimeError || !strings.Contains(stderr.String(), "output error") { - t.Fatalf("writer error not reported: code=%d stderr=%q", code, stderr.String()) - } - if code := configCommand([]string{"sources", "--config", path}, errorOutputWriter{}, io.Discard); code != ExitRuntimeError { - t.Fatalf("source writer error returned %d", code) - } -} - -func TestConfigExplainUsesResolvedHistory(t *testing.T) { - dir := t.TempDir() - if err := os.MkdirAll(filepath.Join(dir, "overlays"), 0o755); err != nil { - t.Fatal(err) - } - base := filepath.Join(dir, "config.yml") - for path, body := range map[string]string{ - base: `promptkit_sources: - - directory: " ./prompts/../prompts " -server: - addr: :8080 -`, - filepath.Join(dir, "overlays", "local.yml"): "server:\n addr: :9090\n", - } { - if err := os.WriteFile(path, []byte(body), 0o600); err != nil { - t.Fatal(err) - } - } - args := []string{"--config", base, "--config-overlay", "local", "--addr", ":9090"} - for _, tc := range []struct { - pointer string - want []string - }{ - {pointer: "/server/addr", want: []string{":8080", ":9090", "--addr"}}, - {pointer: "/promptkit_sources/0/directory", want: []string{"normalized application path", "prompts"}}, - {pointer: "", want: []string{"promptkit_sources", "server", "addr"}}, - } { - code, stdout, stderr := runCLICommand(t, configCommand, append([]string{"explain", "--path", tc.pointer}, args...)) - if code != ExitOK || stderr != "" { - t.Fatalf("explain %q failed: code=%d out=%q err=%q", tc.pointer, code, stdout, stderr) - } - for _, want := range tc.want { - if !strings.Contains(stdout, want) { - t.Fatalf("explain %q lacks %q: %q", tc.pointer, want, stdout) - } - } - } -} - -func TestConfigExplainRejectsUnknownOrMalformedPaths(t *testing.T) { - path := writeAppConfigFile(t, "{}\n") - for _, args := range [][]string{ - {"explain", "--config", path}, - {"explain", "--config", path, "--path", "not-a-pointer"}, - {"explain", "--config", path, "--path", "/missing"}, - } { - code, stdout, stderr := runCLICommand(t, configCommand, args) - if code != ExitRuntimeError || stdout != "" || stderr == "" { - t.Fatalf("invalid explanation path accepted: args=%q code=%d out=%q err=%q", args, code, stdout, stderr) - } - } - var stderr bytes.Buffer - if code := configCommand([]string{"explain", "--config", path, "--path", ""}, errorOutputWriter{}, &stderr); code != ExitRuntimeError || !strings.Contains(stderr.String(), "output error") { - t.Fatalf("explanation writer failure not reported: code=%d err=%q", code, stderr.String()) - } -} diff --git a/internal/adapter/cli/discovery_test.go b/internal/adapter/cli/discovery_test.go index d8738ad..d8f6ddd 100644 --- a/internal/adapter/cli/discovery_test.go +++ b/internal/adapter/cli/discovery_test.go @@ -1,7 +1,9 @@ package cli import ( + "encoding/json" "fmt" + "go.maximumdirect.net/promptkit" "os" "path/filepath" "strings" @@ -41,37 +43,46 @@ func TestCollectionCheckoutDiscoveryAcrossInvocations(t *testing.T) { t.Fatal(err) } } - config := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n", frameworkPath)) + config := filepath.Join(root, "app.yml") + writeNativeFile(t, config, fmt.Sprintf("promptkit:\n configuration: %q\n", frameworkPath)) input := filepath.Join(root, "transcript.md") if err := os.WriteFile(input, []byte("synthetic discovery transcript"), 0o644); err != nil { t.Fatal(err) } args := func(name string) []string { if name == "support-repo" { - return []string{"--config", config, "--prompt", "support/answer", "--profile", "support/fast", "--input", "transcript=" + input} + return []string{"support/answer", "--config", config, "--profile", "support/fast", "--input-file", "transcript=" + input} } - return []string{"--config", config, "--prompt", "review/answer", "--profile", "review/base"} + return []string{"review/answer", "--config", config, "--profile", "review/base"} } check := func(name string) { t.Helper() - code, out, stderr := runCLICommand(t, renderCommand, args(name)) - if code != ExitOK { - t.Fatalf("render %s: code=%d stderr=%s", name, code, stderr) + command := append([]string{"promptkit", "show", "prepared"}, args(name)...) + outcome, out, stderr := executeNative(t, append(command, "--format=json")...) + if outcome.Status != 0 { + t.Fatalf("prepare %s: outcome=%+v stderr=%s", name, outcome, stderr) + } + var prepared promptkit.PreparedRun + if err := json.Unmarshal([]byte(out), &prepared); err != nil { + t.Fatal(err) } if name == "support-repo" { - for _, want := range []string{"prompt: support/answer", "selected_profile_id: support/fast", "model: support-example", "max_tokens: 256", "Answer the support request", "synthetic discovery transcript"} { + for _, want := range []string{"support/answer", "support/fast", "support-example", "Answer the support request", "synthetic discovery transcript"} { if !strings.Contains(out, want) { t.Fatalf("missing %q: %s", want, out) } } - } else if !strings.Contains(out, "prompt: review/answer") || !strings.Contains(out, "model: review-example") || !strings.Contains(out, "Review this material") { + if prepared.EffectiveModelParams.MaxTokens != 256 { + t.Fatalf("inherited profile setting missing: %+v", prepared) + } + } else if !strings.Contains(out, "review/answer") || !strings.Contains(out, "review-example") || !strings.Contains(out, "Review this material") { t.Fatalf("unexpected review render: %s", out) } } install(tc.first) check(tc.first) - code, _, _ := runCLICommand(t, renderCommand, args(tc.second)) - if code != ExitRuntimeError { + outcome, _, _ := executeNative(t, append([]string{"promptkit", "show", "prepared"}, args(tc.second)...)...) + if outcome.Status != 1 { t.Fatal("uninstalled collection unexpectedly selected") } install(tc.second) diff --git a/internal/adapter/cli/examples_test.go b/internal/adapter/cli/examples_test.go new file mode 100644 index 0000000..6d5f819 --- /dev/null +++ b/internal/adapter/cli/examples_test.go @@ -0,0 +1,42 @@ +package cli + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestMaintainedApplicationExamplesUseNativeLoading(t *testing.T) { + root, err := filepath.Abs(filepath.Join("..", "..", "..")) + if err != nil { + t.Fatal(err) + } + t.Chdir(root) + entries, err := os.ReadDir("examples") + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if entry.IsDir() || !strings.HasPrefix(entry.Name(), "config") || !strings.HasSuffix(entry.Name(), ".yml") { + continue + } + path := filepath.Join("examples", entry.Name()) + for _, scope := range []string{"app", "promptkit"} { + outcome, _, diagnostics := executeNative(t, "config", "validate", "--config="+path, "--scope="+scope) + if outcome.Status != 0 { + t.Fatalf("%s/%s: %+v %q", path, scope, outcome, diagnostics) + } + } + } + for _, test := range []struct{ config, overlay string }{{"examples/config.yml", ""}, {"examples/config.composed.yml", "local"}, {"examples/config.collections.yml", "collection"}, {"examples/config.inheritance.yml", ""}} { + args := []string{"promptkit", "show", "prepared", "generic.markdown_summary", "--config=" + test.config, "--profile=local-gpu", "--input-file=transcript=examples/fixtures/transcript.md", "--input-file=glossary=examples/fixtures/glossary.yml", "--format=json"} + if test.overlay != "" { + args = append(args, "--overlay="+test.overlay) + } + outcome, output, diagnostics := executeNative(t, args...) + if outcome.Status != 0 || !strings.Contains(output, "generic.markdown_summary") || !strings.Contains(output, "Session Transcript") { + t.Fatalf("%+v: %+v %q %q", test, outcome, output, diagnostics) + } + } +} diff --git a/internal/adapter/cli/framework_config.go b/internal/adapter/cli/framework_config.go deleted file mode 100644 index fbef4bc..0000000 --- a/internal/adapter/cli/framework_config.go +++ /dev/null @@ -1,159 +0,0 @@ -package cli - -import ( - "encoding/json" - "errors" - "flag" - "fmt" - "io" - - "go.maximumdirect.net/configkit" -) - -type frameworkConfigurationConfig struct { - mode, configPath, configOverlay, schemaDir string - path string - configExplicit bool - pathExplicit bool - overrides []configkit.Override - diagnostics []configkit.Diagnostic -} - -type frameworkConfigurationSource struct { - ID string `json:"id"` - Label string `json:"label"` - Layer configkit.Layer `json:"layer"` - Kind configkit.SourceKind `json:"kind"` - UnderlayIndex int `json:"underlay_index"` - Path string `json:"path,omitempty"` - ImportedFrom *frameworkConfigurationLocation `json:"imported_from,omitempty"` -} - -type frameworkConfigurationLocation struct { - SourceID string `json:"source_id"` - Line int `json:"line"` - Column int `json:"column"` -} - -func inspectFrameworkConfigurationCommand(args []string, stdout, stderr io.Writer) int { - cfg, err := parseFrameworkConfigurationArgs(args) - if err != nil { - writeApplicationError(stderr, "inspect configuration parse", err) - return ExitRuntimeError - } - var explanationPath configkit.Path - if cfg.mode == "explain" { - explanationPath, err = configkit.ParsePath(cfg.path) - if err != nil { - writeApplicationError(stderr, "inspect configuration path", err) - return ExitRuntimeError - } - } - settings, err := resolveAppSettingsForFrameworkConfiguration(cfg) - if err != nil { - writeApplicationError(stderr, "inspect configuration", err) - return ExitRuntimeError - } - if err := writeApplicationWarnings(stderr, cfg.diagnostics); err != nil { - return ExitRuntimeError - } - engine, err := newEngine(settings) - if err != nil { - writeApplicationError(stderr, "engine", err) - return ExitRuntimeError - } - view, err := engine.InspectConfiguration() - if err != nil { - writeApplicationError(stderr, "inspect configuration", err) - return ExitRuntimeError - } - if err := configkit.WriteDiagnostics(stderr, view.Diagnostics()); err != nil { - return ExitRuntimeError - } - if cfg.mode == "validate" { - return ExitOK - } - - if cfg.mode == "explain" { - if err := view.WriteExplanation(stdout, explanationPath); err != nil { - fmt.Fprintf(stderr, "inspect configuration explain %q: %v\n", cfg.path, err) - return ExitRuntimeError - } - return ExitOK - } - - switch cfg.mode { - case "show": - err = view.WriteEffective(stdout) - case "sources": - sources := frameworkConfigurationSources(view.Sources()) - err = json.NewEncoder(stdout).Encode(sources) - } - if err != nil { - fmt.Fprintf(stderr, "inspect configuration output error: %v\n", err) - return ExitRuntimeError - } - return ExitOK -} - -func parseFrameworkConfigurationArgs(args []string) (*frameworkConfigurationConfig, error) { - if len(args) == 0 { - return nil, errors.New("configuration operation is required") - } - cfg := &frameworkConfigurationConfig{mode: args[0]} - switch cfg.mode { - case "validate", "show", "sources", "explain": - default: - return nil, fmt.Errorf("unknown configuration operation %q", cfg.mode) - } - - fs := flag.NewFlagSet("inspect configuration "+cfg.mode, flag.ContinueOnError) - fs.SetOutput(io.Discard) - registerConfigSelectionFlags(fs, &cfg.configPath, &cfg.configOverlay) - fs.StringVar(&cfg.path, "path", "", "JSON Pointer to explain (required for explain)") - if err := fs.Parse(args[1:]); err != nil { - return nil, err - } - if fs.NArg() != 0 { - return nil, fmt.Errorf("unexpected positional args: %v", fs.Args()) - } - cfg.configExplicit = flagWasSet(fs, "config") - cfg.pathExplicit = flagWasSet(fs, "path") - if cfg.mode == "explain" && !cfg.pathExplicit { - return nil, errors.New("--path is required for explain") - } - cfg.overrides = applicationOverrides(fs) - return cfg, nil -} - -func resolveAppSettingsForFrameworkConfiguration(cfg *frameworkConfigurationConfig) (engineSettings, error) { - settings, diagnostics, err := loadApplicationSettings(cfg.configPath, cfg.configOverlay, cfg.configExplicit, cfg.overrides) - if err != nil { - return engineSettings{}, err - } - cfg.diagnostics = diagnostics - return engineSettings{ - promptkitSources: settings.PromptKitSources, - schemaDir: settings.SchemaDir, - transientRetries: settings.TransientRetries, - }, nil -} - -func frameworkConfigurationSources(sources []configkit.SourceInfo) []frameworkConfigurationSource { - result := make([]frameworkConfigurationSource, 0, len(sources)) - for _, source := range sources { - item := frameworkConfigurationSource{ - ID: source.ID, Label: source.Label, Layer: source.Layer, - Kind: source.Kind, UnderlayIndex: source.UnderlayIndex, Path: source.Path, - } - if source.ImportedFrom != nil { - item.ImportedFrom = &frameworkConfigurationLocation{ - SourceID: source.ImportedFrom.SourceID, - Line: source.ImportedFrom.Line, - Column: source.ImportedFrom.Column, - } - } - result = append(result, item) - } - return result -} diff --git a/internal/adapter/cli/framework_config_test.go b/internal/adapter/cli/framework_config_test.go deleted file mode 100644 index 0d95e5b..0000000 --- a/internal/adapter/cli/framework_config_test.go +++ /dev/null @@ -1,391 +0,0 @@ -package cli - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "os" - "path/filepath" - "strings" - "testing" - - renderformat "gitea.maximumdirect.net/eric/scriptorium/internal/format" - configkit "go.maximumdirect.net/configkit" -) - -func TestFrameworkConfigurationCommandsNeedNoPromptOrCredential(t *testing.T) { - configPath := writeAppConfigFile(t, "{}\n") - code, stdout, stderr := runCLICommand(t, inspectCommand, []string{ - "configuration", "validate", "--config", configPath, - }) - if code != ExitOK || stdout != "" || stderr != "" { - t.Fatalf("source-free validation failed: code=%d stdout=%q stderr=%q", code, stdout, stderr) - } - - t.Setenv("SCRIPTORIUM_FRAMEWORK_KEY", "") - root := t.TempDir() - frameworkFile := filepath.Join(root, "framework.yml") - writeFrameworkSourceFile(t, root, "framework.yml", `backends: - - id: local - endpoint: http://127.0.0.1:1/v1 - api_key_env: SCRIPTORIUM_FRAMEWORK_KEY -profiles: - - id: protected - backend: local - model: offline-model - api_key_required: true -`) - configPath = writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n", frameworkFile)) - for _, mode := range []string{"validate", "show", "sources"} { - code, stdout, stderr = runCLICommand(t, inspectCommand, []string{ - "configuration", mode, "--config", configPath, - }) - if code != ExitOK || stderr != "" { - t.Fatalf("%s required a prompt or credential: code=%d stdout=%q stderr=%q", mode, code, stdout, stderr) - } - if mode == "validate" && stdout != "" { - t.Fatalf("validation wrote success output: %q", stdout) - } - if mode == "show" && (!strings.Contains(stdout, "profiles:") || !strings.Contains(stdout, "offline-model")) { - t.Fatalf("effective framework YAML is missing: %q", stdout) - } - } -} - -func TestFrameworkConfigurationSourcesProjectsOrderedNativeRecords(t *testing.T) { - root := t.TempDir() - importDir := filepath.Join(root, "catalog") - if err := os.MkdirAll(importDir, 0o755); err != nil { - t.Fatal(err) - } - mainPath := writeFrameworkSourceFile(t, root, "main.yml", `config: - imports: - - path: catalog/fragment.yml - required: true -backends: - - id: local - endpoint: http://127.0.0.1:1/v1 -`) - fragmentPath := writeFrameworkSourceFile(t, importDir, "fragment.yml", `profiles: - - id: imported - model: imported-model -`) - directory := t.TempDir() - directoryPath := writeFrameworkSourceFile(t, directory, "directory.yml", `profiles: - - id: directory - model: directory-model -`) - configuredMissing := filepath.Join(root, "configured-but-replaced.yml") - configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n", configuredMissing)) - args := []string{ - "configuration", "sources", "--config", configPath, - "--promptkit-source", mainPath, - "--promptkit-source", directory, - } - code, stdout, stderr := runCLICommand(t, inspectCommand, args) - if code != ExitOK || stderr != "" || !strings.HasSuffix(stdout, "\n") { - t.Fatalf("framework sources command failed: code=%d stdout=%q stderr=%q", code, stdout, stderr) - } - var sources []frameworkConfigurationSource - if err := json.Unmarshal([]byte(stdout), &sources); err != nil { - t.Fatalf("decode framework source DTOs: %v\n%s", err, stdout) - } - var rawSources []map[string]any - if err := json.Unmarshal([]byte(stdout), &rawSources); err != nil { - t.Fatalf("decode framework source fields: %v", err) - } - mainIndex := findFrameworkSource(t, sources, mainPath) - fragmentIndex := findFrameworkSource(t, sources, fragmentPath) - directoryIndex := findFrameworkSource(t, sources, directoryPath) - main := sources[mainIndex] - fragment := sources[fragmentIndex] - directorySource := sources[directoryIndex] - if !(mainIndex < fragmentIndex && fragmentIndex < directoryIndex) { - t.Fatalf("captured source order changed: main=%d fragment=%d directory=%d", mainIndex, fragmentIndex, directoryIndex) - } - if main.ID == "" || main.Label == "" || main.Layer != configkit.LayerUnderlay || main.Kind != configkit.SourceKindFile || main.UnderlayIndex < 1 { - t.Fatalf("explicit file source fields are incomplete: %+v", main) - } - for _, key := range []string{"id", "label", "layer", "kind", "underlay_index", "path"} { - if _, exists := rawSources[mainIndex][key]; !exists { - t.Fatalf("source DTO omitted %q: %#v", key, rawSources[mainIndex]) - } - } - for _, key := range []string{"ID", "Label", "Layer", "Kind", "UnderlayIndex", "filesystem"} { - if _, exists := rawSources[mainIndex][key]; exists { - t.Fatalf("source DTO exposed implementation field %q: %#v", key, rawSources[mainIndex]) - } - } - if fragment.Layer != configkit.LayerUnderlay || fragment.Kind != configkit.SourceKindFile || fragment.ImportedFrom == nil || - fragment.ImportedFrom.SourceID != main.ID || fragment.ImportedFrom.Line < 1 || fragment.ImportedFrom.Column < 1 { - t.Fatalf("import source fields are incomplete: %+v", fragment) - } - if importedFrom, ok := rawSources[fragmentIndex]["imported_from"].(map[string]any); !ok || importedFrom["source_id"] != main.ID || importedFrom["line"] == nil || importedFrom["column"] == nil { - t.Fatalf("import location DTO fields are incomplete: %#v", rawSources[fragmentIndex]) - } - if directorySource.Layer != configkit.LayerUnderlay || directorySource.Kind != configkit.SourceKindFile || directorySource.UnderlayIndex <= main.UnderlayIndex { - t.Fatalf("directory-selected source fields/order are incomplete: %+v (main %+v)", directorySource, main) - } -} - -func TestFrameworkConfigurationShowFailsBeforeOutputForInvalidFinalDefinition(t *testing.T) { - root := t.TempDir() - frameworkFile := writeFrameworkSourceFile(t, root, "invalid.yml", `prompts: - - id: unused - messages: - - role: user - content: "{{" - output: - format: text - validation_mode: none -`) - configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n", frameworkFile)) - code, stdout, stderr := runCLICommand(t, configCommand, []string{"validate", "--config", configPath}) - if code != ExitOK || stdout != "" || stderr != "" { - t.Fatalf("application validation loaded framework definitions: code=%d stdout=%q stderr=%q", code, stdout, stderr) - } - for _, args := range [][]string{ - {"configuration", "validate", "--config", configPath}, - {"configuration", "show", "--config", configPath}, - {"configuration", "sources", "--config", configPath}, - {"configuration", "explain", "--config", configPath, "--path", ""}, - } { - code, stdout, stderr = runCLICommand(t, inspectCommand, args) - if code != ExitRuntimeError || stdout != "" || stderr == "" { - t.Fatalf("invalid final framework definition was reported as success: args=%q code=%d stdout=%q stderr=%q", args, code, stdout, stderr) - } - } -} - -func TestFrameworkConfigurationCommandsRejectExtraArgumentsAndReportWriterErrors(t *testing.T) { - configPath := writeAppConfigFile(t, "{}\n") - for _, args := range [][]string{ - {"configuration"}, - {"configuration", "explain"}, - {"configuration", "show", "unexpected"}, - {"configuration", "sources", "--format", "json"}, - } { - code, stdout, stderr := runCLICommand(t, inspectCommand, args) - if code != ExitRuntimeError || stdout != "" || stderr == "" { - t.Fatalf("invalid inspection arguments were accepted: args=%q code=%d stdout=%q stderr=%q", args, code, stdout, stderr) - } - } - - writerError := errors.New("synthetic framework output failure") - var stderr bytes.Buffer - code := inspectCommand([]string{"configuration", "show", "--config", configPath}, frameworkConfigurationFailingWriter{err: writerError}, &stderr) - if code != ExitRuntimeError || !strings.Contains(stderr.String(), writerError.Error()) { - t.Fatalf("effective-output writer failure was not reported: code=%d stderr=%q", code, stderr.String()) - } - stderr.Reset() - code = inspectCommand([]string{"configuration", "explain", "--config", configPath, "--path", ""}, frameworkConfigurationFailingWriter{err: writerError}, &stderr) - if code != ExitRuntimeError || !strings.Contains(stderr.String(), writerError.Error()) { - t.Fatalf("explanation writer failure was not reported: code=%d stderr=%q", code, stderr.String()) - } -} - -func TestFrameworkConfigurationExplainUsesLiteralJSONPointersAndNativeHistory(t *testing.T) { - root := t.TempDir() - lower := writeFrameworkSourceFile(t, root, "lower.yml", `collections: - support: - backends: - - id: local - endpoint: http://127.0.0.1:1/v1 - api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE - extra_params: - token: backend-token-displaced-canary - password: backend-password-canary - visible_option: backend-visible-value - provider.option: dotted-visible-value - profiles: - - id: summary - backend: support/local - model: lower-model - api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE - extra_params: - token: profile-token-displaced-canary - password: profile-password-displaced-canary - visible_option: profile-visible-value - provider.option: dotted-visible-value - prompts: - - id: explainable - default_profile: support/summary - messages: - - role: user - content: hello - output: - format: text - validation_mode: none -`) - upperRoot := t.TempDir() - upper := writeFrameworkSourceFile(t, upperRoot, "upper.yml", `collections: - support: - backends: - - id: local - endpoint: http://127.0.0.1:1/v1 - api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE - extra_params: - token: backend-token-effective-canary - password: backend-password-effective-canary - visible_option: backend-visible-value - provider.option: dotted-visible-value - profiles: - - id: summary - backend: support/local - model: effective-model - api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE - extra_params: - token: profile-token-effective-canary - password: profile-password-effective-canary - visible_option: profile-visible-value - provider.option: dotted-visible-value -`) - configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n - file: %q\n", lower, upper)) - const secret = "environment-value-must-never-be-emitted" - t.Setenv("SCRIPTORIUM_EXPLAIN_REFERENCE", secret) - - for _, tc := range []struct { - path string - want []string - unwant []string - }{ - { - path: "/collections/support/backends/0/extra_params/token", - unwant: []string{"backend-token-displaced-canary", "backend-token-effective-canary"}, - }, - { - path: "/collections/support/backends/0/extra_params/password", - unwant: []string{"backend-password-displaced-canary", "backend-password-effective-canary"}, - }, - { - path: "/collections/support/profiles/0/model", - want: []string{"effective-model", "lower-model"}, - unwant: []string{}, - }, - { - path: "/collections/support/profiles/0/extra_params/token", - unwant: []string{"profile-token-displaced-canary", "profile-token-effective-canary"}, - }, - { - path: "/collections/support/profiles/0/extra_params/password", - unwant: []string{"profile-password-displaced-canary", "profile-password-effective-canary"}, - }, - { - path: "/collections/support/profiles/0/api_key_env", - want: []string{"SCRIPTORIUM_EXPLAIN_REFERENCE"}, - unwant: []string{secret}, - }, - { - path: "/collections/support/profiles/0/extra_params/provider.option", - want: []string{"dotted-visible-value"}, - unwant: []string{"provider/option"}, - }, - { - path: "", - want: []string{"profiles:", "effective-model"}, - unwant: []string{"profile-token-displaced-canary", "backend-password-effective-canary"}, - }, - } { - args := []string{"configuration", "explain", "--config", configPath, "--path", tc.path} - code, stdout, stderr := runCLICommand(t, inspectCommand, args) - if code != ExitOK || stderr != "" { - t.Fatalf("explain %q failed: code=%d stdout=%q stderr=%q", tc.path, code, stdout, stderr) - } - if stdout == "" { - t.Fatalf("explain %q produced no output", tc.path) - } - for _, value := range tc.want { - if !strings.Contains(stdout, value) { - t.Errorf("explain %q omitted %q: %s", tc.path, value, stdout) - } - } - for _, value := range tc.unwant { - if strings.Contains(stdout, value) { - t.Errorf("explain %q disclosed %q: %s", tc.path, value, stdout) - } - } - if strings.Contains(stdout, secret) { - t.Errorf("explain %q disclosed the resolved environment value: %s", tc.path, stdout) - } - } - - code, stdout, stderr := runCLICommand(t, inspectCommand, []string{"configuration", "show", "--config", configPath}) - if code != ExitOK || stderr != "" { - t.Fatalf("framework show failed: code=%d stderr=%q", code, stderr) - } - for _, value := range []string{"SCRIPTORIUM_EXPLAIN_REFERENCE", "backend-visible-value", "profile-visible-value", "dotted-visible-value"} { - if !strings.Contains(stdout, value) { - t.Errorf("effective report omitted visible configuration %q: %s", value, stdout) - } - } - for _, value := range []string{ - "backend-token-displaced-canary", "backend-token-effective-canary", - "backend-password-displaced-canary", "backend-password-effective-canary", - "profile-token-displaced-canary", "profile-token-effective-canary", - "profile-password-displaced-canary", "profile-password-effective-canary", secret, - } { - if strings.Contains(stdout, value) { - t.Errorf("effective report disclosed classified or resolved value %q: %s", value, stdout) - } - } - - profileCode, profileOutput, profileError := runCLICommand(t, inspectCommand, []string{ - "profile", "--config", configPath, "--profile", "support/summary", "--format", "json", - }) - if profileCode != ExitOK || profileError != "" { - t.Fatalf("existing profile inspection failed: code=%d stdout=%q stderr=%q", profileCode, profileOutput, profileError) - } - var profile renderformat.ProfileInspection - if err := json.Unmarshal([]byte(profileOutput), &profile); err != nil { - t.Fatalf("decode profile inspection: %v\n%s", err, profileOutput) - } - if pointerValue(profile.ProfileModelParams.Model) != "effective-model" || pointerValue(profile.ProfileModelParams.ExtraParams)["visible_option"] != "profile-visible-value" { - t.Fatalf("existing profile inspection changed configured provider data: %+v", profile.ProfileModelParams) - } - if strings.Contains(profileOutput, secret) { - t.Fatalf("profile inspection emitted resolved environment data: %s", profileOutput) - } - - code, rendered, stderr := runCLICommand(t, renderCommand, []string{"--config", configPath, "--prompt", "support/explainable"}) - if code != ExitOK || stderr != "" { - t.Fatalf("prepared output failed: code=%d stdout=%q stderr=%q", code, rendered, stderr) - } - if !strings.Contains(rendered, "profile-visible-value") || strings.Contains(rendered, secret) { - t.Fatalf("prepared output did not preserve ordinary provider data or disclosed environment data: %s", rendered) - } -} - -func TestFrameworkConfigurationExplainRejectsMissingMalformedAndUnknownPaths(t *testing.T) { - configPath := writeAppConfigFile(t, "{}\n") - for _, args := range [][]string{ - {"configuration", "explain"}, - {"configuration", "explain", "--path", "not-a-pointer"}, - {"configuration", "explain", "--path", "/profiles/team~2summary/model"}, - {"configuration", "explain", "--config", configPath, "--path", "/profiles/missing/model"}, - } { - code, stdout, stderr := runCLICommand(t, inspectCommand, args) - if code != ExitRuntimeError || stdout != "" || stderr == "" { - t.Fatalf("invalid explanation path was accepted: args=%q code=%d stdout=%q stderr=%q", args, code, stdout, stderr) - } - } -} - -func findFrameworkSource(t *testing.T, sources []frameworkConfigurationSource, path string) int { - t.Helper() - for index, source := range sources { - if filepath.Clean(source.Path) == filepath.Clean(path) { - return index - } - } - t.Fatalf("selected source path %q was not present: %+v", path, sources) - return -1 -} - -type frameworkConfigurationFailingWriter struct { - err error -} - -func (writer frameworkConfigurationFailingWriter) Write([]byte) (int, error) { - return 0, writer.err -} diff --git a/internal/adapter/cli/framework_sources.go b/internal/adapter/cli/framework_sources.go deleted file mode 100644 index 088f777..0000000 --- a/internal/adapter/cli/framework_sources.go +++ /dev/null @@ -1,17 +0,0 @@ -package cli - -import ( - "fmt" - "io" - - "go.maximumdirect.net/configkit" - "go.maximumdirect.net/promptkit" -) - -func writeFrameworkDiagnostics(output io.Writer, engine *promptkit.Engine) error { - view, err := engine.InspectConfiguration() - if err != nil { - return fmt.Errorf("inspect captured PromptKit diagnostics: %w", err) - } - return configkit.WriteDiagnostics(output, view.Diagnostics()) -} diff --git a/internal/adapter/cli/framework_sources_test.go b/internal/adapter/cli/framework_sources_test.go deleted file mode 100644 index c1c7448..0000000 --- a/internal/adapter/cli/framework_sources_test.go +++ /dev/null @@ -1,386 +0,0 @@ -package cli - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "os" - "path/filepath" - "reflect" - "testing" - - appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config" - configkit "go.maximumdirect.net/configkit" - "go.maximumdirect.net/promptkit" -) - -func TestFrameworkSourceGroupsComposeAndValidateEffectiveDefinitions(t *testing.T) { - conflicting := t.TempDir() - writeFrameworkSourceFile(t, conflicting, "first.yml", `profiles: - - id: selected - model: first -`) - writeFrameworkSourceFile(t, conflicting, "second.yml", `profiles: - - id: selected - model: second -`) - if _, err := newEngine(engineSettings{promptkitSources: []appconfig.SourceSelection{{Directory: conflicting}}}); err == nil { - t.Fatal("conflicting files in one source group unexpectedly constructed an engine") - } - - lower := t.TempDir() - upper := t.TempDir() - empty := t.TempDir() - writeFrameworkSourceFile(t, lower, "lower.yml", `profiles: - - id: selected - endpoint: not-a-url - model: lower-model - temperature: 10 -`) - writeFrameworkSourceFile(t, upper, "upper.yml", `profiles: - - id: selected - endpoint: http://127.0.0.1:1/v1 - model: upper-model - temperature: 0.5 -`) - engine := newFrameworkSourceTestEngine(t, []appconfig.SourceSelection{ - {Directory: lower}, {Directory: empty}, {Directory: upper}, - }, "") - inspection, err := engine.InspectProfile(context.Background(), "selected") - if err != nil { - t.Fatalf("inspect profile after later source replacement: %v", err) - } - if pointerValue(inspection.Endpoint) != "http://127.0.0.1:1/v1" || pointerValue(inspection.Model) != "upper-model" || pointerValue(inspection.Temperature) != 0.5 { - t.Fatalf("later source did not replace lower values before validation: %+v", inspection) - } -} - -func TestFrameworkSourceConstructionUsesCombinedDocumentAndRejectsInvalidSources(t *testing.T) { - root := t.TempDir() - combined := filepath.Join(root, "combined.yml") - writeFrameworkSourceFile(t, root, "combined.yml", `backends: - - id: local - endpoint: http://127.0.0.1:1/v1 -profiles: - - id: selected - backend: local - model: combined-model -prompts: - - id: combined.prompt - default_profile: selected - messages: - - role: user - content: hello - output: - format: text - validation_mode: none -`) - client := &frameworkSourceCaptureClient{} - engine := newFrameworkSourceTestEngine(t, []appconfig.SourceSelection{{File: combined}}, "", promptkit.WithLLMClient(client)) - if _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "combined.prompt"}); err != nil { - t.Fatalf("run definitions from one combined document: %v", err) - } - if len(client.requests) != 1 || client.requests[0].Target.BackendID != "local" || client.requests[0].Target.Model != "combined-model" { - t.Fatalf("combined backend/profile/prompt values did not reach generation: %+v", client.requests) - } - - malformed := filepath.Join(root, "malformed.yml") - writeFrameworkSourceFile(t, root, "malformed.yml", "not: [valid\n") - if _, err := newEngine(engineSettings{promptkitSources: []appconfig.SourceSelection{{File: malformed}}}); err == nil { - t.Fatal("malformed framework source unexpectedly constructed an engine") - } - - invalidUnused := filepath.Join(root, "invalid-unused.yml") - writeFrameworkSourceFile(t, root, "invalid-unused.yml", `prompts: - - id: unused.prompt - messages: - - role: user - content: "{{" - output: - format: text - validation_mode: none -`) - if _, err := newEngine(engineSettings{promptkitSources: []appconfig.SourceSelection{{File: invalidUnused}}}); !errors.Is(err, promptkit.ErrInvalidConfig) { - t.Fatalf("invalid unused definition did not fail as framework configuration: %v", err) - } -} - -func TestFrameworkRejectsObsoleteVersionKeyedPrompt(t *testing.T) { - root := t.TempDir() - path := writeFrameworkSourceFile(t, root, "obsolete.yml", `prompts: - old-prompt: - "1": - messages: - - role: user - content: hello - output: - format: text - validation_mode: none -`) - if _, err := newEngine(engineSettings{promptkitSources: []appconfig.SourceSelection{{File: path}}}); !errors.Is(err, promptkit.ErrInvalidConfig) { - t.Fatalf("obsolete version-keyed definition was not rejected by PromptKit: %v", err) - } -} - -func TestFrameworkImportsRetainResourceOriginsAndCaptureAtConstruction(t *testing.T) { - root := t.TempDir() - catalogDir := filepath.Join(root, "catalog") - if err := os.MkdirAll(catalogDir, 0o755); err != nil { - t.Fatal(err) - } - schemaDir := filepath.Join(root, "schemas") - if err := os.MkdirAll(schemaDir, 0o755); err != nil { - t.Fatal(err) - } - mainFile := filepath.Join(root, "main.yml") - writeFrameworkSourceFile(t, root, "main.yml", `config: - imports: - - path: catalog/base.yml - required: true -`) - writeFrameworkSourceFile(t, catalogDir, "base.yml", `profiles: - - id: profile-primary - endpoint: http://127.0.0.1:1/v1 - model: imported-model -prompts: - - id: team-prompt - default_profile: profile-primary - inputs: - - name: source - required: true - messages: - - role: user - content_file: ./body.md - - role: user - content: 'Artifact: {{input "source"}}' - output: - format: json - validation_mode: json_schema - schema_path: response.schema.json -`) - writeFrameworkSourceFile(t, catalogDir, "body.md", "Captured message for {{.audience}}") - artifactPath := filepath.Join(root, "request-artifact.txt") - writeFrameworkSourceFile(t, root, "request-artifact.txt", "initial artifact") - writeFrameworkSourceFile(t, schemaDir, "response.schema.json", `{"type":"object","title":"initial schema"}`) - selections := []appconfig.SourceSelection{{File: mainFile}} - engine := newFrameworkSourceTestEngine(t, selections, schemaDir) - - writeFrameworkSourceFile(t, catalogDir, "body.md", "edited message for {{.audience}}") - writeFrameworkSourceFile(t, root, "request-artifact.txt", "prepared artifact") - writeFrameworkSourceFile(t, schemaDir, "response.schema.json", `{"type":"object","title":"prepared schema"}`) - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: "team-prompt", - Inputs: map[string]promptkit.ArtifactRef{"source": promptkit.File(artifactPath)}, - Vars: map[string]string{"audience": "reader"}, - }) - if err != nil { - t.Fatalf("prepare inherited imported prompt: %v", err) - } - if len(prepared.Messages) != 2 || prepared.Messages[0].Content != "Captured message for reader" || prepared.Messages[1].Content != "Artifact: prepared artifact" { - t.Fatalf("resource origin or capture/preparation behavior changed: %+v", prepared.Messages) - } - if prepared.SelectedProfileID != "profile-primary" { - t.Fatalf("profile ID was not preserved: %+v", prepared) - } - if prepared.StructuredOutput == nil || prepared.StructuredOutput.JSONSchema == nil || prepared.StructuredOutput.JSONSchema.Schema.(map[string]any)["title"] != "prepared schema" { - t.Fatalf("schema was not loaded during preparation: %+v", prepared.StructuredOutput) - } - - newEngine := newFrameworkSourceTestEngine(t, selections, schemaDir) - newPrepared, err := newEngine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: "team-prompt", - Inputs: map[string]promptkit.ArtifactRef{"source": promptkit.File(artifactPath)}, - Vars: map[string]string{"audience": "reader"}, - }) - if err != nil { - t.Fatalf("prepare with newly captured message: %v", err) - } - if newPrepared.Messages[0].Content != "edited message for reader" { - t.Fatalf("new engine did not observe the message edit: %+v", newPrepared.Messages[0]) - } - - configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\nschema_dir: %q\n", mainFile, schemaDir)) - code, stdout, stderr := runCLICommand(t, renderCommand, []string{ - "--config", configPath, - "--prompt", "team-prompt", - "--input", "source=" + artifactPath, - "--var", "audience=reader", - "--format", "json", - }) - if code != ExitOK { - t.Fatalf("render exact literal IDs through CLI: code=%d stderr=%q", code, stderr) - } - var rendered promptkit.PreparedRun - if err := json.Unmarshal([]byte(stdout), &rendered); err != nil { - t.Fatalf("decode rendered literal-ID fixture: %v", err) - } - if rendered.PromptID != "team-prompt" || rendered.SelectedProfileID != "profile-primary" { - t.Fatalf("CLI changed exact prompt/profile IDs: %+v", rendered) - } -} - -func TestNativeProfileInheritanceAndPresenceReachTheClientBoundary(t *testing.T) { - root := t.TempDir() - frameworkFile := filepath.Join(root, "framework.yml") - writeFrameworkSourceFile(t, root, "framework.yml", `backends: - - id: first - endpoint: https://first.example/v1 - api_key_env: FIRST_BACKEND_KEY - extra_params: - source: first - nested: - first_backend: true - - id: second - endpoint: https://second.example/v1 - api_key_env: SECOND_BACKEND_KEY - extra_params: - source: second - nested: - second_backend: true -profiles: - - id: parent - backend: first - model: parent-model - temperature: 0.7 - top_p: 0.6 - max_tokens: 45 - timeout_seconds: 30 - service_tier: flex - extra_params: - nested: - parent_profile: true - retained: "yes" - - id: backend-child - extends: {id: parent} - backend: second - model: child-model - - id: presence-child - extends: {id: parent} - backend: second - api_key_env: null - temperature: null - max_tokens: 0 - timeout_seconds: 0 - extra_params: - nested: - explicit_false: false - - id: map-null - backend: second - model: map-model - extra_params: null - - id: map-empty - backend: second - model: map-model - extra_params: {} -prompts: - - id: execution.prompt - messages: - - role: user - content: hello - output: - format: text - validation_mode: none -`) - client := &frameworkSourceCaptureClient{} - engine := newFrameworkSourceTestEngine(t, []appconfig.SourceSelection{{File: frameworkFile}}, "", promptkit.WithLLMClient(client)) - - for _, profileID := range []string{"backend-child", "presence-child", "map-null", "map-empty"} { - if _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "execution.prompt", ProfileID: profileID}); err != nil { - t.Fatalf("run profile %q: %v", profileID, err) - } - } - if len(client.requests) != 4 { - t.Fatalf("expected four provider-bound requests, got %d", len(client.requests)) - } - - backendChild := client.requests[0].Target - wantBackendParams := map[string]any{"source": "second", "nested": map[string]any{ - "second_backend": true, "parent_profile": true, "retained": "yes", - }} - if backendChild.BackendID != "second" || backendChild.Endpoint != "https://second.example/v1" || backendChild.APIKeyEnv != "SECOND_BACKEND_KEY" || backendChild.Model != "child-model" || - backendChild.Temperature != 0.7 || backendChild.TopP != 0.6 || backendChild.MaxTokens != 45 || backendChild.TimeoutSeconds != 30 || backendChild.ServiceTier != "flex" || !reflect.DeepEqual(backendChild.ExtraParams, wantBackendParams) { - t.Fatalf("backend change lost backend-derived or explicit inherited values: %+v", backendChild) - } - - presenceRequest := client.requests[1] - presenceTarget := presenceRequest.Target - if presenceTarget.APIKeyEnv != "SECOND_BACKEND_KEY" || presenceTarget.Temperature != 0 || presenceTarget.MaxTokens != 0 || presenceTarget.TimeoutSeconds != 0 || - !presenceRequest.TargetPresence.MaxTokens || !presenceRequest.TargetPresence.TimeoutSeconds || presenceRequest.TargetPresence.Temperature || !presenceRequest.TargetPresence.TopP { - t.Fatalf("explicit null/zero presence did not reach the injected client: target=%+v presence=%+v", presenceTarget, presenceRequest.TargetPresence) - } - wantPresenceParams := map[string]any{"source": "second", "nested": map[string]any{ - "second_backend": true, "parent_profile": true, "retained": "yes", "explicit_false": false, - }} - if !reflect.DeepEqual(presenceTarget.ExtraParams, wantPresenceParams) { - t.Fatalf("recursive provider-map retention changed: %#v", presenceTarget.ExtraParams) - } - if !reflect.DeepEqual(client.requests[2].Target.ExtraParams, map[string]any{"source": "second", "nested": map[string]any{"second_backend": true}}) { - t.Fatalf("whole-map null did not fall back to backend defaults: %#v", client.requests[2].Target.ExtraParams) - } - if !reflect.DeepEqual(client.requests[3].Target.ExtraParams, map[string]any{"source": "second", "nested": map[string]any{"second_backend": true}}) { - t.Fatalf("empty provider map unexpectedly cleared backend defaults: %#v", client.requests[3].Target.ExtraParams) - } -} - -func TestCLICompletesPartialFrameworkProfileWithRequestOverrides(t *testing.T) { - root := t.TempDir() - frameworkFile := writeFrameworkSourceFile(t, root, "partial.yml", `profiles: - - id: partial -prompts: - - id: partial.prompt - default_profile: partial - messages: - - role: user - content: hello - output: - format: text - validation_mode: none -`) - provider := newTestLLMServer("completed partial profile", nil) - defer provider.Close() - configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n", frameworkFile)) - code, stdout, stderr := runCLICommand(t, runCommand, []string{ - "--config", configPath, - "--prompt", "partial.prompt", - "--llm-base-url", provider.URL + "/v1", - "--model", "request-model", - }) - if code != ExitOK || stdout != "completed partial profile" { - t.Fatalf("request overrides did not complete the partial profile: code=%d stdout=%q stderr=%q", code, stdout, stderr) - } -} - -func newFrameworkSourceTestEngine(t *testing.T, selections []appconfig.SourceSelection, schemaDir string, options ...promptkit.Option) *promptkit.Engine { - t.Helper() - engine, err := newEngine(engineSettings{promptkitSources: selections, schemaDir: schemaDir}, options...) - if err != nil { - var loadError *configkit.LoadError - if errors.As(err, &loadError) { - t.Fatalf("construct engine from selected framework sources: %v (%+v)", err, loadError.Diagnostics()) - } - t.Fatalf("construct engine from selected framework sources: %v", err) - } - return engine -} - -func writeFrameworkSourceFile(t *testing.T, directory, name, contents string) string { - t.Helper() - if err := os.MkdirAll(directory, 0o755); err != nil { - t.Fatalf("create framework source directory: %v", err) - } - path := filepath.Join(directory, name) - if err := os.WriteFile(path, []byte(contents), 0o644); err != nil { - t.Fatalf("write framework source %q: %v", path, err) - } - return path -} - -type frameworkSourceCaptureClient struct { - requests []promptkit.GenerateRequest -} - -func (client *frameworkSourceCaptureClient) Generate(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { - client.requests = append(client.requests, request) - return &promptkit.GenerateResponse{Content: "ok"}, nil -} diff --git a/internal/adapter/cli/inspect.go b/internal/adapter/cli/inspect.go deleted file mode 100644 index 77c8267..0000000 --- a/internal/adapter/cli/inspect.go +++ /dev/null @@ -1,212 +0,0 @@ -package cli - -import ( - "context" - "errors" - "flag" - "fmt" - "io" - "path/filepath" - "strings" - - appformat "gitea.maximumdirect.net/eric/scriptorium/internal/format" - "go.maximumdirect.net/configkit" -) - -type promptInspectionConfig struct { - configPath, promptID, schemaDir, outputPath string - configOverlay string - configExplicit bool - outputFormat appformat.OutputFormat - overrides []configkit.Override - diagnostics []configkit.Diagnostic -} - -type profileInspectionConfig struct { - configPath, profileID, schemaDir, outputPath string - configOverlay string - configExplicit bool - outputFormat appformat.OutputFormat - overrides []configkit.Override - diagnostics []configkit.Diagnostic -} - -func inspectCommand(args []string, stdout, stderr io.Writer) int { - if len(args) == 0 { - fmt.Fprintln(stderr, "inspect parse error: inspection mode is required") - return ExitRuntimeError - } - if args[0] == "profile" { - return inspectProfileCommand(args[1:], stdout, stderr) - } - if args[0] == "configuration" { - return inspectFrameworkConfigurationCommand(args[1:], stdout, stderr) - } - if args[0] != "prompt" { - fmt.Fprintln(stderr, "inspect parse error: unknown inspection mode") - return ExitRuntimeError - } - cfg, err := parsePromptInspectionArgs(args[1:]) - if err != nil { - writeApplicationError(stderr, "inspect parse", err) - return ExitRuntimeError - } - settings, err := resolveAppSettingsForPromptInspection(cfg) - if err != nil { - writeApplicationError(stderr, "inspect", err) - return ExitRuntimeError - } - if err := writeApplicationWarnings(stderr, cfg.diagnostics); err != nil { - return ExitRuntimeError - } - engine, err := newEngine(settings) - if err != nil { - writeApplicationError(stderr, "engine", err) - return ExitRuntimeError - } - if err := writeFrameworkDiagnostics(stderr, engine); err != nil { - fmt.Fprintf(stderr, "engine diagnostic write error: %v\n", err) - return ExitRuntimeError - } - inspection, err := engine.InspectPrompt(context.Background(), cfg.promptID) - if err != nil { - fmt.Fprintf(stderr, "inspect error: %v\n", err) - return ExitRuntimeError - } - data, err := appformat.FormatPromptInspection(inspection, cfg.outputFormat) - if err != nil { - fmt.Fprintf(stderr, "inspect error: %v\n", err) - return ExitRuntimeError - } - if err := writeOutput(stdout, cfg.outputPath, data); err != nil { - fmt.Fprintf(stderr, "output write error: %v\n", err) - return ExitRuntimeError - } - return ExitOK -} - -func inspectProfileCommand(args []string, stdout, stderr io.Writer) int { - cfg, err := parseProfileInspectionArgs(args) - if err != nil { - writeApplicationError(stderr, "inspect parse", err) - return ExitRuntimeError - } - settings, err := resolveAppSettingsForProfileInspection(cfg) - if err != nil { - writeApplicationError(stderr, "inspect", err) - return ExitRuntimeError - } - if err := writeApplicationWarnings(stderr, cfg.diagnostics); err != nil { - return ExitRuntimeError - } - engine, err := newEngine(settings) - if err != nil { - writeApplicationError(stderr, "engine", err) - return ExitRuntimeError - } - if err := writeFrameworkDiagnostics(stderr, engine); err != nil { - fmt.Fprintf(stderr, "engine diagnostic write error: %v\n", err) - return ExitRuntimeError - } - inspection, err := engine.InspectProfile(context.Background(), cfg.profileID) - if err != nil { - fmt.Fprintf(stderr, "inspect error: %v\n", err) - return ExitRuntimeError - } - data, err := appformat.FormatProfileInspection(inspection, cfg.outputFormat) - if err != nil { - fmt.Fprintf(stderr, "inspect error: %v\n", err) - return ExitRuntimeError - } - if err := writeOutput(stdout, cfg.outputPath, data); err != nil { - fmt.Fprintf(stderr, "output write error: %v\n", err) - return ExitRuntimeError - } - return ExitOK -} - -func parseProfileInspectionArgs(args []string) (*profileInspectionConfig, error) { - cfg := &profileInspectionConfig{outputFormat: appformat.DefaultOutputFormat} - fs := flag.NewFlagSet("inspect profile", flag.ContinueOnError) - fs.SetOutput(io.Discard) - registerConfigSelectionFlags(fs, &cfg.configPath, &cfg.configOverlay) - var promptkitSources promptKitSourceFlagValue - fs.Var(&promptkitSources, "promptkit-source", "selected PromptKit source path (repeatable)") - fs.StringVar(&cfg.schemaDir, "schema-dir", "", "base directory for validation schemas") - fs.StringVar(&cfg.profileID, "profile", "", "profile ID to inspect") - rawFormat := "" - fs.StringVar(&rawFormat, "format", "", "output format: text or json") - fs.StringVar(&cfg.outputPath, "out", "", "optional output file path") - if err := fs.Parse(args); err != nil { - return nil, err - } - cfg.configExplicit = flagWasSet(fs, "config") - cfg.overrides = applicationOverrides(fs) - if fs.NArg() > 0 { - return nil, fmt.Errorf("unexpected positional args: %v", fs.Args()) - } - if strings.TrimSpace(cfg.profileID) == "" { - return nil, errors.New("--profile is required") - } - format, err := appformat.ParseOutputFormat(rawFormat) - if err != nil { - return nil, err - } - cfg.outputFormat = format - if cfg.outputPath != "" { - cfg.outputPath = filepath.Clean(cfg.outputPath) - } - return cfg, nil -} - -func resolveAppSettingsForProfileInspection(cfg *profileInspectionConfig) (engineSettings, error) { - settings, diagnostics, err := loadApplicationSettings(cfg.configPath, cfg.configOverlay, cfg.configExplicit, cfg.overrides) - if err != nil { - return engineSettings{}, err - } - cfg.diagnostics = diagnostics - return engineSettings{promptkitSources: settings.PromptKitSources, schemaDir: settings.SchemaDir, transientRetries: settings.TransientRetries}, nil -} - -func parsePromptInspectionArgs(args []string) (*promptInspectionConfig, error) { - cfg := &promptInspectionConfig{outputFormat: appformat.DefaultOutputFormat} - fs := flag.NewFlagSet("inspect prompt", flag.ContinueOnError) - fs.SetOutput(io.Discard) - registerConfigSelectionFlags(fs, &cfg.configPath, &cfg.configOverlay) - var promptkitSources promptKitSourceFlagValue - fs.Var(&promptkitSources, "promptkit-source", "selected PromptKit source path (repeatable)") - fs.StringVar(&cfg.schemaDir, "schema-dir", "", "base directory for validation schemas") - fs.StringVar(&cfg.promptID, "prompt", "", "prompt ID to inspect") - rawFormat := "" - fs.StringVar(&rawFormat, "format", "", "output format: text or json") - fs.StringVar(&cfg.outputPath, "out", "", "optional output file path") - if err := fs.Parse(args); err != nil { - return nil, err - } - cfg.configExplicit = flagWasSet(fs, "config") - cfg.overrides = applicationOverrides(fs) - if fs.NArg() > 0 { - return nil, fmt.Errorf("unexpected positional args: %v", fs.Args()) - } - if strings.TrimSpace(cfg.promptID) == "" { - return nil, errors.New("--prompt is required") - } - format, err := appformat.ParseOutputFormat(rawFormat) - if err != nil { - return nil, err - } - cfg.outputFormat = format - if cfg.outputPath != "" { - cfg.outputPath = filepath.Clean(cfg.outputPath) - } - return cfg, nil -} - -func resolveAppSettingsForPromptInspection(cfg *promptInspectionConfig) (engineSettings, error) { - settings, diagnostics, err := loadApplicationSettings(cfg.configPath, cfg.configOverlay, cfg.configExplicit, cfg.overrides) - if err != nil { - return engineSettings{}, err - } - cfg.diagnostics = diagnostics - return engineSettings{promptkitSources: settings.PromptKitSources, schemaDir: settings.SchemaDir, transientRetries: settings.TransientRetries}, nil -} diff --git a/internal/adapter/cli/run.go b/internal/adapter/cli/run.go deleted file mode 100644 index beda55c..0000000 --- a/internal/adapter/cli/run.go +++ /dev/null @@ -1,636 +0,0 @@ -package cli - -import ( - "context" - "errors" - "flag" - "fmt" - "io" - "net/http" - "os" - "path/filepath" - "strings" - "time" - - httpadapter "gitea.maximumdirect.net/eric/scriptorium/internal/adapter/http" - appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config" - "gitea.maximumdirect.net/eric/scriptorium/internal/defaults" - renderformat "gitea.maximumdirect.net/eric/scriptorium/internal/format" - "go.maximumdirect.net/commandkit" - "go.maximumdirect.net/configkit" - "go.maximumdirect.net/promptkit" -) - -const ( - ExitOK = 0 - ExitRuntimeError = 1 - ExitValidationFailed = 2 -) - -type runConfig struct { - configPath string - - promptkitSources []appconfig.SourceSelection - promptID string - profileID string - sessionID string - inputRaw listFlag - varRaw listFlag - outputPath string - executionReportPath string - llmBaseURL string - apiKeyEnv string - model string - temperature float64 - maxTokens int - topP float64 - reasoningEffort string - schemaDir string - timeout time.Duration - runTimeout time.Duration - transientRetries int - configuredTransientRetries *int - diagnostics []configkit.Diagnostic - configOverlay string - - defaultRenderFormat renderformat.PreparedRunOutputFormat - - llmBaseURLSet bool - apiKeyEnvSet bool - modelSet bool - temperatureSet bool - maxTokensSet bool - topPSet bool - reasoningEffortSet bool - timeoutSet bool - transientRetriesSet bool -} - -type renderConfig struct { - runConfig - outputFormat renderformat.PreparedRunOutputFormat -} - -type serveConfig struct { - configPath string - - addr string - promptkitSources []appconfig.SourceSelection - schemaDir string - artifactRoot string - maxRequestBytes int64 - maxArtifactBytes int64 - maxResponseBytes int64 - transientRetries *int - diagnostics []configkit.Diagnostic - configOverlay string -} - -type commonCommandSettings struct { - promptkitSources []appconfig.SourceSelection - schemaDir string - serverAddr string - artifactRoot string - maxRequestBytes int64 - maxArtifactBytes int64 - maxResponseBytes int64 - defaultRenderFormat renderformat.PreparedRunOutputFormat - transientRetries *int - diagnostics []configkit.Diagnostic -} - -type listFlag []string - -func (l *listFlag) String() string { - return strings.Join(*l, ",") -} - -func (l *listFlag) Set(v string) error { - *l = append(*l, v) - return nil -} - -func Run(args []string, stdout, stderr io.Writer) int { - return Execute(context.Background(), commandkit.Identity{Name: "scriptorium", Version: "dev"}, args, stdout, stderr).Status -} - -func runCommand(args []string, stdout, stderr io.Writer) int { - cfg, err := parseRunArgs(args) - if err != nil { - writeApplicationError(stderr, "run parse", err) - return ExitRuntimeError - } - if err := writeApplicationWarnings(stderr, cfg.diagnostics); err != nil { - return ExitRuntimeError - } - - req, err := buildRunRequestFromConfig(cfg) - if err != nil { - fmt.Fprintf(stderr, "run parse error: %v\n", err) - return ExitRuntimeError - } - - engine, err := newEngine(cfg.engineSettings()) - if err != nil { - writeApplicationError(stderr, "engine", err) - return ExitRuntimeError - } - if err := writeFrameworkDiagnostics(stderr, engine); err != nil { - fmt.Fprintf(stderr, "engine diagnostic write error: %v\n", err) - return ExitRuntimeError - } - return executeRun(cfg, req, engine.Run, stdout, stderr) -} - - -func renderCommand(args []string, stdout, stderr io.Writer) int { - cfg, err := parseRenderArgs(args) - if err != nil { - writeApplicationError(stderr, "render parse", err) - return ExitRuntimeError - } - if err := writeApplicationWarnings(stderr, cfg.diagnostics); err != nil { - return ExitRuntimeError - } - - req, err := buildRunRequestFromConfig(&cfg.runConfig) - if err != nil { - fmt.Fprintf(stderr, "render parse error: %v\n", err) - return ExitRuntimeError - } - - engine, err := newEngine(cfg.runConfig.engineSettings()) - if err != nil { - writeApplicationError(stderr, "engine", err) - return ExitRuntimeError - } - if err := writeFrameworkDiagnostics(stderr, engine); err != nil { - fmt.Fprintf(stderr, "engine diagnostic write error: %v\n", err) - return ExitRuntimeError - } - - prepared, prepErr := engine.Prepare(context.Background(), req) - if prepErr != nil { - fmt.Fprintf(stderr, "render error: %v\n", prepErr) - return ExitRuntimeError - } - - out, err := renderformat.FormatPreparedRun(prepared, cfg.outputFormat) - if err != nil { - fmt.Fprintf(stderr, "render format error: %v\n", err) - return ExitRuntimeError - } - - if err := writeOutput(stdout, cfg.outputPath, out); err != nil { - fmt.Fprintf(stderr, "output write error: %v\n", err) - return ExitRuntimeError - } - return ExitOK -} - -func serveCommand(args []string, stderr io.Writer) int { - cfg, err := parseServeArgs(args) - if err != nil { - writeApplicationError(stderr, "serve parse", err) - return ExitRuntimeError - } - if err := writeApplicationWarnings(stderr, cfg.diagnostics); err != nil { - return ExitRuntimeError - } - - artifactReader, err := httpadapter.NewRestrictedArtifactReader(cfg.artifactRoot, cfg.maxArtifactBytes) - if err != nil { - fmt.Fprintf(stderr, "artifact root error: %v\n", err) - return ExitRuntimeError - } - - engine, err := newEngine(cfg.engineSettings(), promptkit.WithArtifactReader(artifactReader)) - if err != nil { - writeApplicationError(stderr, "engine", err) - return ExitRuntimeError - } - if err := writeFrameworkDiagnostics(stderr, engine); err != nil { - fmt.Fprintf(stderr, "engine diagnostic write error: %v\n", err) - return ExitRuntimeError - } - - h := httpadapter.NewHandlerWithOptions(engine, httpadapter.HandlerOptions{ - MaxRequestBytes: cfg.maxRequestBytes, - MaxResponseBytes: cfg.maxResponseBytes, - }) - srv := &http.Server{ - Addr: cfg.addr, - Handler: h, - ReadHeaderTimeout: defaults.HTTPReadHeaderTimeoutDefault, - } - - fmt.Fprintf(stderr, "serving on %s\n", cfg.addr) - if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { - fmt.Fprintf(stderr, "server error: %v\n", err) - return ExitRuntimeError - } - return ExitOK -} - -func parseRunArgs(args []string) (*runConfig, error) { - cfg := &runConfig{} - fs := flag.NewFlagSet("run", flag.ContinueOnError) - fs.SetOutput(io.Discard) - - registerConfigSelectionFlags(fs, &cfg.configPath, &cfg.configOverlay) - registerExecutionRequestFlags(fs, cfg) - fs.DurationVar(&cfg.runTimeout, "run-timeout", 0, "whole-run execution deadline") - fs.StringVar(&cfg.executionReportPath, "execution-report", "", "JSON execution report file path") - - if err := fs.Parse(args); err != nil { - return nil, err - } - if err := finalizeExecutionRequestConfig(fs, cfg); err != nil { - return nil, err - } - if cfg.runTimeout < 0 { - return nil, errors.New("--run-timeout must be greater than or equal to 0") - } - if flagWasSet(fs, "execution-report") { - if cfg.executionReportPath == "" { - return nil, errors.New("--execution-report requires a nonempty path") - } - cfg.executionReportPath = filepath.Clean(cfg.executionReportPath) - if cfg.outputPath != "" { - collision, err := outputPathsCollide(cfg.outputPath, cfg.executionReportPath) - if err != nil { - return nil, err - } - if collision { - return nil, errors.New("--out and --execution-report select the same file") - } - } - } - return cfg, nil -} - -func outputPathsCollide(first, second string) (bool, error) { - firstAbs, err := filepath.Abs(first) - if err != nil { - return false, err - } - secondAbs, err := filepath.Abs(second) - if err != nil { - return false, err - } - if firstAbs == secondAbs { - return true, nil - } - firstInfo, firstErr := os.Stat(firstAbs) - secondInfo, secondErr := os.Stat(secondAbs) - return firstErr == nil && secondErr == nil && os.SameFile(firstInfo, secondInfo), nil -} - -func parseRenderArgs(args []string) (*renderConfig, error) { - cfg := &renderConfig{ - outputFormat: renderformat.DefaultPreparedRunOutputFormat, - } - fs := flag.NewFlagSet("render", flag.ContinueOnError) - fs.SetOutput(io.Discard) - - registerConfigSelectionFlags(fs, &cfg.configPath, &cfg.configOverlay) - registerExecutionRequestFlags(fs, &cfg.runConfig) - - var rawFormat string - fs.StringVar(&rawFormat, "format", "", "render output format (text|json)") - - if err := fs.Parse(args); err != nil { - return nil, err - } - if err := finalizeExecutionRequestConfig(fs, &cfg.runConfig); err != nil { - return nil, err - } - - if flagWasSet(fs, "format") { - format, err := renderformat.ParsePreparedRunOutputFormat(rawFormat) - if err != nil { - return nil, err - } - cfg.outputFormat = format - } else { - cfg.outputFormat = cfg.defaultRenderFormat - } - return cfg, nil -} - -func parseServeArgs(args []string) (*serveConfig, error) { - cfg := &serveConfig{} - fs := flag.NewFlagSet("serve", flag.ContinueOnError) - fs.SetOutput(io.Discard) - - registerConfigSelectionFlags(fs, &cfg.configPath, &cfg.configOverlay) - fs.StringVar(&cfg.addr, "addr", "", "HTTP listen address") - fs.StringVar(&cfg.artifactRoot, "artifact-root", "", "base directory for HTTP file input artifacts") - fs.Int64Var(&cfg.maxRequestBytes, "max-request-bytes", 0, "maximum HTTP request body bytes; 0 disables the limit") - fs.Int64Var(&cfg.maxArtifactBytes, "max-artifact-bytes", 0, "maximum HTTP file artifact bytes; 0 disables the limit") - fs.Int64Var(&cfg.maxResponseBytes, "max-response-bytes", 0, "maximum HTTP response body bytes; 0 disables the limit") - - if err := fs.Parse(args); err != nil { - return nil, err - } - if fs.NArg() > 0 { - return nil, fmt.Errorf("unexpected positional args: %v", fs.Args()) - } - - settings, err := resolveCommonSettings(fs, cfg.configPath, cfg.configOverlay) - if err != nil { - return nil, err - } - - cfg.promptkitSources = settings.promptkitSources - cfg.schemaDir = settings.schemaDir - cfg.addr = settings.serverAddr - cfg.artifactRoot = settings.artifactRoot - cfg.maxRequestBytes = settings.maxRequestBytes - cfg.maxArtifactBytes = settings.maxArtifactBytes - cfg.maxResponseBytes = settings.maxResponseBytes - cfg.transientRetries = settings.transientRetries - cfg.diagnostics = settings.diagnostics - - return cfg, nil -} - -func registerExecutionRequestFlags(fs *flag.FlagSet, cfg *runConfig) { - fs.StringVar(&cfg.promptID, "prompt", "", "prompt ID to run") - fs.StringVar(&cfg.profileID, "profile", "", "optional execution profile ID; if omitted, prompt default_profile is used") - fs.StringVar(&cfg.sessionID, "session-id", "", "optional session ID") - fs.Var(&cfg.inputRaw, "input", "input mapping(s): name=path (repeatable, comma-separated)") - fs.Var(&cfg.varRaw, "var", "variable mapping(s): name=value (repeatable, comma-separated)") - fs.StringVar(&cfg.outputPath, "out", "", "optional output file path") - fs.StringVar(&cfg.llmBaseURL, "llm-base-url", "", "OpenAI-compatible base URL including /v1") - fs.StringVar(&cfg.apiKeyEnv, "api-key-env", "", "environment variable name containing API key") - fs.StringVar(&cfg.model, "model", "", "model name") - fs.Float64Var(&cfg.temperature, "temperature", 0, "optional temperature override") - fs.IntVar(&cfg.maxTokens, "max-tokens", 0, "optional max tokens override") - fs.Float64Var(&cfg.topP, "top-p", 0, "optional top_p override") - fs.StringVar(&cfg.reasoningEffort, "reasoning-effort", "", "optional reasoning effort override") - fs.DurationVar(&cfg.timeout, "timeout", 0, "LLM request timeout") - fs.IntVar(&cfg.transientRetries, "transient-retries", 0, "additional transient generation attempts") - fs.StringVar(&cfg.promptID, "prompt-id", "", "deprecated alias for --prompt") - fs.StringVar(&cfg.profileID, "profile-id", "", "deprecated alias for --profile") -} - -func finalizeExecutionRequestConfig(fs *flag.FlagSet, cfg *runConfig) error { - if fs.NArg() > 0 { - return fmt.Errorf("unexpected positional args: %v", fs.Args()) - } - - settings, err := resolveCommonSettings(fs, cfg.configPath, cfg.configOverlay) - if err != nil { - return err - } - - cfg.promptkitSources = settings.promptkitSources - cfg.schemaDir = settings.schemaDir - cfg.defaultRenderFormat = settings.defaultRenderFormat - cfg.configuredTransientRetries = settings.transientRetries - cfg.diagnostics = settings.diagnostics - - if strings.TrimSpace(cfg.promptID) == "" { - return errors.New("--prompt is required") - } - if cfg.outputPath != "" { - cfg.outputPath = filepath.Clean(cfg.outputPath) - } - cfg.llmBaseURLSet = flagWasSet(fs, "llm-base-url") - cfg.apiKeyEnvSet = flagWasSet(fs, "api-key-env") - cfg.modelSet = flagWasSet(fs, "model") - cfg.temperatureSet = flagWasSet(fs, "temperature") - cfg.maxTokensSet = flagWasSet(fs, "max-tokens") - cfg.topPSet = flagWasSet(fs, "top-p") - cfg.reasoningEffortSet = flagWasSet(fs, "reasoning-effort") - cfg.timeoutSet = flagWasSet(fs, "timeout") - cfg.transientRetriesSet = flagWasSet(fs, "transient-retries") - if cfg.transientRetriesSet && cfg.transientRetries < 0 { - return errors.New("--transient-retries must be greater than or equal to 0") - } - return nil -} - -func registerConfigSelectionFlags(fs *flag.FlagSet, target, overlay *string) { - fs.StringVar( - target, - "config", - "", - fmt.Sprintf( - "application config path (default search: %s, then %s)", - appconfig.DefaultConfigPathLocal, - appconfig.DefaultConfigPath, - ), - ) - fs.StringVar(overlay, "config-overlay", "", "selected application configuration overlay") -} - -func resolveCommonSettings(fs *flag.FlagSet, configPath, overlay string) (commonCommandSettings, error) { - settings, diagnostics, err := loadApplicationSettings(configPath, overlay, flagWasSet(fs, "config"), applicationOverrides(fs)) - if err != nil { - return commonCommandSettings{}, err - } - return commonCommandSettings{ - promptkitSources: settings.PromptKitSources, - schemaDir: settings.SchemaDir, - serverAddr: settings.ServerAddr, - artifactRoot: settings.ArtifactRoot, - maxRequestBytes: settings.MaxRequestBytes, - maxArtifactBytes: settings.MaxArtifactBytes, - maxResponseBytes: settings.MaxResponseBytes, - defaultRenderFormat: renderformat.DefaultPreparedRunOutputFormat, - transientRetries: settings.TransientRetries, - diagnostics: diagnostics, - }, nil -} - -type engineSettings struct { - promptkitSources []appconfig.SourceSelection - schemaDir string - transientRetries *int -} - -func (c runConfig) engineSettings() engineSettings { - return engineSettings{promptkitSources: c.promptkitSources, schemaDir: c.schemaDir, transientRetries: c.configuredTransientRetries} -} - -func (c serveConfig) engineSettings() engineSettings { - return engineSettings{promptkitSources: c.promptkitSources, schemaDir: c.schemaDir, transientRetries: c.transientRetries} -} - - -func buildRunRequestFromConfig(cfg *runConfig) (promptkit.RunRequest, error) { - var inputMappings map[string]string - var err error - if len(cfg.inputRaw) > 0 { - inputMappings, err = parseMappings(cfg.inputRaw, false) - if err != nil { - return promptkit.RunRequest{}, fmt.Errorf("input parse error: %w", err) - } - } - - varMappings := map[string]string{} - if len(cfg.varRaw) > 0 { - varMappings, err = parseMappings(cfg.varRaw, false) - if err != nil { - return promptkit.RunRequest{}, fmt.Errorf("var parse error: %w", err) - } - } - - var inputs map[string]promptkit.ArtifactRef - if len(inputMappings) > 0 { - inputs = make(map[string]promptkit.ArtifactRef, len(inputMappings)) - for name, path := range inputMappings { - inputs[name] = promptkit.File(path) - } - } - - var modelOverride *promptkit.ExecutionTargetOverride - if cfg.llmBaseURLSet || cfg.modelSet || cfg.temperatureSet || cfg.maxTokensSet || cfg.topPSet || cfg.reasoningEffortSet || cfg.apiKeyEnvSet || cfg.timeoutSet { - modelOverride = &promptkit.ExecutionTargetOverride{ - Endpoint: cfg.llmBaseURL, - Model: cfg.model, - APIKeyEnv: cfg.apiKeyEnv, - } - if cfg.temperatureSet { - modelOverride.Temperature = &cfg.temperature - } - if cfg.maxTokensSet { - modelOverride.MaxTokens = &cfg.maxTokens - } - if cfg.topPSet { - modelOverride.TopP = &cfg.topP - } - if cfg.reasoningEffortSet { - modelOverride.ReasoningEffort = &cfg.reasoningEffort - } - if cfg.timeoutSet { - timeoutSeconds := int(cfg.timeout.Seconds()) - modelOverride.TimeoutSeconds = &timeoutSeconds - } - } - - var transientRetries *int - if cfg.transientRetriesSet { - transientRetries = &cfg.transientRetries - } - return promptkit.RunRequest{ - PromptID: cfg.promptID, - ProfileID: cfg.profileID, - SessionID: cfg.sessionID, - Inputs: inputs, - Vars: varMappings, - Execution: modelOverride, - TransientRetries: transientRetries, - }, nil -} - -func parseMappings(raw []string, allowEmptyValue bool) (map[string]string, error) { - out := make(map[string]string) - for _, entry := range raw { - for _, piece := range strings.Split(entry, ",") { - piece = strings.TrimSpace(piece) - if piece == "" { - continue - } - key, value, err := parseMapping(piece) - if err != nil { - return nil, err - } - if !allowEmptyValue && strings.TrimSpace(value) == "" { - return nil, fmt.Errorf("mapping %q has empty value", piece) - } - out[key] = value - } - } - if len(out) == 0 { - return nil, errors.New("no valid mappings provided") - } - return out, nil -} - -func parseMapping(value string) (string, string, error) { - idx := strings.IndexRune(value, '=') - if idx <= 0 { - return "", "", fmt.Errorf("invalid mapping %q, expected name=value", value) - } - key := strings.TrimSpace(value[:idx]) - val := strings.TrimSpace(value[idx+1:]) - if key == "" { - return "", "", fmt.Errorf("invalid mapping %q, empty name", value) - } - return key, val, nil -} - -func flagWasSet(fs *flag.FlagSet, name string) bool { - set := false - fs.Visit(func(f *flag.Flag) { - if f.Name == name { - set = true - } - }) - return set -} - -func writeOutput(stdout io.Writer, outputPath string, body []byte) error { - if outputPath == "" { - _, err := stdout.Write(body) - return err - } - return os.WriteFile(outputPath, body, 0644) -} - -func determineExitCode(runErr error, result *promptkit.RunResult) int { - if runErr != nil { - return ExitRuntimeError - } - if result != nil && result.Validation.Status == promptkit.ValidationFailed { - return ExitValidationFailed - } - return ExitOK -} - -func printSummary(stderr io.Writer, res *promptkit.RunResult) { - if res == nil { - return - } - fmt.Fprintf(stderr, "prompt=%s selected_profile=%s model=%s validation=%s mode=%s validation_errors=%d prompt_hash=%s inputs=%d usage=%d/%d/%d", - res.PromptID, - res.SelectedProfileID, - res.ModelName, - res.Validation.Status, - res.Validation.Mode, - len(res.Validation.Errors), - res.RenderedPromptHash, - len(res.InputHashes), - res.Usage.PromptTokens, - res.Usage.CompletionTokens, - res.Usage.TotalTokens, - ) - if res.Usage.CachedTokens != 0 || res.Usage.CacheWriteTokens != 0 { - fmt.Fprintf(stderr, " cached_tokens=%d cache_write_tokens=%d", res.Usage.CachedTokens, res.Usage.CacheWriteTokens) - } - if res.SelectedBackendID != "" { - fmt.Fprintf(stderr, " backend=%s", res.SelectedBackendID) - } - fmt.Fprintln(stderr) -} - -func runErrorMessage(err error) string { - if errors.Is(err, promptkit.ErrCapacityExceeded) { - return "run error: model backend capacity is exhausted" - } - return fmt.Sprintf("run error: %v", err) -} - -func printUsage(w io.Writer) { - fmt.Fprintln(w, "usage: scriptorium ...") - fmt.Fprintln(w, " run: scriptorium run [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--schema-dir DIR] --prompt ID [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--out path] [--execution-report path] [--timeout 10m] [--run-timeout 10m] [--transient-retries N]") - fmt.Fprintln(w, " render: scriptorium render [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--schema-dir DIR] --prompt ID [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--format text|json] [--out path] [--timeout 10m] [--transient-retries N]") - fmt.Fprintf(w, " serve: scriptorium serve [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--addr %s] [--schema-dir DIR] [--artifact-root DIR] [--max-request-bytes N] [--max-artifact-bytes N] [--max-response-bytes N]\n", defaults.HTTPAddrDefault) - fmt.Fprintln(w, " inspect prompt: scriptorium inspect prompt --prompt ID [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--schema-dir DIR] [--format text|json] [--out path]") - fmt.Fprintln(w, " inspect profile: scriptorium inspect profile --profile ID [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--schema-dir DIR] [--format text|json] [--out path]") - fmt.Fprintln(w, " inspect configuration: scriptorium inspect configuration [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--schema-dir DIR] [--path JSON_POINTER (required for explain)]") - fmt.Fprintln(w, " config: scriptorium config [--config PATH] [--config-overlay NAME] [application overrides] [--path /pointer for explain]") -} diff --git a/internal/adapter/cli/run_test.go b/internal/adapter/cli/run_test.go deleted file mode 100644 index e46dc77..0000000 --- a/internal/adapter/cli/run_test.go +++ /dev/null @@ -1,2807 +0,0 @@ -package cli - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "flag" - "fmt" - "io" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "reflect" - "strings" - "sync/atomic" - "testing" - "time" - - appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config" - "gitea.maximumdirect.net/eric/scriptorium/internal/defaults" - renderformat "gitea.maximumdirect.net/eric/scriptorium/internal/format" - "go.maximumdirect.net/configkit" - "go.maximumdirect.net/promptkit" -) - -func TestParseMappingsSingleAndRepeated(t *testing.T) { - got, err := parseMappings([]string{"transcript=./t.md", "glossary=./g.yml"}, false) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if got["transcript"] != "./t.md" || got["glossary"] != "./g.yml" { - t.Fatalf("unexpected mappings: %#v", got) - } -} - -func TestParseMappingsCommaSeparated(t *testing.T) { - got, err := parseMappings([]string{"transcript=./t.md,glossary=./g.yml"}, false) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if got["transcript"] != "./t.md" || got["glossary"] != "./g.yml" { - t.Fatalf("unexpected mappings: %#v", got) - } -} - -func TestParseMappingsVarWithEqualsInValue(t *testing.T) { - got, err := parseMappings([]string{"session_note=a=b=c"}, false) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if got["session_note"] != "a=b=c" { - t.Fatalf("unexpected variable value: %#v", got) - } -} - -func TestParseMappingsMalformed(t *testing.T) { - tests := []string{"", "novalue", "=emptyname", "name="} - for _, tc := range tests { - _, err := parseMappings([]string{tc}, false) - if err == nil { - t.Fatalf("expected error for %q", tc) - } - } -} - -func TestParseRunArgsRequiredFlags(t *testing.T) { - configPath := writeAppConfigFile(t, "{}\n") - - cfg, err := parseRunArgs([]string{"--config", configPath, "--prompt", "p", "--input", "a=b"}) - if err != nil { - t.Fatalf("expected omitted source list to be accepted, got %v", err) - } - if len(cfg.promptkitSources) != 0 { - t.Fatalf("expected no caller sources, got %#v", cfg.promptkitSources) - } - - _, err = parseRunArgs([]string{"--config", configPath}) - if err == nil { - t.Fatal("expected missing --prompt error") - } - - _, err = parseRunArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--prompt", "p"}) - if err == nil { - t.Fatal("expected retired --prompt-dir flag to be rejected") - } - _, err = parseRunArgs([]string{"--config", configPath, "--profile-dir", "./profiles", "--prompt", "p"}) - if err == nil { - t.Fatal("expected retired --profile-dir flag to be rejected") - } - - cfg, err = parseRunArgs([]string{"--config", configPath, "--prompt", "p"}) - if err != nil { - t.Fatalf("expected omitted --input and source list to be accepted, got %v", err) - } - if len(cfg.inputRaw) != 0 { - t.Fatalf("expected no input mappings, got %#v", cfg.inputRaw) - } -} - -func TestParseRunArgsFlagMapping(t *testing.T) { - lib := newCLITestLibrary(t) - configPath := writeAppConfigFile(t, "{}\n") - sourceFile := filepath.Join(lib.rootDir, "promptkit,extra.yml") - if err := os.WriteFile(sourceFile, []byte("not parsed by application loading\n"), 0o600); err != nil { - t.Fatal(err) - } - cfg, err := parseRunArgs([]string{ - "--config", configPath, - "--promptkit-source", lib.promptDir, - "--promptkit-source", sourceFile, - "--prompt", "support/prompt.a", - "--profile", "support/profile.a", - "--session-id", "session-1", - "--input", "a=b", - "--llm-base-url", "http://x/v1", - "--model", "m", - "--temperature", "0.7", - "--max-tokens", "111", - "--top-p", "0.8", - "--reasoning-effort", "medium", - "--timeout", "30s", - "--api-key-env", "SCRIPTORIUM_API_KEY", - }) - if err != nil { - t.Fatalf("expected valid args, got %v", err) - } - if !reflect.DeepEqual(cfg.promptkitSources, []appconfig.SourceSelection{{Directory: lib.promptDir}, {File: sourceFile}}) { - t.Fatalf("unexpected source selections: %#v", cfg.promptkitSources) - } - if cfg.promptID != "support/prompt.a" || cfg.profileID != "support/profile.a" { - t.Fatalf("unexpected prompt/profile ids: %q %q", cfg.promptID, cfg.profileID) - } - if cfg.sessionID != "session-1" || cfg.reasoningEffort != "medium" || !cfg.reasoningEffortSet { - t.Fatalf("unexpected session or reasoning configuration: %+v", cfg) - } - if !cfg.llmBaseURLSet || !cfg.modelSet || !cfg.temperatureSet || !cfg.maxTokensSet || !cfg.topPSet || !cfg.timeoutSet || !cfg.apiKeyEnvSet { - t.Fatalf("expected override flags set, got %+v", cfg) - } -} - -func TestPromptKitSourceFlagsReplaceConfiguredSequenceAsOneOverride(t *testing.T) { - lib := newCLITestLibrary(t) - sourceFile := filepath.Join(lib.rootDir, "fixture,part.yml") - if err := os.WriteFile(sourceFile, []byte("contents are not read by application parsing\n"), 0o600); err != nil { - t.Fatal(err) - } - configPath := writeAppConfigFile(t, "promptkit_sources:\n - directory: ./configured-but-not-loaded\n") - - omitted, err := parseRunArgs([]string{"--config", configPath, "--prompt", "p"}) - if err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(omitted.promptkitSources, []appconfig.SourceSelection{{Directory: "configured-but-not-loaded"}}) { - t.Fatalf("omitting source flags changed configured selections: %#v", omitted.promptkitSources) - } - - args := []string{"--config", configPath, "--promptkit-source", lib.promptDir, "--promptkit-source", sourceFile, "--prompt", "p"} - selected, err := parseRunArgs(args) - if err != nil { - t.Fatal(err) - } - want := []appconfig.SourceSelection{{Directory: lib.promptDir}, {File: sourceFile}} - if !reflect.DeepEqual(selected.promptkitSources, want) { - t.Fatalf("explicit source flags did not replace config in argument order: got %#v, want %#v", selected.promptkitSources, want) - } - - flags := flag.NewFlagSet("source override", flag.ContinueOnError) - flags.SetOutput(io.Discard) - var sources promptKitSourceFlagValue - flags.Var(&sources, "promptkit-source", "") - if err := flags.Parse([]string{"--promptkit-source", lib.promptDir, "--promptkit-source", sourceFile}); err != nil { - t.Fatal(err) - } - overrides := applicationOverrides(flags) - if len(overrides) != 1 || overrides[0].Path.String() != "/promptkit_sources" || overrides[0].Flag != "--promptkit-source" { - t.Fatalf("expected one whole-list source override, got %#v", overrides) - } - if !reflect.DeepEqual(overrides[0].Value, []any{ - map[string]any{"directory": lib.promptDir}, - map[string]any{"file": sourceFile}, - }) { - t.Fatalf("source override is not a plain ordered sequence of maps: %#v", overrides[0].Value) - } - - _, err = parseRunArgs([]string{"--promptkit-source", filepath.Join(lib.rootDir, "missing.yml"), "--prompt", "p"}) - if err == nil { - t.Fatal("expected unavailable explicit source path to be rejected") - } -} - -func TestParseRunArgsAllowsOmittedModelAndBaseURL(t *testing.T) { - configPath := writeAppConfigFile(t, "{}\n") - cfg, err := parseRunArgs([]string{ - "--config", configPath, - "--prompt", "p", - "--input", "a=b", - }) - if err != nil { - t.Fatalf("expected valid args without model/base url, got %v", err) - } - if cfg.llmBaseURL != "" || cfg.model != "" { - t.Fatalf("expected empty model/baseurl, got model=%q base=%q", cfg.model, cfg.llmBaseURL) - } -} - -func TestParseRunArgsRejectsRawLLMAPIKeyFlag(t *testing.T) { - _, err := parseRunArgs([]string{ - "--prompt", "p", - "--input", "a=b", - "--llm-api-key", "secret", - }) - if err == nil { - t.Fatal("expected unknown flag error for --llm-api-key") - } -} - -func TestParseServeArgsRequiredFlags(t *testing.T) { - configPath := writeAppConfigFile(t, "{}\n") - - cfg, err := parseServeArgs([]string{"--config", configPath}) - if err != nil { - t.Fatalf("expected source list to be optional, got %v", err) - } - if len(cfg.promptkitSources) != 0 { - t.Fatalf("expected no caller sources, got %#v", cfg.promptkitSources) - } - if cfg.addr != defaults.HTTPAddrDefault { - t.Fatalf("expected default addr %s, got %q", defaults.HTTPAddrDefault, cfg.addr) - } - if cfg.schemaDir != defaults.SchemaDirDefault { - t.Fatalf("expected default schema dir %q, got %q", defaults.SchemaDirDefault, cfg.schemaDir) - } -} - -func TestParseServeArgsRejectsRuntimeOverrideFlags(t *testing.T) { - base := []string{} - tests := [][]string{ - {"--llm-base-url", "http://localhost:8000/v1"}, - {"--model", "gpt-4o-mini"}, - {"--timeout", "30s"}, - } - - for _, tc := range tests { - _, err := parseServeArgs(append(base, tc...)) - if err == nil { - t.Fatalf("expected unknown flag error for %q", tc[0]) - } - } -} - -func TestUsageAdvertisesFrameworkConfigurationInspection(t *testing.T) { - code, stdout, usage := runCLICommand(t, Run, nil) - if code != ExitRuntimeError || stdout != "" { - t.Fatalf("usage exit/streams changed: code=%d stdout=%q", code, stdout) - } - - var frameworkUsage string - for _, line := range strings.Split(usage, "\n") { - if strings.Contains(line, "inspect configuration") { - frameworkUsage = line - break - } - } - if frameworkUsage == "" { - t.Fatalf("usage omits framework configuration inspection: %s", usage) - } - for _, want := range []string{ - "validate", "show", "sources", "explain", - "--config", "--config-overlay", "--promptkit-source", "--schema-dir", "--path", - } { - if !strings.Contains(frameworkUsage, want) { - t.Errorf("framework configuration usage omits %q: %s", want, frameworkUsage) - } - } -} - -func TestUsageIncludesExecutionAndServeFlags(t *testing.T) { - var stderr bytes.Buffer - code := Run(nil, io.Discard, &stderr) - if code != ExitRuntimeError { - t.Fatalf("expected usage path to return runtime error, got %d", code) - } - - usage := stderr.String() - for _, want := range []string{ - "--session-id ID", - "--reasoning-effort VALUE", - "--artifact-root", - "--max-request-bytes", - "--max-artifact-bytes", - "--max-response-bytes", - "inspect prompt", - "inspect profile", - "--promptkit-source PATH", - } { - if !strings.Contains(usage, want) { - t.Fatalf("expected usage to include %q, got %q", want, usage) - } - } -} - -func TestParseRunArgsTimeout(t *testing.T) { - configPath := writeAppConfigFile(t, "{}\n") - cfg, err := parseRunArgs([]string{ - "--config", configPath, - "--prompt", "p", - "--input", "a=b", - }) - if err != nil { - t.Fatalf("expected valid run args, got %v", err) - } - if cfg.timeout != 0 { - t.Fatalf("expected omitted timeout to remain unset, got %s", cfg.timeout) - } - - cfg, err = parseRunArgs([]string{ - "--config", configPath, - "--prompt", "p", - "--input", "a=b", - "--timeout", "2m30s", - }) - if err != nil { - t.Fatalf("expected valid run args with timeout override, got %v", err) - } - if cfg.timeout != 2*time.Minute+30*time.Second { - t.Fatalf("expected timeout override 2m30s, got %s", cfg.timeout) - } -} - -func TestParseRenderArgsDefaultsAndFormat(t *testing.T) { - configPath := writeAppConfigFile(t, "{}\n") - cfg, err := parseRenderArgs([]string{ - "--config", configPath, - "--prompt", "p", - "--input", "a=b", - }) - if err != nil { - t.Fatalf("expected valid render args, got %v", err) - } - if cfg.outputFormat != renderformat.DefaultPreparedRunOutputFormat { - t.Fatalf("expected default render format %q, got %q", renderformat.DefaultPreparedRunOutputFormat, cfg.outputFormat) - } -} - -func TestParseRenderArgsExplicitFormatsAndUnknown(t *testing.T) { - configPath := writeAppConfigFile(t, "{}\n") - cfg, err := parseRenderArgs([]string{ - "--config", configPath, - "--prompt", "p", - "--input", "a=b", - "--format", "text", - }) - if err != nil { - t.Fatalf("expected valid text format, got %v", err) - } - if cfg.outputFormat != renderformat.PreparedRunFormatText { - t.Fatalf("expected text format, got %q", cfg.outputFormat) - } - - cfg, err = parseRenderArgs([]string{ - "--config", configPath, - "--prompt", "p", - "--input", "a=b", - "--format", "json", - }) - if err != nil { - t.Fatalf("expected valid json format, got %v", err) - } - if cfg.outputFormat != renderformat.PreparedRunFormatJSON { - t.Fatalf("expected json format, got %q", cfg.outputFormat) - } - - _, err = parseRenderArgs([]string{ - "--config", configPath, - "--prompt", "p", - "--input", "a=b", - "--format", "yaml", - }) - if err == nil { - t.Fatal("expected unknown format error") - } - if !strings.Contains(err.Error(), "unknown prepared run format") { - t.Fatalf("expected clear unknown format error, got %v", err) - } -} - -func TestParseRunArgsWithExplicitConfigLoadsSourceSelections(t *testing.T) { - configPath := writeAppConfigFile(t, ` -promptkit_sources: - - directory: ./from-config/prompts - - directory: ./from-config/profiles -schema_dir: ./from-config/schemas -`) - - cfg, err := parseRunArgs([]string{ - "--config", configPath, - "--prompt", "p", - "--input", "a=b", - }) - if err != nil { - t.Fatalf("expected valid args, got %v", err) - } - - if !reflect.DeepEqual(cfg.promptkitSources, []appconfig.SourceSelection{{Directory: "from-config/prompts"}, {Directory: "from-config/profiles"}}) { - t.Fatalf("unexpected source selections from config: %#v", cfg.promptkitSources) - } - if cfg.schemaDir != filepath.Clean("./from-config/schemas") { - t.Fatalf("expected schema dir from config, got %q", cfg.schemaDir) - } -} - -func TestParseRunArgsMissingExplicitConfigReturnsError(t *testing.T) { - _, err := parseRunArgs([]string{ - "--config", filepath.Join(t.TempDir(), "missing.yml"), - "--prompt", "p", - "--input", "a=b", - }) - if err == nil { - t.Fatal("expected explicit config missing error") - } -} - -func TestParseRunArgsInvalidExplicitConfigReturnsError(t *testing.T) { - configPath := writeAppConfigFile(t, "api_key: secret\n") - - _, err := parseRunArgs([]string{ - "--config", configPath, - "--prompt", "p", - "--input", "a=b", - }) - if err == nil { - t.Fatal("expected invalid explicit config error") - } - var loadError *configkit.LoadError - if !errors.As(err, &loadError) { - t.Fatalf("expected structured application config error, got %v", err) - } -} - -func TestParseRunArgsCLIOverridesConfiguredSourceList(t *testing.T) { - lib := newCLITestLibrary(t) - file := filepath.Join(lib.rootDir, "explicit.yml") - if err := os.WriteFile(file, []byte("not read by application loading\n"), 0o600); err != nil { - t.Fatal(err) - } - configPath := writeAppConfigFile(t, ` -promptkit_sources: - - directory: ./from-config/prompts - - directory: ./from-config/profiles -schema_dir: ./from-config/schemas -`) - - cfg, err := parseRunArgs([]string{ - "--config", configPath, - "--promptkit-source", lib.promptDir, - "--promptkit-source", file, - "--schema-dir", "./from-cli/schemas", - "--prompt", "p", - "--input", "a=b", - }) - if err != nil { - t.Fatalf("expected valid args, got %v", err) - } - - if !reflect.DeepEqual(cfg.promptkitSources, []appconfig.SourceSelection{{Directory: lib.promptDir}, {File: file}}) { - t.Fatalf("expected explicit CLI list to replace config, got %#v", cfg.promptkitSources) - } - if cfg.schemaDir != filepath.Clean("./from-cli/schemas") { - t.Fatalf("expected CLI schema dir override, got %q", cfg.schemaDir) - } -} - -func TestApplicationFlagsPreserveExplicitEmptyValues(t *testing.T) { - configPath := writeAppConfigFile(t, `promptkit_sources: - - directory: ./configured/prompts - - directory: ./configured/profiles -schema_dir: ./configured/schemas -server: - addr: ":9090" - artifact_root: ./configured/artifacts -`) - if _, err := parseRunArgs([]string{"--config", configPath, "--prompt", "p", "--promptkit-source="}); err == nil { - t.Fatal("blank explicit source path should be rejected") - } - render, err := parseRenderArgs([]string{"--config", configPath, "--prompt", "p", "--schema-dir="}) - if err != nil || len(render.promptkitSources) != 2 || render.schemaDir != "" { - t.Fatalf("render did not preserve configured sources and explicit empty schema dir: cfg=%+v err=%v", render, err) - } - serve, err := parseServeArgs([]string{"--config", configPath, "--addr=", "--artifact-root="}) - if err != nil || serve.addr != "" || serve.artifactRoot != "" { - t.Fatalf("serve did not preserve explicit empty settings: cfg=%+v err=%v", serve, err) - } - - prompt, err := parsePromptInspectionArgs([]string{"--config", configPath, "--prompt", "p"}) - if err != nil { - t.Fatal(err) - } - if settings, err := resolveAppSettingsForPromptInspection(prompt); err != nil || len(settings.promptkitSources) != 2 { - t.Fatalf("prompt inspection should accept configured source selectors: settings=%+v err=%v", settings, err) - } - profile, err := parseProfileInspectionArgs([]string{"--config", configPath, "--profile", "p"}) - if err != nil { - t.Fatal(err) - } - settings, err := resolveAppSettingsForProfileInspection(profile) - if err != nil || len(settings.promptkitSources) != 2 { - t.Fatalf("profile inspection should retain configured source selectors: settings=%+v err=%v", settings, err) - } -} - -func TestApplicationLoadUsesNativeYAMLAndSafeDiagnostics(t *testing.T) { - for _, body := range []string{ - "", - "server: null\n", - "server:\n addr: null\n", - "server:\n unknown: 1\n", - "promptkit_sources: &alias []\n", - "promptkit_sources: []\n---\nschema_dir: ./schemas\n", - } { - path := writeAppConfigFile(t, body) - _, err := parseRunArgs([]string{"--config", path, "--prompt", "p"}) - var loadError *configkit.LoadError - if !errors.As(err, &loadError) { - t.Fatalf("native YAML failure %q lost structured diagnostics: %v", body, err) - } - } - - const rejected = "sensitive-synthetic-marker" - path := writeAppConfigFile(t, "defaults:\n render_format: "+rejected+"\n") - code, stdout, stderr := runCLICommand(t, renderCommand, []string{"--config", path, "--prompt", "p"}) - if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "validate.error") || strings.Contains(stderr, rejected) { - t.Fatalf("unsafe or misclassified load diagnostic: code=%d stdout=%q stderr=%q", code, stdout, stderr) - } -} - -type failingDiagnosticWriter struct{} - -func (failingDiagnosticWriter) Write([]byte) (int, error) { - return 0, errors.New("diagnostic sink unavailable") -} - -func TestApplicationDiagnosticWriterFailure(t *testing.T) { - diagnostics := []configkit.Diagnostic{{Code: "test.error", Message: "safe diagnostic"}} - if err := writeApplicationWarnings(failingDiagnosticWriter{}, diagnostics); err == nil { - t.Fatal("expected diagnostic writer failure") - } - path := writeAppConfigFile(t, "unknown_field: secret-value\n") - if code := runCommand([]string{"--config", path, "--prompt", "sample"}, io.Discard, failingDiagnosticWriter{}); code != ExitRuntimeError { - t.Fatalf("diagnostic writer failure must retain runtime error exit status, got %d", code) - } -} - -func TestConfigurationErrorsUseTheirNativeDiagnosticWriters(t *testing.T) { - application := writeAppConfigFile(t, "unknown_field: value\n") - framework := writeFrameworkFile(t, t.TempDir(), "invalid.yml", "not: [valid\n") - frameworkConfig := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n", framework)) - for _, path := range []string{application, frameworkConfig} { - code, stdout, stderr := runCLICommand(t, renderCommand, []string{"--config", path, "--prompt", "sample"}) - if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "error [") { - t.Fatalf("expected native diagnostics: code=%d stdout=%q stderr=%q", code, stdout, stderr) - } - if code := renderCommand([]string{"--config", path, "--prompt", "sample"}, io.Discard, failingDiagnosticWriter{}); code != ExitRuntimeError { - t.Fatalf("diagnostic write failure returned %d", code) - } - } - _, err := newEngine(engineSettings{promptkitSources: []appconfig.SourceSelection{{File: framework}}}) - var loadError *configkit.LoadError - if !errors.As(err, &loadError) { - t.Fatalf("framework load lost diagnostic identity: %v", err) - } -} - -func TestParseRenderArgsWithExplicitConfigLoadsDirectoriesAndFormat(t *testing.T) { - configPath := writeAppConfigFile(t, ` -promptkit_sources: - - directory: ./from-config/prompts - - directory: ./from-config/profiles -defaults: - render_format: json -`) - - cfg, err := parseRenderArgs([]string{ - "--config", configPath, - "--prompt", "p", - "--input", "a=b", - }) - if err != nil { - t.Fatalf("expected valid args, got %v", err) - } - - if len(cfg.promptkitSources) != 2 { - t.Fatalf("expected configured source selectors, got %#v", cfg.promptkitSources) - } - if cfg.outputFormat != renderformat.PreparedRunFormatJSON { - t.Fatalf("expected render format from config, got %q", cfg.outputFormat) - } -} - -func TestParseRenderArgsExplicitFormatOverridesConfigDefaultFormat(t *testing.T) { - configPath := writeAppConfigFile(t, ` -promptkit_sources: - - directory: ./from-config/prompts - - directory: ./from-config/profiles -defaults: - render_format: json -`) - - cfg, err := parseRenderArgs([]string{ - "--config", configPath, - "--prompt", "p", - "--input", "a=b", - "--format", "text", - }) - if err != nil { - t.Fatalf("expected valid args, got %v", err) - } - if cfg.outputFormat != renderformat.PreparedRunFormatText { - t.Fatalf("expected explicit --format text to override config default, got %q", cfg.outputFormat) - } -} - -func TestParseServeArgsWithExplicitConfigLoadsSettingsAndCLIAddrOverrides(t *testing.T) { - configPath := writeAppConfigFile(t, ` -promptkit_sources: - - directory: ./from-config/prompts - - directory: ./from-config/profiles -schema_dir: ./from-config/schemas -server: - addr: 127.0.0.1:9000 - artifact_root: ./from-config/artifacts - max_request_bytes: 1024 - max_artifact_bytes: 2048 - max_response_bytes: 4096 -`) - - cfg, err := parseServeArgs([]string{ - "--config", configPath, - "--addr", ":7777", - "--artifact-root", "./from-cli/artifacts", - "--max-request-bytes", "0", - "--max-artifact-bytes", "8192", - "--max-response-bytes", "16384", - }) - if err != nil { - t.Fatalf("expected valid args, got %v", err) - } - - if len(cfg.promptkitSources) != 2 { - t.Fatalf("expected configured source selectors, got %#v", cfg.promptkitSources) - } - if cfg.schemaDir != filepath.Clean("./from-config/schemas") { - t.Fatalf("expected schema dir from config, got %q", cfg.schemaDir) - } - if cfg.addr != ":7777" { - t.Fatalf("expected CLI addr override, got %q", cfg.addr) - } - if cfg.artifactRoot != filepath.Clean("./from-cli/artifacts") { - t.Fatalf("expected CLI artifact root override, got %q", cfg.artifactRoot) - } - if cfg.maxRequestBytes != 0 { - t.Fatalf("expected CLI max request bytes override, got %d", cfg.maxRequestBytes) - } - if cfg.maxArtifactBytes != 8192 { - t.Fatalf("expected CLI max artifact bytes override, got %d", cfg.maxArtifactBytes) - } - if cfg.maxResponseBytes != 16384 { - t.Fatalf("expected CLI max response bytes override, got %d", cfg.maxResponseBytes) - } -} - -func TestParseServeArgsWithConfigUsesConfiguredSourcesAndAddr(t *testing.T) { - configPath := writeAppConfigFile(t, ` -promptkit_sources: - - directory: ./from-config/prompts - - directory: ./from-config/profiles -schema_dir: ./from-config/schemas -server: - addr: 127.0.0.1:9000 - artifact_root: ./from-config/artifacts - max_request_bytes: 1024 - max_artifact_bytes: 2048 - max_response_bytes: 4096 -`) - - cfg, err := parseServeArgs([]string{ - "--config", configPath, - }) - if err != nil { - t.Fatalf("expected valid args, got %v", err) - } - - if len(cfg.promptkitSources) != 2 { - t.Fatalf("expected configured source selectors, got %#v", cfg.promptkitSources) - } - if cfg.schemaDir != filepath.Clean("./from-config/schemas") { - t.Fatalf("expected schema dir from config, got %q", cfg.schemaDir) - } - if cfg.addr != "127.0.0.1:9000" { - t.Fatalf("expected addr from config, got %q", cfg.addr) - } - if cfg.artifactRoot != filepath.Clean("./from-config/artifacts") { - t.Fatalf("expected artifact root from config, got %q", cfg.artifactRoot) - } - if cfg.maxRequestBytes != 1024 { - t.Fatalf("expected max request bytes from config, got %d", cfg.maxRequestBytes) - } - if cfg.maxArtifactBytes != 2048 { - t.Fatalf("expected max artifact bytes from config, got %d", cfg.maxArtifactBytes) - } - if cfg.maxResponseBytes != 4096 { - t.Fatalf("expected max response bytes from config, got %d", cfg.maxResponseBytes) - } -} - -func TestParseServeArgsRejectsNegativeSizeLimits(t *testing.T) { - tests := []struct { - name string - flag string - }{ - {name: "request", flag: "--max-request-bytes"}, - {name: "artifact", flag: "--max-artifact-bytes"}, - {name: "response", flag: "--max-response-bytes"}, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - _, err := parseServeArgs([]string{ - tc.flag, "-1", - }) - if err == nil { - t.Fatal("expected negative size limit error") - } - }) - } -} - -func TestRunAndRenderRejectServeSizeLimitFlags(t *testing.T) { - for _, tc := range []struct { - name string - parse func([]string) error - }{ - { - name: "run", - parse: func(args []string) error { - _, err := parseRunArgs(args) - return err - }, - }, - { - name: "render", - parse: func(args []string) error { - _, err := parseRenderArgs(args) - return err - }, - }, - } { - t.Run(tc.name, func(t *testing.T) { - err := tc.parse([]string{ - "--prompt", "p", - "--input", "a=b", - "--max-request-bytes", "1024", - }) - if err == nil { - t.Fatal("expected unsupported flag error") - } - }) - } -} - -func TestRunAndRenderBuildEquivalentRuntimeOverrideRequestsForSharedFlags(t *testing.T) { - configPath := writeAppConfigFile(t, "{}\n") - runCfg, err := parseRunArgs([]string{ - "--config", configPath, - "--prompt", "prompt-1", - "--profile", "profile-1", - "--input", "transcript=./transcript.md", - "--var", "session_date=2026-05-01", - "--llm-base-url", "http://localhost:8000/v1", - "--model", "model-x", - "--temperature", "0.8", - "--max-tokens", "123", - "--top-p", "0.6", - "--timeout", "90s", - "--api-key-env", "SCRIPTORIUM_API_KEY", - }) - if err != nil { - t.Fatalf("expected valid run args, got %v", err) - } - - renderCfg, err := parseRenderArgs([]string{ - "--config", configPath, - "--prompt", "prompt-1", - "--profile", "profile-1", - "--input", "transcript=./transcript.md", - "--var", "session_date=2026-05-01", - "--llm-base-url", "http://localhost:8000/v1", - "--model", "model-x", - "--temperature", "0.8", - "--max-tokens", "123", - "--top-p", "0.6", - "--timeout", "90s", - "--api-key-env", "SCRIPTORIUM_API_KEY", - }) - if err != nil { - t.Fatalf("expected valid render args, got %v", err) - } - - runReq, err := buildRunRequestFromConfig(runCfg) - if err != nil { - t.Fatalf("expected run request build success, got %v", err) - } - renderReq, err := buildRunRequestFromConfig(&renderCfg.runConfig) - if err != nil { - t.Fatalf("expected render request build success, got %v", err) - } - - if !reflect.DeepEqual(runReq, renderReq) { - t.Fatalf("expected run/render shared flag requests to match.\nrun=%#v\nrender=%#v", runReq, renderReq) - } -} - -func TestBuildRunRequestPreservesNumericOverridePresence(t *testing.T) { - omitted, err := buildRunRequestFromConfig(&runConfig{ - promptID: "prompt-1", - inputRaw: []string{"transcript=./transcript.md"}, - }) - if err != nil { - t.Fatalf("expected omitted override request to build, got %v", err) - } - if omitted.Execution != nil { - t.Fatalf("expected omitted numeric flags to leave execution override nil, got %#v", omitted.Execution) - } - - explicitZeros, err := buildRunRequestFromConfig(&runConfig{ - promptID: "prompt-1", - inputRaw: []string{"transcript=./transcript.md"}, - temperatureSet: true, - maxTokensSet: true, - topPSet: true, - timeoutSet: true, - }) - if err != nil { - t.Fatalf("expected explicit zero override request to build, got %v", err) - } - if explicitZeros.Execution == nil { - t.Fatal("expected explicit numeric flags to create execution override") - } - if explicitZeros.Execution.Temperature == nil || explicitZeros.Execution.MaxTokens == nil || explicitZeros.Execution.TopP == nil || explicitZeros.Execution.TimeoutSeconds == nil { - t.Fatalf("expected explicit zero numeric overrides to remain non-nil, got %#v", explicitZeros.Execution) - } - if *explicitZeros.Execution.Temperature != 0 || *explicitZeros.Execution.MaxTokens != 0 || *explicitZeros.Execution.TopP != 0 || *explicitZeros.Execution.TimeoutSeconds != 0 { - t.Fatalf("expected explicit numeric overrides to retain zero values, got %#v", explicitZeros.Execution) - } -} - -func TestBuildRunRequestPreservesReasoningEffortPresenceAndSessionID(t *testing.T) { - omitted, err := buildRunRequestFromConfig(&runConfig{promptID: "prompt-1"}) - if err != nil { - t.Fatalf("expected omitted request to build, got %v", err) - } - if omitted.Execution != nil { - t.Fatalf("expected omitted reasoning flag to leave execution nil, got %#v", omitted.Execution) - } - - for _, tc := range []struct { - name string - value string - }{ - {name: "replacement", value: "high"}, - {name: "clear", value: ""}, - } { - t.Run(tc.name, func(t *testing.T) { - req, err := buildRunRequestFromConfig(&runConfig{ - promptID: "prompt-1", - sessionID: "session-1", - reasoningEffort: tc.value, - reasoningEffortSet: true, - }) - if err != nil { - t.Fatalf("expected request to build, got %v", err) - } - if req.SessionID != "session-1" || req.Execution == nil || req.Execution.ReasoningEffort == nil || *req.Execution.ReasoningEffort != tc.value { - t.Fatalf("unexpected mapped request: %#v", req) - } - }) - } -} - -func TestBuildRunRequestAllowsOmittedInputsAndPreservesQualifiedIDs(t *testing.T) { - req, err := buildRunRequestFromConfig(&runConfig{ - promptID: "support/prompt-1", - profileID: "support/profile-1", - }) - if err != nil { - t.Fatalf("expected request without inputs to build, got %v", err) - } - if req.PromptID != "support/prompt-1" || req.ProfileID != "support/profile-1" { - t.Fatalf("qualified IDs changed: %+v", req) - } - if req.Inputs != nil { - t.Fatalf("expected omitted inputs to remain nil, got %#v", req.Inputs) - } -} - -func TestExecutionCommandsRejectPromptVersion(t *testing.T) { - args := []string{"--prompt", "support/answer", "--prompt-version", "1"} - if _, err := parseRunArgs(args); err == nil { - t.Fatal("run accepted removed prompt version flag") - } - if _, err := parseRenderArgs(args); err == nil { - t.Fatal("render accepted removed prompt version flag") - } -} - -func TestRunOperationTimeoutAttribution(t *testing.T) { - for _, tc := range []struct { - name string - err error - deadline bool - }{ - {name: "operation budget", err: fmt.Errorf("wrapped: %w", promptkit.ErrOperationTimeout), deadline: true}, - {name: "provider deadline", err: fmt.Errorf("provider: %w", context.DeadlineExceeded)}, - } { - t.Run(tc.name, func(t *testing.T) { - var stderr bytes.Buffer - code := executeRun(&runConfig{runTimeout: 125 * time.Millisecond}, promptkit.RunRequest{}, func(ctx context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) { - if req.OperationTimeout != 125*time.Millisecond { - t.Errorf("operation timeout = %v", req.OperationTimeout) - } - if _, ok := ctx.Deadline(); ok { - t.Error("adapter added a context deadline") - } - return nil, tc.err - }, io.Discard, &stderr) - if code != ExitRuntimeError || strings.Contains(stderr.String(), "execution deadline exceeded") != tc.deadline { - t.Fatalf("unexpected outcome: code=%d stderr=%q", code, stderr.String()) - } - }) - } -} - -func TestRunAndRenderParsingDoesNotRequireCallerSources(t *testing.T) { - configPath := writeAppConfigFile(t, "{}\n") - if cfg, err := parseRunArgs([]string{"--config", configPath, "--prompt", "p"}); err != nil || len(cfg.promptkitSources) != 0 { - t.Fatalf("run should parse without caller sources: cfg=%+v err=%v", cfg, err) - } - if cfg, err := parseRenderArgs([]string{"--config", configPath, "--prompt", "p"}); err != nil || len(cfg.promptkitSources) != 0 { - t.Fatalf("render should parse without caller sources: cfg=%+v err=%v", cfg, err) - } -} - -func TestDetermineExitCode(t *testing.T) { - if got := determineExitCode(errors.New("boom"), nil); got != ExitRuntimeError { - t.Fatalf("expected runtime exit code, got %d", got) - } - if got := determineExitCode(nil, &promptkit.RunResult{Validation: promptkit.ValidationResult{Status: promptkit.ValidationFailed}}); got != ExitValidationFailed { - t.Fatalf("expected validation exit code, got %d", got) - } - if got := determineExitCode(nil, &promptkit.RunResult{Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed}}); got != ExitOK { - t.Fatalf("expected success exit code for passed validation, got %d", got) - } - if got := determineExitCode(nil, &promptkit.RunResult{Validation: promptkit.ValidationResult{Status: promptkit.ValidationSkipped}}); got != ExitOK { - t.Fatalf("expected success exit code for skipped validation, got %d", got) - } -} - -func TestRunCommandVarsOptional(t *testing.T) { - lib := newCLITestLibrary(t) - configPath := writeAppConfigFile(t, "{}\n") - writePromptFile(t, lib.promptDir, "p", "local-default") - writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "m") - inputPath := lib.writeInputFile(t, "t.md", "hello") - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := runCommand([]string{ - "--config", configPath, - "--promptkit-source", lib.promptDir, - "--promptkit-source", lib.profileDir, - "--prompt", "p", - "--input", "transcript=" + inputPath, - "--llm-base-url", "http://[::1", - "--model", "m", - }, &stdout, &stderr) - - if code != ExitRuntimeError { - t.Fatalf("expected runtime error exit code, got %d", code) - } - if strings.Contains(stderr.String(), "var parse error") { - t.Fatalf("expected --var to be optional, got stderr=%q", stderr.String()) - } - if !strings.Contains(stderr.String(), "llm client error") && !strings.Contains(stderr.String(), "run error") { - t.Fatalf("expected post-parse execution error, got stderr=%q", stderr.String()) - } - if stdout.Len() != 0 { - t.Fatalf("expected no stdout output on error, got %q", stdout.String()) - } -} - -func TestRunCommandSucceedsWithPromptAndProfileSourcesFromConfig(t *testing.T) { - lib := newCLITestLibrary(t) - inputPath := lib.writeInputFile(t, "transcript.md", "hello") - - ts := newTestLLMServer("from-config-dirs", nil) - defer ts.Close() - - writePromptFile(t, lib.promptDir, "prompt.default", "local-default") - writeProfileFile(t, lib.profileDir, "local-default", ts.URL+"/v1", "profile-model") - configPath := writeAppConfigFile(t, sourceSelectionsYAML(lib.promptDir, lib.profileDir, "")) - - code, stdout, stderr := runCLICommand(t, runCommand, []string{ - "--config", configPath, - "--prompt", "prompt.default", - "--input", "transcript=" + inputPath, - }) - if code != ExitOK { - t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr) - } - if stdout != "from-config-dirs" { - t.Fatalf("unexpected stdout output: %q", stdout) - } -} - -func TestRenderCommandDefaultFormatTextIncludesPreparedDetailsAndNoSecrets(t *testing.T) { - const envName = "SCRIPTORIUM_RENDER_TEST_API_KEY" - const secret = "super-secret-render-key" - t.Setenv(envName, secret) - - lib := newCLITestLibrary(t) - inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript") - - writePromptFileWithTemplate(t, lib.promptDir, "prompt.render", "local-default", "Date {{.session_date}} - Summarize: {{input \"transcript\"}}") - writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model") - - code, stdout, stderr := runCLICommand(t, renderCommand, []string{ - "--config", writeAppConfigFile(t, "{}\n"), - "--promptkit-source", lib.promptDir, - "--promptkit-source", lib.profileDir, - "--prompt", "prompt.render", - "--profile", "local-default", - "--input", "transcript=" + inputPath, - "--var", "session_date=2026-05-04", - "--llm-base-url", "http://override.local/v1", - "--model", "override-model", - "--temperature", "0.7", - "--max-tokens", "55", - "--top-p", "0.2", - "--timeout", "20s", - "--api-key-env", envName, - }) - if code != ExitOK { - t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr) - } - if stderr != "" { - t.Fatalf("expected empty stderr on success, got %q", stderr) - } - - out := stdout - for _, want := range []string{ - "prompt: prompt.render", - "selected_profile_id: local-default", - "endpoint: http://override.local/v1", - "model: override-model", - "temperature: 0.7", - "max_tokens: 55", - "top_p: 0.2", - "timeout_seconds: 20", - "api_key_env: " + envName, - "rendered_prompt_hash:", - "messages:", - "Date 2026-05-04", - "Summarize:", - "hello transcript", - } { - if !strings.Contains(out, want) { - t.Fatalf("expected render text output to include %q, got:\n%s", want, out) - } - } - if strings.Contains(out, secret) { - t.Fatalf("render output unexpectedly contained secret api key value: %s", out) - } -} - -func TestRenderCommandExplicitZeroTemperatureReachesEffectiveSettings(t *testing.T) { - lib := newCLITestLibrary(t) - inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript") - - writePromptFile(t, lib.promptDir, "prompt.render", "local-default") - profile := `profiles: - - id: local-default - endpoint: http://127.0.0.1:1/v1 - model: profile-model - temperature: 0.7 -` - if err := os.WriteFile(filepath.Join(lib.profileDir, "local-default.yaml"), []byte(profile), 0o644); err != nil { - t.Fatalf("failed to write profile fixture: %v", err) - } - - code, stdout, stderr := runCLICommand(t, renderCommand, []string{ - "--config", lib.configPath, - "--promptkit-source", lib.promptDir, - "--promptkit-source", lib.profileDir, - "--prompt", "prompt.render", - "--input", "transcript=" + inputPath, - "--temperature", "0", - }) - if code != ExitOK { - t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr) - } - if !strings.Contains(stdout, "\n temperature: 0\n") { - t.Fatalf("expected explicit zero temperature in effective settings, got:\n%s", stdout) - } -} - -func TestRenderCommandSucceedsWithPromptAndProfileSourcesFromConfig(t *testing.T) { - lib := newCLITestLibrary(t) - inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript") - - writePromptFile(t, lib.promptDir, "prompt.render", "local-default") - writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model") - configPath := writeAppConfigFile(t, sourceSelectionsYAML(lib.promptDir, lib.profileDir, "")) - - code, stdout, stderr := runCLICommand(t, renderCommand, []string{ - "--config", configPath, - "--prompt", "prompt.render", - "--input", "transcript=" + inputPath, - }) - if code != ExitOK { - t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr) - } - if !strings.Contains(stdout, "prompt: prompt.render") { - t.Fatalf("expected rendered output, got %q", stdout) - } -} - -func TestRenderCommandLoadsImportsAndOverlayWithoutChangingExecutionProfile(t *testing.T) { - lib := newCLITestLibrary(t) - inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript") - writePromptFile(t, lib.promptDir, "prompt.render", "local-default") - writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model") - if err := os.MkdirAll(filepath.Join(lib.rootDir, "fragments"), 0o755); err != nil { - t.Fatal(err) - } - if err := os.MkdirAll(filepath.Join(lib.rootDir, "overlays"), 0o755); err != nil { - t.Fatal(err) - } - base := filepath.Join(lib.rootDir, "config.yml") - framework := writeFrameworkSourceFile(t, lib.rootDir, "framework.yml", "config:\n imports:\n - path: prompts/prompt.render.yaml\n required: true\n") - for path, body := range map[string]string{ - base: "config:\n imports:\n - path: fragments/sources.yml\n required: true\n", - filepath.Join(lib.rootDir, "fragments", "sources.yml"): fmt.Sprintf("promptkit_sources:\n - file: %q\n - directory: %q\n", framework, lib.profileDir), - filepath.Join(lib.rootDir, "overlays", "local.yml"): "defaults:\n render_format: json\n", - } { - if err := os.WriteFile(path, []byte(body), 0o644); err != nil { - t.Fatal(err) - } - } - args := []string{"--config", base, "--config-overlay", "local", "--profile", "local-default", "--prompt", "prompt.render", "--input", "transcript=" + inputPath} - cfg, err := parseRenderArgs(args) - if err != nil { - t.Fatal(err) - } - if cfg.profileID != "local-default" || cfg.outputFormat != renderformat.PreparedRunFormatJSON { - t.Fatalf("overlay changed execution profile or output selection: %+v", cfg) - } - code, stdout, stderr := runCLICommand(t, renderCommand, args) - if code != ExitOK || !strings.Contains(stdout, `"selected_profile_id": "local-default"`) { - t.Fatalf("render did not use composed config and independent execution profile: code=%d out=%q err=%q", code, stdout, stderr) - } -} - -func TestRenderCommandUsesConfiguredCustomBackend(t *testing.T) { - lib := newCLITestLibrary(t) - writePromptDefinition(t, lib.promptDir, "custom.yaml", `prompts: - - id: custom - default_profile: local-gpu - messages: - - role: user - content: "hello" - output: - format: text - validation_mode: none -`) - if err := os.WriteFile(filepath.Join(lib.profileDir, "local-gpu.yaml"), []byte(`profiles: - - id: local-gpu - backend: local-gpu - model: local-model -`), 0o644); err != nil { - t.Fatalf("write profile fixture: %v", err) - } - configPath := writeCLIFrameworkConfig(t, lib, `backends: - - id: local-gpu - endpoint: http://localhost:11434/v1 - extra_params: - provider_option: enabled - concurrency_limit: 2 - queue_capacity: 0 -`, "") - - code, stdout, stderr := runCLICommand(t, renderCommand, []string{ - "--config", configPath, - "--prompt", "custom", - }) - if code != ExitOK { - t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr) - } - if !strings.Contains(stdout, "selected_backend_id: local-gpu") { - t.Fatalf("expected configured backend in prepared output, got:\n%s", stdout) - } -} - -func TestRenderCommandUsesInheritedCustomBackend(t *testing.T) { - lib := newCLITestLibrary(t) - writePromptDefinition(t, lib.promptDir, "custom.yaml", `prompts: - - id: custom - default_profile: local-gpu - messages: - - role: user - content: "hello" - output: - format: text - validation_mode: none -`) - if err := os.WriteFile(filepath.Join(lib.profileDir, "local-gpu.yaml"), []byte(`profiles: - - id: local-gpu - backend: local-gpu - model: local-model -`), 0o644); err != nil { - t.Fatalf("write profile fixture: %v", err) - } - configPath := writeCLIFrameworkConfig(t, lib, `backends: - - id: local-base - endpoint: http://localhost:11434/v1 - api_key_env: LOCAL_GPU_API_KEY - concurrency_limit: 2 - - id: local-gpu - extends: {id: local-base} - queue_capacity: 0 - extra_params: - provider_option: false -`, "") - - code, stdout, stderr := runCLICommand(t, renderCommand, []string{ - "--config", configPath, - "--prompt", "custom", - "--format", "json", - }) - if code != ExitOK { - t.Fatalf("expected inherited backend preparation to succeed, got %d stderr=%q", code, stderr) - } - var prepared promptkit.PreparedRun - if err := json.Unmarshal([]byte(stdout), &prepared); err != nil { - t.Fatalf("decode prepared output: %v\n%s", err, stdout) - } - if prepared.SelectedBackendID != "local-gpu" || prepared.EffectiveModelParams.BackendID != "local-gpu" { - t.Fatalf("prepared run did not select the child backend: %+v", prepared) - } - if prepared.EffectiveModelParams.Endpoint != "http://localhost:11434/v1" { - t.Fatalf("prepared run did not use the inherited endpoint: %+v", prepared.EffectiveModelParams) - } -} - -func TestConfiguredBackendPreservesIntegerParameterAndOptionalCredential(t *testing.T) { - lib := newCLITestLibrary(t) - writePromptDefinition(t, lib.promptDir, "custom.yaml", `prompts: - - id: custom - default_profile: local-gpu - messages: - - role: user - content: "hello" - output: - format: text - validation_mode: none -`) - if err := os.WriteFile(filepath.Join(lib.profileDir, "local-gpu.yaml"), []byte("profiles:\n - id: local-gpu\n backend: local-gpu\n model: local-model\n"), 0o644); err != nil { - t.Fatal(err) - } - const credentialName = "SCRIPTORIUM_OPTIONAL_BACKEND_KEY" - t.Setenv(credentialName, "") - type observedRequest struct { - number json.Number - authorization string - } - observed := make(chan observedRequest, 1) - provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - request := observedRequest{authorization: r.Header.Get("Authorization")} - var body map[string]any - decoder := json.NewDecoder(r.Body) - decoder.UseNumber() - if err := decoder.Decode(&body); err == nil { - request.number, _ = body["synthetic_count"].(json.Number) - } - observed <- request - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"choices":[{"message":{"role":"assistant","content":"generated"}}]}`)) - })) - defer provider.Close() - configPath := writeCLIFrameworkConfig(t, lib, fmt.Sprintf(`backends: - - id: local-gpu - endpoint: %q - api_key_env: %s - extra_params: - synthetic_count: 7 -`, provider.URL+"/v1", credentialName), "") - cfg, err := parseRunArgs([]string{"--config", configPath, "--prompt", "custom"}) - if err != nil { - t.Fatalf("ConfigKit application config did not parse: cfg=%+v err=%v", cfg, err) - } - code, stdout, stderr := runCLICommand(t, runCommand, []string{"--config", configPath, "--prompt", "custom"}) - if code != ExitOK { - t.Fatalf("backend run failed: code=%d out=%q err=%q", code, stdout, stderr) - } - request := <-observed - if !strings.Contains(stdout, "generated") || request.number != "7" || request.authorization != "" { - t.Fatalf("backend request changed: code=%d out=%q err=%q count=%q auth=%q", code, stdout, stderr, request.number, request.authorization) - } -} - -func TestRenderCommandMapsReasoningEffortAndSessionID(t *testing.T) { - lib := newCLITestLibrary(t) - writePromptDefinition(t, lib.promptDir, "session.yaml", `prompts: - - id: session - default_profile: local - session_id: definition-session - messages: - - role: user - content: "hello" - output: - format: text - validation_mode: none -`) - if err := os.WriteFile(filepath.Join(lib.profileDir, "local.yaml"), []byte(`profiles: - - id: local - endpoint: http://127.0.0.1:1/v1 - model: local-model - reasoning_effort: low -`), 0o644); err != nil { - t.Fatalf("write profile fixture: %v", err) - } - - for _, tc := range []struct { - name string - args []string - wantReasoning string - wantSessionID string - absentReasoning bool - }{ - {name: "omitted reasoning inherits profile", wantReasoning: "low", wantSessionID: "definition-session"}, - {name: "nonblank reasoning replaces profile", args: []string{"--reasoning-effort", "high"}, wantReasoning: "high", wantSessionID: "definition-session"}, - {name: "empty reasoning clears profile", args: []string{"--reasoning-effort="}, wantSessionID: "definition-session", absentReasoning: true}, - {name: "direct session replaces definition", args: []string{"--session-id", "direct-session"}, wantReasoning: "low", wantSessionID: "direct-session"}, - } { - t.Run(tc.name, func(t *testing.T) { - args := []string{"--config", lib.configPath, "--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir, "--prompt", "session"} - args = append(args, tc.args...) - code, stdout, stderr := runCLICommand(t, renderCommand, args) - if code != ExitOK { - t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr) - } - if !strings.Contains(stdout, "session_id: "+tc.wantSessionID) { - t.Fatalf("expected session ID %q, got:\n%s", tc.wantSessionID, stdout) - } - hasReasoning := strings.Contains(stdout, "reasoning_effort:") - if tc.absentReasoning { - if hasReasoning { - t.Fatalf("expected cleared reasoning to be omitted, got:\n%s", stdout) - } - return - } - if !strings.Contains(stdout, "reasoning_effort: "+tc.wantReasoning) { - t.Fatalf("expected reasoning effort %q, got:\n%s", tc.wantReasoning, stdout) - } - }) - } -} - -func TestRenderCommandOmitsEmptyEffectiveSessionID(t *testing.T) { - lib := newCLITestLibrary(t) - writePromptDefinition(t, lib.promptDir, "plain.yaml", `prompts: - - id: plain - default_profile: local - messages: - - role: user - content: "hello" - output: - format: text - validation_mode: none -`) - writeProfileFile(t, lib.profileDir, "local", "http://127.0.0.1:1/v1", "local-model") - - code, stdout, stderr := runCLICommand(t, renderCommand, []string{ - "--config", lib.configPath, - "--promptkit-source", lib.promptDir, - "--promptkit-source", lib.profileDir, - "--prompt", "plain", - }) - if code != ExitOK { - t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr) - } - if strings.Contains(stdout, "session_id:") { - t.Fatalf("expected no effective session ID, got:\n%s", stdout) - } -} - -func TestRenderCommandRejectsOverlongSessionID(t *testing.T) { - lib := newCLITestLibrary(t) - writePromptDefinition(t, lib.promptDir, "session.yaml", `prompts: - - id: session - default_profile: local - messages: - - role: user - content: "hello" - output: - format: text - validation_mode: none -`) - writeProfileFile(t, lib.profileDir, "local", "http://127.0.0.1:1/v1", "local-model") - - code, _, stderr := runCLICommand(t, renderCommand, []string{ - "--config", lib.configPath, - "--promptkit-source", lib.promptDir, - "--promptkit-source", lib.profileDir, - "--prompt", "session", - "--session-id", strings.Repeat("x", 257), - }) - if code != ExitRuntimeError { - t.Fatalf("expected runtime error, got %d stderr=%q", code, stderr) - } - if !strings.Contains(stderr, "invalid run request") { - t.Fatalf("expected invalid-request context, got %q", stderr) - } -} - -func TestInspectPromptCommandFormatsDefinitionWithoutProfileOrGeneration(t *testing.T) { - lib := newCLITestLibrary(t) - writePromptDefinition(t, lib.promptDir, "inspect.yaml", `prompts: - - id: inspect - inputs: [] - messages: - - role: user - content: "private message" - output: - format: text - validation_mode: none -`) - code, stdout, stderr := runCLICommand(t, inspectCommand, []string{"prompt", "--config", lib.configPath, "--promptkit-source", lib.promptDir, "--prompt", "inspect", "--format", "json"}) - if code != ExitOK { - t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr) - } - if !strings.Contains(stdout, `"prompt_id": "inspect"`) || !strings.Contains(stdout, `"inputs": []`) || strings.Contains(stdout, "private message") || strings.Contains(stdout, "prompt_version") { - t.Fatalf("unexpected inspection output: %s", stdout) - } - code, stdout, stderr = runCLICommand(t, inspectCommand, []string{"prompt", "--config", lib.configPath, "--promptkit-source", lib.promptDir, "--prompt", "missing"}) - if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "inspect error") { - t.Fatalf("expected failed inspection without output, got code=%d stdout=%q stderr=%q", code, stdout, stderr) - } - if _, err := parsePromptInspectionArgs([]string{"--prompt", "inspect", "--prompt-version", "1"}); err == nil { - t.Fatal("inspection accepted removed prompt version flag") - } -} - -func TestInspectionParsersPreserveExplicitEmptyConfigPath(t *testing.T) { - promptConfig, err := parsePromptInspectionArgs([]string{"--config=", "--prompt", "fixture"}) - if err != nil { - t.Fatalf("parse prompt inspection: %v", err) - } - if promptConfig.configPath != "" || !promptConfig.configExplicit { - t.Fatalf("expected explicit empty prompt config path, got %+v", promptConfig) - } - - profileConfig, err := parseProfileInspectionArgs([]string{"--config=", "--profile", "fixture"}) - if err != nil { - t.Fatalf("parse profile inspection: %v", err) - } - if profileConfig.configPath != "" || !profileConfig.configExplicit { - t.Fatalf("expected explicit empty profile config path, got %+v", profileConfig) - } -} - -func TestPromptkitDefinitionsRenderThroughCLI(t *testing.T) { - fixtureRoot := definitionFixtureRoot(t) - configPath := writeDefinitionFixtureConfig(t, fixtureRoot) - inputPath := filepath.Join(fixtureRoot, "inputs", "source.md") - - code, stdout, stderr := runCLICommand(t, renderCommand, []string{ - "--config", configPath, - "--prompt", "compat.complete", - "--input", "source=" + inputPath, - "--var", "topic=testing", - "--format", "json", - }) - if code != ExitOK { - t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr) - } - - var payload map[string]any - if err := json.Unmarshal([]byte(stdout), &payload); err != nil { - t.Fatalf("decode rendered fixture: %v\nbody=%s", err, stdout) - } - if _, present := payload["prompt_version"]; present || payload["selected_profile_id"] != "custom-derived" || payload["selected_backend_id"] != "fixture-custom" { - t.Fatalf("unexpected selected definition and target: %#v", payload) - } - if payload["session_id"] != "fixture-testing" { - t.Fatalf("expected rendered session template, got %#v", payload["session_id"]) - } - - outputContract := payload["output_contract"].(map[string]any) - if outputContract["format"] != "json" || outputContract["validation_mode"] != "json_schema" || outputContract["repair_attempts"] != float64(2) { - t.Fatalf("unexpected output contract: %#v", outputContract) - } - if _, ok := payload["structured_output"].(map[string]any)["json_schema"]; !ok { - t.Fatalf("expected loaded JSON schema metadata, got %#v", payload["structured_output"]) - } - - messages := payload["messages"].([]any) - wantRoles := []string{"developer", "system", "user", "assistant"} - for i, wantRole := range wantRoles { - message := messages[i].(map[string]any) - if message["role"] != wantRole { - t.Fatalf("message %d: expected role %q, got %#v", i, wantRole, message["role"]) - } - } - cacheControl := messages[0].(map[string]any)["cache_control"].(map[string]any) - if cacheControl["type"] != "ephemeral" || cacheControl["ttl"] != "1h" { - t.Fatalf("unexpected cache control: %#v", cacheControl) - } - if !strings.Contains(messages[2].(map[string]any)["content"].(string), "stable, synthetic material") { - t.Fatalf("file-backed input template was not rendered: %#v", messages[2]) - } - inputHashes := payload["input_hashes"].(map[string]any) - if len(inputHashes) != 1 || inputHashes["source"] == "" { - t.Fatalf("required and omitted optional inputs were not preserved: %#v", inputHashes) - } -} - -func TestTransientRetrySettingsAndCLIOverride(t *testing.T) { - lib := newCLITestLibrary(t) - inputPath := lib.writeInputFile(t, "transcript.md", "hello") - writePromptFile(t, lib.promptDir, "retry.prompt", "local-default") - var hits int32 - provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if atomic.AddInt32(&hits, 1) == 1 { - http.Error(w, "temporary", http.StatusServiceUnavailable) - return - } - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"choices":[{"message":{"role":"assistant","content":"recovered"}}]}`)) - })) - defer provider.Close() - writeProfileFile(t, lib.profileDir, "local-default", provider.URL+"/v1", "local-model") - configPath := writeAppConfigFile(t, sourceSelectionsYAML(lib.promptDir, lib.profileDir, "")+"transient_retries: 1\n") - args := []string{"--config", configPath, "--prompt", "retry.prompt", "--input", "transcript=" + inputPath} - - serve, err := parseServeArgs([]string{"--config", configPath}) - if err != nil || serve.transientRetries == nil || *serve.transientRetries != 1 { - t.Fatalf("server did not inherit retry setting: cfg=%+v err=%v", serve, err) - } - if _, err := newEngine(serve.engineSettings()); err != nil { - t.Fatalf("server engine configuration: %v", err) - } - - code, stdout, stderr := runCLICommand(t, runCommand, args) - if code != ExitOK || stdout != "recovered" || atomic.LoadInt32(&hits) != 2 { - t.Fatalf("configured retry was not applied: code=%d stdout=%q stderr=%q hits=%d", code, stdout, stderr, atomic.LoadInt32(&hits)) - } - atomic.StoreInt32(&hits, 0) - code, stdout, stderr = runCLICommand(t, runCommand, append(append([]string{}, args...), "--transient-retries", "0")) - if code != ExitRuntimeError || stdout != "" || atomic.LoadInt32(&hits) != 1 { - t.Fatalf("explicit zero did not override configuration: code=%d stdout=%q stderr=%q hits=%d", code, stdout, stderr, atomic.LoadInt32(&hits)) - } - - for _, tc := range []struct { - name string - args []string - want *int - }{ - {name: "omitted", args: args}, - {name: "zero", args: append(append([]string{}, args...), "--transient-retries", "0"), want: intPointer(0)}, - {name: "positive", args: append(append([]string{}, args...), "--transient-retries", "3"), want: intPointer(3)}, - } { - t.Run(tc.name, func(t *testing.T) { - cfg, err := parseRunArgs(tc.args) - if err != nil { - t.Fatal(err) - } - req, err := buildRunRequestFromConfig(cfg) - if err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(req.TransientRetries, tc.want) { - t.Fatalf("request retry override: got %v, want %v", req.TransientRetries, tc.want) - } - }) - } - for _, invalid := range []string{"-1", "wrong"} { - if _, err := parseRunArgs(append(append([]string{}, args...), "--transient-retries", invalid)); err == nil { - t.Fatalf("run accepted invalid retry count %q", invalid) - } - if _, err := parseRenderArgs(append(append([]string{}, args...), "--transient-retries", invalid)); err == nil { - t.Fatalf("render accepted invalid retry count %q", invalid) - } - } - code, stdout, stderr = runCLICommand(t, renderCommand, append(append([]string{}, args...), "--transient-retries", "0", "--format", "json")) - if code != ExitOK || strings.Contains(stdout, "transient_retries") { - t.Fatalf("render retry override changed prepared output: code=%d stdout=%q stderr=%q", code, stdout, stderr) - } -} - -func TestRunTimeoutParsingAndCancellation(t *testing.T) { - lib := newCLITestLibrary(t) - inputPath := lib.writeInputFile(t, "transcript.md", "hello") - args := []string{"--config", lib.configPath, "--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir, "--prompt", "timed", "--input", "transcript=" + inputPath} - for _, tc := range []struct { - name string - flag []string - want time.Duration - }{ - {name: "omitted"}, - {name: "zero", flag: []string{"--run-timeout", "0s"}}, - {name: "subsecond", flag: []string{"--run-timeout", "125ms"}, want: 125 * time.Millisecond}, - } { - t.Run(tc.name, func(t *testing.T) { - cfg, err := parseRunArgs(append(append([]string{}, args...), tc.flag...)) - if err != nil || cfg.runTimeout != tc.want { - t.Fatalf("run timeout: cfg=%+v err=%v", cfg, err) - } - }) - } - if _, err := parseRunArgs(append(append([]string{}, args...), "--run-timeout", "-1ms")); err == nil { - t.Fatal("negative run timeout accepted") - } - if _, err := parseRenderArgs(append(append([]string{}, args...), "--run-timeout", "1s")); err == nil { - t.Fatal("render accepted run-only timeout") - } - - started := make(chan struct{}, 1) - release := make(chan struct{}) - provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - started <- struct{}{} - <-release - })) - defer provider.Close() - defer close(release) - writePromptFile(t, lib.promptDir, "timed", "local-default") - writeProfileFile(t, lib.profileDir, "local-default", provider.URL+"/v1", "local-model") - code, stdout, stderr := runCLICommand(t, runCommand, append(append([]string{}, args...), "--run-timeout", "500ms", "--transient-retries", "0")) - if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "execution deadline exceeded") { - t.Fatalf("deadline outcome: code=%d stdout=%q stderr=%q", code, stdout, stderr) - } - select { - case <-started: - default: - t.Fatal("provider call did not start before deadline") - } -} - -func TestRunCommandWritesSeparateExecutionReport(t *testing.T) { - lib := newCLITestLibrary(t) - inputPath := lib.writeInputFile(t, "transcript.md", "hello") - var hits int32 - provider := newTestLLMServer("generated", &hits) - defer provider.Close() - writePromptFile(t, lib.promptDir, "reported", "local-default") - writeProfileFile(t, lib.profileDir, "local-default", provider.URL+"/v1", "local-model") - path := filepath.Join(t.TempDir(), "execution.json") - code, stdout, stderr := runCLICommand(t, runCommand, []string{ - "--config", lib.configPath, - "--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir, - "--prompt", "reported", "--input", "transcript=" + inputPath, - "--execution-report", path, - }) - if code != ExitOK || stdout != "generated" || atomic.LoadInt32(&hits) != 1 { - t.Fatalf("run with report: code=%d stdout=%q stderr=%q hits=%d", code, stdout, stderr, atomic.LoadInt32(&hits)) - } - data, err := os.ReadFile(path) - if err != nil { - t.Fatal(err) - } - if !bytes.HasSuffix(data, []byte("\n")) { - t.Fatalf("report is not newline-terminated: %q", data) - } - var report map[string]any - if err := json.Unmarshal(data, &report); err != nil { - t.Fatal(err) - } - if report["prompt_id"] != "reported" || report["outcome"] != "success" || report["attempts"] != float64(1) { - t.Fatalf("unexpected execution report: %#v", report) - } -} - -func TestExecutionReportFailureAndOutputHandling(t *testing.T) { - terminal := func(ctx context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) { - if req.Observer != nil { - req.Observer(promptkit.ExecutionEvent{Kind: promptkit.ExecutionFinished, PromptID: "p", Outcome: promptkit.ExecutionOutcomeSuccess, Attempts: 1}) - } - return &promptkit.RunResult{Artifact: promptkit.Artifact{Body: []byte("artifact")}, Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed}}, nil - } - for _, tc := range []struct { - name string - modify func(*runConfig, string) - run func(context.Context, promptkit.RunRequest) (*promptkit.RunResult, error) - wantCode int - wantOutput string - wantDiagnostic string - wantReport bool - }{ - {name: "default streams", run: terminal, wantCode: ExitOK, wantOutput: "artifact"}, - {name: "validation failure", run: func(ctx context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) { - req.Observer(promptkit.ExecutionEvent{Kind: promptkit.ExecutionFinished, Outcome: promptkit.ExecutionOutcomeInvalidOutput}) - return &promptkit.RunResult{Artifact: promptkit.Artifact{Body: []byte("invalid")}, Validation: promptkit.ValidationResult{Status: promptkit.ValidationFailed}}, nil - }, wantCode: ExitValidationFailed, wantOutput: "invalid", wantReport: true}, - {name: "execution failure", run: func(ctx context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) { - req.Observer(promptkit.ExecutionEvent{Kind: promptkit.ExecutionFinished, Outcome: promptkit.ExecutionOutcomeError, Failure: promptkit.ExecutionFailureProviderStatus}) - return nil, promptkit.ErrLLMGenerate - }, wantCode: ExitRuntimeError, wantDiagnostic: "run error", wantReport: true}, - {name: "artifact write failure still writes report", run: terminal, modify: func(cfg *runConfig, root string) { - cfg.outputPath = filepath.Join(root, "missing", "artifact.txt") - }, wantCode: ExitRuntimeError, wantDiagnostic: "output write error", wantReport: true}, - {name: "report write failure preserves artifact", run: terminal, modify: func(cfg *runConfig, root string) { - cfg.executionReportPath = filepath.Join(root, "missing", "execution.json") - }, wantCode: ExitRuntimeError, wantOutput: "artifact", wantDiagnostic: "execution report write error"}, - {name: "execution and report failures are both visible", run: func(ctx context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) { - req.Observer(promptkit.ExecutionEvent{Kind: promptkit.ExecutionFinished, Outcome: promptkit.ExecutionOutcomeError}) - return nil, promptkit.ErrLLMGenerate - }, modify: func(cfg *runConfig, root string) { - cfg.executionReportPath = filepath.Join(root, "missing", "execution.json") - }, wantCode: ExitRuntimeError, wantDiagnostic: "execution report write error"}, - } { - t.Run(tc.name, func(t *testing.T) { - root := t.TempDir() - cfg := &runConfig{executionReportPath: filepath.Join(root, "execution.json")} - if tc.name == "default streams" { - cfg.executionReportPath = "" - } - if tc.modify != nil { - tc.modify(cfg, root) - } - var stdout, stderr bytes.Buffer - code := executeRun(cfg, promptkit.RunRequest{PromptID: "p"}, tc.run, &stdout, &stderr) - if code != tc.wantCode || stdout.String() != tc.wantOutput || (tc.wantDiagnostic != "" && !strings.Contains(stderr.String(), tc.wantDiagnostic)) { - t.Fatalf("unexpected output: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) - } - if tc.wantReport { - if _, err := os.Stat(cfg.executionReportPath); err != nil { - t.Fatalf("terminal report missing: %v", err) - } - } - if tc.name == "execution and report failures are both visible" && !strings.Contains(stderr.String(), "run error") { - t.Fatalf("execution failure diagnostic was lost: %s", stderr.String()) - } - }) - } -} - -func TestExecutionReportLeavesExistingFileWhenNoTerminalEvent(t *testing.T) { - path := filepath.Join(t.TempDir(), "execution.json") - if err := os.WriteFile(path, []byte("old report"), 0o644); err != nil { - t.Fatal(err) - } - cfg := &runConfig{executionReportPath: path} - var stdout, stderr bytes.Buffer - code := executeRun(cfg, promptkit.RunRequest{PromptID: "p"}, func(ctx context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) { - if req.Observer == nil { - t.Fatal("requested report did not attach observer") - } - return &promptkit.RunResult{Artifact: promptkit.Artifact{Body: []byte("artifact")}, Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed}}, nil - }, &stdout, &stderr) - data, err := os.ReadFile(path) - if err != nil { - t.Fatal(err) - } - if code != ExitOK || stdout.String() != "artifact" || string(data) != "old report" || !strings.Contains(stderr.String(), "no terminal observation") { - t.Fatalf("no-terminal handling: code=%d stdout=%q stderr=%q report=%q", code, stdout.String(), stderr.String(), data) - } -} - -func TestExecutionReportPathValidationAndPreInvocationFailure(t *testing.T) { - root := t.TempDir() - configPath := writeAppConfigFile(t, "{}\n") - args := []string{"--config", configPath, "--promptkit-source", root, "--prompt", "p"} - path := filepath.Join(root, "execution.json") - for _, tail := range [][]string{ - {"--execution-report", ""}, - {"--out", path, "--execution-report", path}, - {"--out", path, "--execution-report", filepath.Join(root, "subdir", "..", "execution.json")}, - } { - if _, err := parseRunArgs(append(append([]string{}, args...), tail...)); err == nil { - t.Fatalf("accepted invalid report destination: %q", tail) - } - } - if err := os.WriteFile(path, []byte("old report"), 0o644); err != nil { - t.Fatal(err) - } - alias := filepath.Join(root, "alias.json") - if err := os.Link(path, alias); err != nil { - t.Fatal(err) - } - if _, err := parseRunArgs(append(append([]string{}, args...), "--out", path, "--execution-report", alias)); err == nil { - t.Fatal("accepted two names for the same output file") - } - if cfg, err := parseRunArgs(append(append([]string{}, args...), "--execution-report", "-")); err != nil || cfg.executionReportPath != "-" { - t.Fatalf("ordinary dash path was rejected: cfg=%+v err=%v", cfg, err) - } - if _, err := parseRenderArgs(append(append([]string{}, args...), "--execution-report", path)); err == nil { - t.Fatal("render accepted run-only report flag") - } - code, stdout, stderr := runCLICommand(t, runCommand, []string{"--config", filepath.Join(root, "missing.yml"), "--prompt", "p", "--execution-report", path}) - data, err := os.ReadFile(path) - if err != nil { - t.Fatal(err) - } - if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "source.missing") || string(data) != "old report" { - t.Fatalf("pre-invocation failure changed report: code=%d stdout=%q stderr=%q report=%q", code, stdout, stderr, data) - } -} - -func intPointer(value int) *int { return &value } - -func pointerValue[T any](value *T) T { - if value == nil { - var zero T - return zero - } - return *value -} - -func TestPromptkitPromptInspectionSelectsIDsAndContracts(t *testing.T) { - fixtureRoot := definitionFixtureRoot(t) - configPath := writeDefinitionFixtureConfig(t, fixtureRoot) - tests := []struct { - promptID string - format string - mode string - }{ - {promptID: "compat.complete", format: "json", mode: "json_schema"}, - {promptID: "compat.complete-text", format: "text", mode: "basic"}, - {promptID: "compat.json", format: "json", mode: "json"}, - {promptID: "compat.none", format: "text", mode: "none"}, - } - - for _, tc := range tests { - t.Run(tc.promptID, func(t *testing.T) { - code, stdout, stderr := runCLICommand(t, inspectCommand, []string{ - "prompt", - "--config", configPath, - "--prompt", tc.promptID, - "--format", "json", - }) - if code != ExitOK { - t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr) - } - var inspection renderformat.PromptInspection - if err := json.Unmarshal([]byte(stdout), &inspection); err != nil { - t.Fatalf("decode prompt inspection: %v\nbody=%s", err, stdout) - } - if inspection.PromptID != tc.promptID || inspection.OutputContract.Format != tc.format || inspection.OutputContract.ValidationMode != tc.mode { - t.Fatalf("unexpected prompt inspection: %+v", inspection) - } - }) - } -} - -func TestPromptkitProfileInspectionResolvesSupportedTargets(t *testing.T) { - const secret = "sentinel-profile-secret" - t.Setenv("FIXTURE_PROFILE_API_KEY", secret) - - fixtureRoot := definitionFixtureRoot(t) - configPath := writeDefinitionFixtureConfig(t, fixtureRoot) - tests := []struct { - name string - profileID string - wantBackend string - wantModel string - wantAPIKeyEnv string - }{ - {name: "inherited custom backend", profileID: "custom-derived", wantBackend: "fixture-custom", wantModel: "fixture-derived-model", wantAPIKeyEnv: "FIXTURE_PROFILE_API_KEY"}, - {name: "endpoint only", profileID: "endpoint-only", wantModel: "fixture-endpoint-model"}, - {name: "built in", profileID: "auto", wantBackend: "openrouter", wantModel: "openrouter/auto"}, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - args := []string{"profile", "--config", configPath, "--profile", tc.profileID, "--format", "json"} - code, stdout, stderr := runCLICommand(t, inspectCommand, args) - if code != ExitOK { - t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr) - } - if strings.Contains(stdout, secret) || strings.Contains(stderr, secret) { - t.Fatalf("inspection exposed environment secret: stdout=%q stderr=%q", stdout, stderr) - } - var inspection renderformat.ProfileInspection - if err := json.Unmarshal([]byte(stdout), &inspection); err != nil { - t.Fatalf("decode profile inspection: %v\nbody=%s", err, stdout) - } - params := inspection.ProfileModelParams - if inspection.ProfileID != tc.profileID || pointerValue(params.BackendID) != tc.wantBackend || pointerValue(params.Model) != tc.wantModel || pointerValue(params.APIKeyEnv) != tc.wantAPIKeyEnv { - t.Fatalf("unexpected effective profile: %+v", inspection) - } - if tc.profileID == "custom-derived" { - if pointerValue(params.ServiceTier) != "flex" || pointerValue(params.ReasoningEffort) != "high" || pointerValue(params.TimeoutSeconds) != 45 || len(pointerValue(params.ExtraParams)) == 0 { - t.Fatalf("inherited profile controls were not resolved: %+v", params) - } - } - }) - } - - configWithoutPromptSource := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - directory: %q\n - file: %q\n", - filepath.Join(fixtureRoot, "profiles"), - filepath.Join(fixtureRoot, "framework", "backends.yml"), - )) - code, _, stderr := runCLICommand(t, inspectCommand, []string{ - "profile", - "--config", configWithoutPromptSource, - "--profile", "custom-derived", - }) - if code != ExitOK { - t.Fatalf("profile inspection unexpectedly required a prompt source: %q", stderr) - } -} - -func TestProfileInspectionUsesCombinedSourcesAndValidatesUnusedPrompts(t *testing.T) { - lib := newCLITestLibrary(t) - combined := `backends: - - id: fixture - endpoint: http://127.0.0.1:1/v1 -profiles: - - id: inspectable - backend: fixture - model: fixture-model -prompts: - - id: unused - messages: - - role: user - content: "valid unused prompt" - output: - format: text - validation_mode: none -` - frameworkPath := writeFrameworkFile(t, lib.rootDir, "combined.yml", combined) - configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n", frameworkPath)) - code, stdout, stderr := runCLICommand(t, inspectCommand, []string{ - "profile", "--config", configPath, "--profile", "inspectable", "--format", "json", - }) - if code != ExitOK { - t.Fatalf("combined-source profile inspection failed: code=%d out=%q err=%q", code, stdout, stderr) - } - var inspection renderformat.ProfileInspection - if err := json.Unmarshal([]byte(stdout), &inspection); err != nil { - t.Fatalf("decode combined-source profile inspection: %v", err) - } - if pointerValue(inspection.ProfileModelParams.BackendID) != "fixture" || pointerValue(inspection.ProfileModelParams.Model) != "fixture-model" { - t.Fatalf("combined source did not provide the selected profile: %+v", inspection.ProfileModelParams) - } - - invalidCombined := strings.Replace(combined, `content: "valid unused prompt"`, `content: "{{"`, 1) - invalidFrameworkPath := writeFrameworkFile(t, lib.rootDir, "combined-invalid.yml", invalidCombined) - invalidConfigPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n", invalidFrameworkPath)) - code, stdout, stderr = runCLICommand(t, inspectCommand, []string{ - "profile", "--config", invalidConfigPath, "--profile", "inspectable", - }) - if code != ExitRuntimeError || stdout != "" || stderr == "" { - t.Fatalf("invalid unused prompt did not fail construction: code=%d out=%q err=%q", code, stdout, stderr) - } -} - -func TestMaintainedProfilesResolveAcrossSelectionAndSourcePrecedence(t *testing.T) { - promptDir := t.TempDir() - profileDir := t.TempDir() - writePromptDefinition(t, promptDir, "maintained.yaml", `prompts: - - id: maintained - default_profile: gpt-sol-latest - messages: - - role: user - content: Hello - output: - format: text - validation_mode: none -`) - configPath := writeAppConfigFile(t, sourceSelectionsYAML(promptDir, profileDir, "")) - - for _, profileID := range []string{"gpt-sol-latest", "auto"} { - t.Run(profileID+" inspection", func(t *testing.T) { - code, stdout, stderr := runCLICommand(t, inspectCommand, []string{"profile", "--config", configPath, "--profile", profileID, "--format", "json"}) - if code != ExitOK { - t.Fatalf("inspect %s: code=%d stderr=%q", profileID, code, stderr) - } - var inspection renderformat.ProfileInspection - if err := json.Unmarshal([]byte(stdout), &inspection); err != nil { - t.Fatal(err) - } - if inspection.ProfileID != profileID || pointerValue(inspection.ProfileModelParams.BackendID) != "openrouter" || pointerValue(inspection.ProfileModelParams.Model) == "" { - t.Fatalf("maintained profile did not resolve: %+v", inspection) - } - if profileID == "auto" && pointerValue(inspection.ProfileModelParams.Model) != "openrouter/auto" { - t.Fatalf("unexpected auto target: %+v", inspection.ProfileModelParams) - } - }) - } - - for _, tc := range []struct { - name string - selection []string - wantID string - }{ - {name: "prompt default", wantID: "gpt-sol-latest"}, - {name: "explicit selection", selection: []string{"--profile", "auto"}, wantID: "auto"}, - } { - t.Run(tc.name, func(t *testing.T) { - args := append([]string{"--config", configPath, "--prompt", "maintained", "--format", "json"}, tc.selection...) - code, stdout, stderr := runCLICommand(t, renderCommand, args) - if code != ExitOK { - t.Fatalf("render: code=%d stderr=%q", code, stderr) - } - var rendered struct { - SelectedProfileID string `json:"selected_profile_id"` - SelectedBackendID string `json:"selected_backend_id"` - } - if err := json.Unmarshal([]byte(stdout), &rendered); err != nil { - t.Fatal(err) - } - if rendered.SelectedProfileID != tc.wantID || rendered.SelectedBackendID != "openrouter" { - t.Fatalf("unexpected selection: %+v", rendered) - } - }) - } - - writePromptDefinition(t, profileDir, "derived.yaml", "profiles:\n - id: derived-maintained\n extends: {id: gpt-sol-latest}\n model: fixture-inherited-model\n") - writePromptDefinition(t, profileDir, "replacement.yaml", "profiles:\n - id: auto\n backend: openrouter\n model: fixture-override-model\n") - for _, tc := range []struct{ id, model string }{ - {id: "derived-maintained", model: "fixture-inherited-model"}, - {id: "auto", model: "fixture-override-model"}, - } { - t.Run(tc.id+" source", func(t *testing.T) { - code, stdout, stderr := runCLICommand(t, inspectCommand, []string{"profile", "--config", configPath, "--profile", tc.id, "--format", "json"}) - if code != ExitOK { - t.Fatalf("inspect: code=%d stderr=%q", code, stderr) - } - var inspection renderformat.ProfileInspection - if err := json.Unmarshal([]byte(stdout), &inspection); err != nil { - t.Fatal(err) - } - if pointerValue(inspection.ProfileModelParams.BackendID) != "openrouter" || pointerValue(inspection.ProfileModelParams.Model) != tc.model { - t.Fatalf("unexpected effective target: %+v", inspection) - } - }) - } -} - -func TestProfileInspectionHonorsDirectoryPrecedenceOutputAndFailures(t *testing.T) { - fixtureRoot := definitionFixtureRoot(t) - configPath := writeDefinitionFixtureConfig(t, fixtureRoot) - overrideDir := t.TempDir() - writeProfileFile(t, overrideDir, "custom-derived", "http://127.0.0.1:9000/v1", "override-model") - outPath := filepath.Join(t.TempDir(), "inspection.json") - - code, stdout, stderr := runCLICommand(t, inspectCommand, []string{ - "profile", - "--config", configPath, - "--promptkit-source", filepath.Join(fixtureRoot, "prompts"), - "--promptkit-source", overrideDir, - "--promptkit-source", filepath.Join(fixtureRoot, "framework", "backends.yml"), - "--profile", "custom-derived", - "--format", "json", - "--out", outPath, - }) - if code != ExitOK || stdout != "" { - t.Fatalf("expected file output, got code=%d stdout=%q stderr=%q", code, stdout, stderr) - } - output, err := os.ReadFile(outPath) - if err != nil { - t.Fatalf("read profile inspection output: %v", err) - } - if !strings.Contains(string(output), `"model": "override-model"`) || strings.Contains(string(output), `"backend_id"`) { - t.Fatalf("profile source override was not used: %s", output) - } - - for _, profileID := range []string{"missing", "invalid"} { - t.Run(profileID, func(t *testing.T) { - args := []string{"profile", "--config", configPath, "--profile", profileID} - if profileID == "invalid" { - profileDir := t.TempDir() - writePromptDefinition(t, profileDir, "invalid.yaml", "profiles:\n - id: invalid\n endpoint: not-a-url\n model: fixture\n") - args = append(args, - "--promptkit-source", filepath.Join(fixtureRoot, "prompts"), - "--promptkit-source", profileDir, - "--promptkit-source", filepath.Join(fixtureRoot, "framework", "backends.yml"), - ) - } - code, stdout, stderr := runCLICommand(t, inspectCommand, args) - if code != ExitRuntimeError || stdout != "" || (profileID == "invalid" && !strings.Contains(stderr, "endpoint must use http or https")) { - t.Fatalf("expected inspection failure without partial output, got code=%d stdout=%q stderr=%q", code, stdout, stderr) - } - }) - } -} - -func TestConfiguredBackendValidationComesFromPromptkit(t *testing.T) { - lib := newCLITestLibrary(t) - - for _, tc := range []struct { - name string - backend string - }{ - { - name: "invalid endpoint", - backend: `backends: - - id: local-gpu - endpoint: not-a-url -`, - }, - { - name: "invalid capacity relationship", - backend: `backends: - - id: local-gpu - endpoint: http://localhost:11434/v1 - queue_capacity: 0 -`, - }, - { - name: "reserved extra parameter", - backend: `backends: - - id: local-gpu - endpoint: http://localhost:11434/v1 - extra_params: - model: forbidden -`, - }, - } { - t.Run(tc.name, func(t *testing.T) { - configPath := writeCLIFrameworkConfig(t, lib, tc.backend, "") - cfg, err := parseRenderArgs([]string{"--config", configPath, "--prompt", "custom"}) - if err != nil { - t.Fatalf("expected application config decoding to succeed, got %v", err) - } - _, err = newEngine(cfg.runConfig.engineSettings()) - if !errors.Is(err, promptkit.ErrInvalidConfig) { - t.Fatalf("expected Promptkit ErrInvalidConfig, got %v", err) - } - if !strings.Contains(err.Error(), "engine initialization from application configuration") { - t.Fatalf("expected application context, got %v", err) - } - }) - } - - t.Run("built-in backend overlay", func(t *testing.T) { - configPath := writeCLIFrameworkConfig(t, lib, `backends: - - id: openrouter - endpoint: http://localhost:11434/v1 -`, "") - code, stdout, stderr := runCLICommand(t, inspectCommand, []string{ - "profile", "--config", configPath, - "--profile", "auto", "--format", "json", - }) - if code != ExitOK { - t.Fatalf("built-in backend overlay failed: code=%d out=%q err=%q", code, stdout, stderr) - } - var inspection renderformat.ProfileInspection - if err := json.Unmarshal([]byte(stdout), &inspection); err != nil { - t.Fatalf("decode built-in profile inspection: %v", err) - } - if pointerValue(inspection.ProfileModelParams.BackendID) != "openrouter" || inspection.ProfileModelParams.Endpoint != nil { - t.Fatalf("profile inspection unexpectedly completed backend settings: %+v", inspection.ProfileModelParams) - } - }) -} - -func TestInheritedBackendParentStillReceivesPromptkitValidation(t *testing.T) { - lib := newCLITestLibrary(t) - configPath := writeCLIFrameworkConfig(t, lib, `backends: - - id: base - endpoint: not-a-url - - id: child - extends: {id: base} - endpoint: http://localhost:11434/v1 -`, "") - cfg, err := parseRenderArgs([]string{"--config", configPath, "--prompt", "custom"}) - if err != nil { - t.Fatalf("application loading rejected the configured parent before Promptkit: %v", err) - } - if _, err := newEngine(cfg.runConfig.engineSettings()); !errors.Is(err, promptkit.ErrInvalidConfig) { - t.Fatalf("invalid parent backend was not validated by Promptkit: %v", err) - } -} - -func TestRenderCommandExplicitTextFormatWorks(t *testing.T) { - lib := newCLITestLibrary(t) - inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript") - - writePromptFile(t, lib.promptDir, "prompt.render", "local-default") - writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model") - - code, stdout, stderr := runCLICommand(t, renderCommand, []string{ - "--config", lib.configPath, - "--promptkit-source", lib.promptDir, - "--promptkit-source", lib.profileDir, - "--prompt", "prompt.render", - "--input", "transcript=" + inputPath, - "--format", "text", - }) - if code != ExitOK { - t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr) - } - if !strings.Contains(stdout, "prompt: prompt.render") { - t.Fatalf("expected text output for explicit --format text, got %q", stdout) - } -} - -func TestRenderCommandExplicitJSONFormatOutputsValidJSON(t *testing.T) { - lib := newCLITestLibrary(t) - inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript") - - writePromptFile(t, lib.promptDir, "prompt.render", "local-default") - writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model") - - code, stdout, stderr := runCLICommand(t, renderCommand, []string{ - "--config", lib.configPath, - "--promptkit-source", lib.promptDir, - "--promptkit-source", lib.profileDir, - "--prompt", "prompt.render", - "--input", "transcript=" + inputPath, - "--format", "json", - }) - if code != ExitOK { - t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr) - } - - var payload map[string]any - if err := json.Unmarshal([]byte(stdout), &payload); err != nil { - t.Fatalf("expected valid json output, got %v\nbody=%s", err, stdout) - } - if payload["prompt_id"] != "prompt.render" { - t.Fatalf("expected prompt_id, got %#v", payload["prompt_id"]) - } - if payload["selected_profile_id"] != "local-default" { - t.Fatalf("expected selected_profile_id, got %#v", payload["selected_profile_id"]) - } - if _, ok := payload["messages"]; !ok { - t.Fatalf("expected messages in render json output, got %#v", payload) - } -} - -func TestRenderCommandUnknownFormatFailsClearly(t *testing.T) { - lib := newCLITestLibrary(t) - var stdout bytes.Buffer - var stderr bytes.Buffer - code := renderCommand([]string{ - "--config", lib.configPath, - "--promptkit-source", lib.promptDir, - "--promptkit-source", lib.profileDir, - "--prompt", "p", - "--input", "transcript=./x.md", - "--format", "yaml", - }, &stdout, &stderr) - if code != ExitRuntimeError { - t.Fatalf("expected ExitRuntimeError, got %d", code) - } - if !strings.Contains(stderr.String(), "render parse error") || !strings.Contains(stderr.String(), "unknown prepared run format") { - t.Fatalf("expected clear unknown-format parse error, got %q", stderr.String()) - } -} - -func TestRenderCommandOutWritesToFile(t *testing.T) { - lib := newCLITestLibrary(t) - inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript") - outPath := filepath.Join(lib.rootDir, "render.txt") - - writePromptFile(t, lib.promptDir, "prompt.render", "local-default") - writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model") - - code, stdout, stderr := runCLICommand(t, renderCommand, []string{ - "--config", lib.configPath, - "--promptkit-source", lib.promptDir, - "--promptkit-source", lib.profileDir, - "--prompt", "prompt.render", - "--input", "transcript=" + inputPath, - "--out", outPath, - }) - if code != ExitOK { - t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr) - } - if stdout != "" { - t.Fatalf("expected empty stdout when --out is set, got %q", stdout) - } - out, err := os.ReadFile(outPath) - if err != nil { - t.Fatalf("failed reading render output file: %v", err) - } - if !strings.Contains(string(out), "prompt: prompt.render") { - t.Fatalf("expected render output in file, got %q", string(out)) - } -} - -func TestRenderCommandPromptDefaultProfileWorksThroughCLIPath(t *testing.T) { - lib := newCLITestLibrary(t) - inputPath := lib.writeInputFile(t, "transcript.md", "hello") - - writePromptFile(t, lib.promptDir, "prompt.default", "local-default") - writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "default-model") - - code, stdout, stderr := runCLICommand(t, renderCommand, []string{ - "--config", lib.configPath, - "--promptkit-source", lib.promptDir, - "--promptkit-source", lib.profileDir, - "--prompt", "prompt.default", - "--input", "transcript=" + inputPath, - }) - - if code != ExitOK { - t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr) - } - out := stdout - if !strings.Contains(out, "selected_profile_id: local-default") { - t.Fatalf("expected prompt default profile in output, got %q", out) - } - if !strings.Contains(out, "model: default-model") { - t.Fatalf("expected model from default profile in output, got %q", out) - } -} - -func TestRenderCommandUsesBuiltInProfileWithoutCustomProfileSource(t *testing.T) { - t.Setenv("OPENROUTER_API_KEY", "test-key") - lib := newCLITestLibrary(t) - inputPath := lib.writeInputFile(t, "transcript.md", "hello") - - writePromptFile(t, lib.promptDir, "prompt.builtin", "auto") - - code, stdout, stderr := runCLICommand(t, renderCommand, []string{ - "--config", lib.configPath, - "--promptkit-source", lib.promptDir, - "--prompt", "prompt.builtin", - "--input", "transcript=" + inputPath, - }) - if code != ExitOK { - t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr) - } - if !strings.Contains(stdout, "selected_profile_id: auto") { - t.Fatalf("expected built-in selected profile, got %q", stdout) - } - if !strings.Contains(stdout, "model: openrouter/auto") { - t.Fatalf("expected built-in model, got %q", stdout) - } -} - -func TestRenderCommandExplicitProfileOverridesPromptDefault(t *testing.T) { - lib := newCLITestLibrary(t) - inputPath := lib.writeInputFile(t, "transcript.md", "hello") - - writePromptFile(t, lib.promptDir, "prompt.default", "local-default") - writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "default-model") - writeProfileFile(t, lib.profileDir, "quality", "http://127.0.0.1:1/v1", "quality-model") - - code, stdout, stderr := runCLICommand(t, renderCommand, []string{ - "--config", lib.configPath, - "--promptkit-source", lib.promptDir, - "--promptkit-source", lib.profileDir, - "--prompt", "prompt.default", - "--profile", "quality", - "--input", "transcript=" + inputPath, - }) - - if code != ExitOK { - t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr) - } - out := stdout - if !strings.Contains(out, "selected_profile_id: quality") { - t.Fatalf("expected explicit profile in output, got %q", out) - } - if !strings.Contains(out, "model: quality-model") { - t.Fatalf("expected model from explicit profile in output, got %q", out) - } -} - -func TestRenderCommandUsesDefinitionInputRules(t *testing.T) { - lib := newCLITestLibrary(t) - writeProfileFile(t, lib.profileDir, "local", "http://127.0.0.1:1/v1", "model") - - writePromptDefinition(t, lib.promptDir, "sole.yaml", `prompts: - - id: sole - default_profile: local - messages: - - role: user - content: "hello" - output: - format: text - validation_mode: none -`) - writePromptDefinition(t, lib.promptDir, "optional.yaml", `prompts: - - id: optional - default_profile: local - inputs: - - name: note - required: false - messages: - - role: user - content: "hello" - output: - format: text - validation_mode: none -`) - writePromptDefinition(t, lib.promptDir, "required.yaml", `prompts: - - id: required - default_profile: local - inputs: - - name: note - required: true - messages: - - role: user - content: "hello" - output: - format: text - validation_mode: none -`) - writePromptDefinition(t, lib.promptDir, "template.yaml", `prompts: - - id: template - default_profile: local - messages: - - role: user - content: '{{input "note"}}' - output: - format: text - validation_mode: none -`) - - baseArgs := []string{"--config", lib.configPath, "--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir} - for _, tc := range []struct { - name string - args []string - wantCode int - wantText string - }{ - {name: "no declared inputs", args: []string{"--prompt", "sole"}, wantCode: ExitOK, wantText: "prompt: sole"}, - {name: "optional input omitted", args: []string{"--prompt", "optional"}, wantCode: ExitOK, wantText: "prompt: optional"}, - {name: "required input omitted", args: []string{"--prompt", "required"}, wantCode: ExitRuntimeError, wantText: "required"}, - {name: "template input omitted", args: []string{"--prompt", "template"}, wantCode: ExitRuntimeError, wantText: "note"}, - } { - t.Run(tc.name, func(t *testing.T) { - code, stdout, stderr := runCLICommand(t, renderCommand, append(append([]string{}, baseArgs...), tc.args...)) - if code != tc.wantCode { - t.Fatalf("expected exit %d, got %d stderr=%q", tc.wantCode, code, stderr) - } - if !strings.Contains(stdout+stderr, tc.wantText) { - t.Fatalf("expected output to contain %q, stdout=%q stderr=%q", tc.wantText, stdout, stderr) - } - }) - } -} - -func TestRunCommandPromptDefaultProfileWorksThroughCLIPath(t *testing.T) { - lib := newCLITestLibrary(t) - inputPath := lib.writeInputFile(t, "transcript.md", "hello") - - ts := newTestLLMServer("default-output", nil) - defer ts.Close() - - writePromptFile(t, lib.promptDir, "prompt.default", "local-default") - writeProfileFile(t, lib.profileDir, "local-default", ts.URL+"/v1", "profile-model") - - code, stdout, stderr := runCLICommand(t, runCommand, []string{ - "--config", lib.configPath, - "--promptkit-source", lib.promptDir, - "--promptkit-source", lib.profileDir, - "--prompt", "prompt.default", - "--input", "transcript=" + inputPath, - }) - - if code != ExitOK { - t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr) - } - if stdout != "default-output" { - t.Fatalf("unexpected stdout output: %q", stdout) - } - if !strings.Contains(stderr, "selected_profile=local-default") { - t.Fatalf("expected selected profile in summary, got %q", stderr) - } -} - -func TestRunCommandExplicitProfileOverridesPromptDefault(t *testing.T) { - lib := newCLITestLibrary(t) - inputPath := lib.writeInputFile(t, "transcript.md", "hello") - - defaultServer := newTestLLMServer("from-default", nil) - defer defaultServer.Close() - overrideServer := newTestLLMServer("from-override", nil) - defer overrideServer.Close() - - writePromptFile(t, lib.promptDir, "prompt.default", "local-default") - writeProfileFile(t, lib.profileDir, "local-default", defaultServer.URL+"/v1", "default-model") - writeProfileFile(t, lib.profileDir, "quality", overrideServer.URL+"/v1", "quality-model") - - code, stdout, stderr := runCLICommand(t, runCommand, []string{ - "--config", lib.configPath, - "--promptkit-source", lib.promptDir, - "--promptkit-source", lib.profileDir, - "--prompt", "prompt.default", - "--profile", "quality", - "--input", "transcript=" + inputPath, - }) - if code != ExitOK { - t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr) - } - if stdout != "from-override" { - t.Fatalf("expected explicit profile output, got %q", stdout) - } - if !strings.Contains(stderr, "selected_profile=quality") { - t.Fatalf("expected selected profile quality, got %q", stderr) - } -} - -func TestRunCommandRuntimeFlagsOverrideSelectedProfileValues(t *testing.T) { - lib := newCLITestLibrary(t) - inputPath := lib.writeInputFile(t, "transcript.md", "hello") - - var baseHits int32 - baseServer := newTestLLMServer("base", &baseHits) - defer baseServer.Close() - - var overrideHits int32 - var observedBody string - overrideServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - atomic.AddInt32(&overrideHits, 1) - body, _ := io.ReadAll(r.Body) - observedBody = string(body) - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"choices":[{"message":{"role":"assistant","content":"override"}}],"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}`)) - })) - defer overrideServer.Close() - - writePromptFile(t, lib.promptDir, "prompt.default", "local-default") - writeProfileFile(t, lib.profileDir, "local-default", baseServer.URL+"/v1", "profile-model") - - code, stdout, stderr := runCLICommand(t, runCommand, []string{ - "--config", lib.configPath, - "--promptkit-source", lib.promptDir, - "--promptkit-source", lib.profileDir, - "--prompt", "prompt.default", - "--input", "transcript=" + inputPath, - "--llm-base-url", overrideServer.URL + "/v1", - "--model", "override-model", - "--temperature", "0.7", - "--max-tokens", "55", - "--top-p", "0.2", - "--timeout", "20s", - }) - if code != ExitOK { - t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr) - } - if atomic.LoadInt32(&baseHits) != 0 { - t.Fatalf("expected base profile endpoint not to be hit, got %d", baseHits) - } - if atomic.LoadInt32(&overrideHits) != 1 { - t.Fatalf("expected override endpoint to be hit once, got %d", overrideHits) - } - if stdout != "override" { - t.Fatalf("unexpected stdout output: %q", stdout) - } - if !strings.Contains(observedBody, `"model":"override-model"`) { - t.Fatalf("expected override model in request body, got %s", observedBody) - } - if !strings.Contains(observedBody, `"temperature":0.7`) || !strings.Contains(observedBody, `"max_tokens":55`) || !strings.Contains(observedBody, `"top_p":0.2`) { - t.Fatalf("expected override generation params in request body, got %s", observedBody) - } -} - -func TestWriteOutputAndSummaryUseSeparateWriters(t *testing.T) { - var stdout bytes.Buffer - var stderr bytes.Buffer - - if err := writeOutput(&stdout, "", []byte("artifact-body")); err != nil { - t.Fatalf("unexpected writeOutput error: %v", err) - } - printSummary(&stderr, &promptkit.RunResult{ - PromptID: "p", - SelectedProfileID: "exec", - ModelName: "m", - Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic}, - RenderedPromptHash: "h", - InputHashes: map[string]string{"in": "x"}, - }) - - if stdout.String() != "artifact-body" { - t.Fatalf("expected artifact output on stdout, got %q", stdout.String()) - } - if !strings.Contains(stderr.String(), "prompt=p ") || strings.Contains(stderr.String(), "prompt_version") { - t.Fatalf("expected summary on stderr, got %q", stderr.String()) - } - if strings.Contains(stderr.String(), "cached_tokens=") || strings.Contains(stderr.String(), "cache_write_tokens=") { - t.Fatalf("expected zero cache usage to be omitted from summary, got %q", stderr.String()) - } - if strings.Contains(stderr.String(), "backend=") { - t.Fatalf("expected endpoint-only backend to be omitted from summary, got %q", stderr.String()) - } -} - -func TestPrintSummaryIncludesCacheUsageWhenPresent(t *testing.T) { - var stderr bytes.Buffer - - printSummary(&stderr, &promptkit.RunResult{ - PromptID: "p", - SelectedProfileID: "exec", - ModelName: "m", - Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic}, - RenderedPromptHash: "h", - InputHashes: map[string]string{"in": "x"}, - Usage: promptkit.TokenUsage{ - PromptTokens: 10, - CompletionTokens: 5, - TotalTokens: 15, - CachedTokens: 0, - CacheWriteTokens: 3, - }, - }) - - summary := stderr.String() - if !strings.Contains(summary, "usage=10/5/15") { - t.Fatalf("expected base usage summary, got %q", summary) - } - if !strings.Contains(summary, "cached_tokens=0 cache_write_tokens=3") { - t.Fatalf("expected cache usage in summary, got %q", summary) - } - if strings.Contains(summary, "backend=") { - t.Fatalf("expected endpoint-only backend to be omitted from summary, got %q", summary) - } -} - -func TestPrintSummaryIncludesBackendWhenPresent(t *testing.T) { - var stderr bytes.Buffer - printSummary(&stderr, &promptkit.RunResult{ - PromptID: "p", - SelectedProfileID: "exec", - SelectedBackendID: "local", - ModelName: "m", - Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic}, - RenderedPromptHash: "h", - }) - if !strings.Contains(stderr.String(), "backend=local") { - t.Fatalf("expected backend in summary, got %q", stderr.String()) - } -} - -func TestRunErrorMessageDoesNotExposeCapacityDetails(t *testing.T) { - got := runErrorMessage(&promptkit.CapacityError{BackendID: "private-backend"}) - if got != "run error: model backend capacity is exhausted" { - t.Fatalf("unexpected capacity diagnostic: %q", got) - } -} - -type cliTestLibrary struct { - rootDir string - promptDir string - profileDir string - configPath string -} - -func newCLITestLibrary(t *testing.T) *cliTestLibrary { - t.Helper() - root := t.TempDir() - lib := &cliTestLibrary{ - rootDir: root, - promptDir: filepath.Join(root, "prompts"), - profileDir: filepath.Join(root, "profiles"), - configPath: writeAppConfigFile(t, "{}\n"), - } - if err := os.MkdirAll(lib.promptDir, 0o755); err != nil { - t.Fatalf("failed to create prompt fixture directory: %v", err) - } - if err := os.MkdirAll(lib.profileDir, 0o755); err != nil { - t.Fatalf("failed to create profile fixture directory: %v", err) - } - return lib -} - -func (l *cliTestLibrary) writeInputFile(t *testing.T, name, body string) string { - t.Helper() - path := filepath.Join(l.rootDir, name) - if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { - t.Fatalf("failed to create input fixture directory: %v", err) - } - if err := os.WriteFile(path, []byte(body), 0o644); err != nil { - t.Fatalf("failed to write input fixture: %v", err) - } - return path -} - -func runCLICommand(t *testing.T, command func([]string, io.Writer, io.Writer) int, args []string) (int, string, string) { - t.Helper() - var stdout bytes.Buffer - var stderr bytes.Buffer - code := command(args, &stdout, &stderr) - return code, stdout.String(), stderr.String() -} - -func definitionFixtureRoot(t *testing.T) string { - t.Helper() - root, err := filepath.Abs(filepath.Join("..", "..", "..", "testdata", "promptkit-v0.9")) - if err != nil { - t.Fatalf("resolve Promptkit v0.9 fixture root: %v", err) - } - if _, err := os.Stat(root); err != nil { - t.Fatalf("stat Promptkit v0.9 fixture root: %v", err) - } - return root -} - -func writeDefinitionFixtureConfig(t *testing.T, fixtureRoot string) string { - t.Helper() - frameworkPath := filepath.Join(fixtureRoot, "framework", "backends.yml") - if _, err := os.Stat(frameworkPath); err != nil { - t.Fatalf("stat framework backend fixture: %v", err) - } - - sources := fmt.Sprintf("promptkit_sources:\n - directory: %q\n - directory: %q\n - file: %q\nschema_dir: %q\n", - filepath.Join(fixtureRoot, "prompts"), - filepath.Join(fixtureRoot, "profiles"), - frameworkPath, - filepath.Join(fixtureRoot, "schemas"), - ) - return writeAppConfigFile(t, sources) -} - -func writePromptFile(t *testing.T, dir, id, defaultProfile string) { - t.Helper() - writePromptFileWithTemplate(t, dir, id, defaultProfile, "Summarize: {{input \"transcript\"}}") -} - -func writePromptDefinition(t *testing.T, dir, name, definition string) { - t.Helper() - if err := os.WriteFile(filepath.Join(dir, name), []byte(definition), 0o644); err != nil { - t.Fatalf("write prompt definition: %v", err) - } -} - -func writePromptFileWithTemplate(t *testing.T, dir, id, defaultProfile, templateContent string) { - t.Helper() - data := fmt.Sprintf(`prompts: - - id: %q - default_profile: %q - inputs: - - name: transcript - required: true - messages: - - role: user - content: %q - output: - format: text - validation_mode: none - repair_attempts: 0 -`, id, defaultProfile, templateContent) - if err := os.WriteFile(filepath.Join(dir, id+".yaml"), []byte(data), 0o644); err != nil { - t.Fatalf("failed to write prompt fixture: %v", err) - } -} - -func writeProfileFile(t *testing.T, dir, id, endpoint, model string) { - t.Helper() - data := fmt.Sprintf("profiles:\n - id: %q\n endpoint: %q\n model: %q\n", id, endpoint, model) - if err := os.WriteFile(filepath.Join(dir, id+".yaml"), []byte(data), 0o644); err != nil { - t.Fatalf("failed to write profile fixture: %v", err) - } -} - -func newTestLLMServer(content string, hitCounter *int32) *httptest.Server { - return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if hitCounter != nil { - atomic.AddInt32(hitCounter, 1) - } - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(fmt.Sprintf(`{"choices":[{"message":{"role":"assistant","content":%q}}],"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}`, content))) - })) -} - -func writeAppConfigFile(t *testing.T, content string) string { - t.Helper() - path := filepath.Join(t.TempDir(), "config.yml") - if err := os.WriteFile(path, []byte(content), 0o644); err != nil { - t.Fatalf("failed to write app config fixture: %v", err) - } - return path -} - -func writeFrameworkFile(t *testing.T, root, name, definition string) string { - t.Helper() - dir := filepath.Join(root, "framework") - if err := os.MkdirAll(dir, 0o755); err != nil { - t.Fatalf("create framework fixture directory: %v", err) - } - path := filepath.Join(dir, name) - if err := os.WriteFile(path, []byte(definition), 0o644); err != nil { - t.Fatalf("write framework fixture: %v", err) - } - return path -} - -func sourceSelectionsYAML(promptDir, profileDir, frameworkFile string) string { - selectors := fmt.Sprintf("promptkit_sources:\n - directory: %q\n - directory: %q\n", promptDir, profileDir) - if frameworkFile != "" { - selectors += fmt.Sprintf(" - file: %q\n", frameworkFile) - } - return selectors -} - -func writeCLIFrameworkConfig(t *testing.T, lib *cliTestLibrary, frameworkDefinition, applicationFields string) string { - t.Helper() - frameworkPath := writeFrameworkFile(t, lib.rootDir, "definitions.yml", frameworkDefinition) - return writeAppConfigFile(t, sourceSelectionsYAML(lib.promptDir, lib.profileDir, frameworkPath)+applicationFields) -} diff --git a/internal/adapter/dependency_test.go b/internal/adapter/dependency_test.go index b576dd2..88ef1d3 100644 --- a/internal/adapter/dependency_test.go +++ b/internal/adapter/dependency_test.go @@ -153,7 +153,7 @@ import ( _ "gitea.maximumdirect.net/eric/scriptorium/internal/adapter/http" _ "gitea.maximumdirect.net/eric/scriptorium/internal/config" _ "gitea.maximumdirect.net/eric/scriptorium/internal/defaults" - _ "gitea.maximumdirect.net/eric/scriptorium/internal/format" + _ "gitea.maximumdirect.net/eric/scriptorium/internal/report" ) ` if err := os.WriteFile(sourcePath, []byte(source), 0o644); err != nil { diff --git a/internal/format/inspection.go b/internal/format/inspection.go deleted file mode 100644 index 42535be..0000000 --- a/internal/format/inspection.go +++ /dev/null @@ -1,179 +0,0 @@ -package format - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "strconv" - - "go.maximumdirect.net/promptkit" -) - -type PromptInspection struct { - PromptID string `json:"prompt_id"` - PromptHash string `json:"prompt_hash"` - DefaultProfileID *string `json:"default_profile_id,omitempty"` - Inputs *[]PromptInspectionInput `json:"inputs,omitempty"` - OutputContract OutputContract `json:"output_contract"` -} - -type PromptInspectionInput struct { - Name string `json:"name"` - Required *bool `json:"required,omitempty"` - ContentType *string `json:"content_type,omitempty"` - Description *string `json:"description,omitempty"` -} - -type OutputContract struct { - Format string `json:"format"` - ValidationMode string `json:"validation_mode"` - SchemaPath *string `json:"schema_path,omitempty"` - RepairAttempts *int `json:"repair_attempts,omitempty"` -} - -type ProfileInspection struct { - ProfileID string `json:"profile_id"` - ProfileModelParams ProfileModelParams `json:"profile_model_params"` - APIKeyRequired *bool `json:"api_key_required,omitempty"` -} - -type ProfileModelParams struct { - BackendID *string `json:"backend_id,omitempty"` - Endpoint *string `json:"endpoint,omitempty"` - Model *string `json:"model,omitempty"` - Temperature *float64 `json:"temperature,omitempty"` - MaxTokens *int `json:"max_tokens,omitempty"` - TopP *float64 `json:"top_p,omitempty"` - TimeoutSeconds *int `json:"timeout_seconds,omitempty"` - ServiceTier *string `json:"service_tier,omitempty"` - ReasoningEffort *string `json:"reasoning_effort,omitempty"` - APIKeyEnv *string `json:"api_key_env,omitempty"` - ExtraParams *map[string]any `json:"extra_params,omitempty"` -} - -func FormatPromptInspection(value *promptkit.PromptInspection, outputFormat OutputFormat) ([]byte, error) { - if value == nil { - return nil, errors.New("prompt inspection is nil") - } - dto := PromptInspection{ - PromptID: value.PromptID, PromptHash: value.PromptHash, DefaultProfileID: value.DefaultProfileID, - OutputContract: OutputContract{Format: string(value.OutputContract.Format), ValidationMode: string(value.OutputContract.ValidationMode), SchemaPath: value.OutputContract.SchemaPath, RepairAttempts: value.OutputContract.RepairAttempts}, - } - if value.Inputs != nil { - inputs := make([]PromptInspectionInput, len(*value.Inputs)) - for i, input := range *value.Inputs { - inputs[i] = PromptInspectionInput{Name: input.Name, Required: input.Required, ContentType: input.ContentType, Description: input.Description} - } - dto.Inputs = &inputs - } - switch outputFormat { - case OutputFormatJSON: - data, err := json.MarshalIndent(dto, "", " ") - if err != nil { - return nil, err - } - return append(data, '\n'), nil - case OutputFormatText: - var b bytes.Buffer - fmt.Fprintf(&b, "prompt_id: %s\nprompt_hash: %s\n", dto.PromptID, dto.PromptHash) - if dto.DefaultProfileID != nil { - fmt.Fprintf(&b, "default_profile_id: %s\n", *dto.DefaultProfileID) - } - if dto.Inputs != nil { - if len(*dto.Inputs) == 0 { - fmt.Fprintln(&b, "inputs: []") - } else { - fmt.Fprintln(&b, "inputs:") - for _, input := range *dto.Inputs { - fmt.Fprintf(&b, " - name: %s\n", input.Name) - if input.Required != nil { - fmt.Fprintf(&b, " required: %t\n", *input.Required) - } - if input.ContentType != nil { - fmt.Fprintf(&b, " content_type: %s\n", *input.ContentType) - } - if input.Description != nil { - fmt.Fprintf(&b, " description: %s\n", strconv.Quote(*input.Description)) - } - } - } - } - fmt.Fprintf(&b, "output_contract:\n format: %s\n validation_mode: %s\n", dto.OutputContract.Format, dto.OutputContract.ValidationMode) - if dto.OutputContract.SchemaPath != nil { - fmt.Fprintf(&b, " schema_path: %s\n", *dto.OutputContract.SchemaPath) - } - if dto.OutputContract.RepairAttempts != nil { - fmt.Fprintf(&b, " repair_attempts: %d\n", *dto.OutputContract.RepairAttempts) - } - return b.Bytes(), nil - default: - return nil, fmt.Errorf("%w: %q", ErrUnknownPreparedRunFormat, outputFormat) - } -} - -func FormatProfileInspection(value *promptkit.ProfileInspection, outputFormat OutputFormat) ([]byte, error) { - if value == nil { - return nil, errors.New("profile inspection is nil") - } - dto := ProfileInspection{ProfileID: value.ProfileID, APIKeyRequired: value.APIKeyRequired, ProfileModelParams: ProfileModelParams{ - BackendID: value.Backend, Endpoint: value.Endpoint, Model: value.Model, Temperature: value.Temperature, - MaxTokens: value.MaxTokens, TopP: value.TopP, TimeoutSeconds: value.TimeoutSeconds, ServiceTier: value.ServiceTier, - ReasoningEffort: value.ReasoningEffort, APIKeyEnv: value.APIKeyEnv, ExtraParams: value.ExtraParams, - }} - switch outputFormat { - case OutputFormatJSON: - data, err := json.MarshalIndent(dto, "", " ") - if err != nil { - return nil, err - } - return append(data, '\n'), nil - case OutputFormatText: - var b bytes.Buffer - fmt.Fprintf(&b, "profile_id: %s\nprofile_model_params:\n", dto.ProfileID) - p := dto.ProfileModelParams - if p.BackendID != nil { - fmt.Fprintf(&b, " backend_id: %s\n", *p.BackendID) - } - if p.Endpoint != nil { - fmt.Fprintf(&b, " endpoint: %s\n", *p.Endpoint) - } - if p.Model != nil { - fmt.Fprintf(&b, " model: %s\n", *p.Model) - } - if p.Temperature != nil { - fmt.Fprintf(&b, " temperature: %g\n", *p.Temperature) - } - if p.MaxTokens != nil { - fmt.Fprintf(&b, " max_tokens: %d\n", *p.MaxTokens) - } - if p.TopP != nil { - fmt.Fprintf(&b, " top_p: %g\n", *p.TopP) - } - if p.TimeoutSeconds != nil { - fmt.Fprintf(&b, " timeout_seconds: %d\n", *p.TimeoutSeconds) - } - if p.ServiceTier != nil { - fmt.Fprintf(&b, " service_tier: %s\n", *p.ServiceTier) - } - if p.ReasoningEffort != nil { - fmt.Fprintf(&b, " reasoning_effort: %s\n", *p.ReasoningEffort) - } - if p.APIKeyEnv != nil { - fmt.Fprintf(&b, " api_key_env: %s\n", *p.APIKeyEnv) - } - if p.ExtraParams != nil { - paramsJSON, err := json.Marshal(p.ExtraParams) - if err != nil { - return nil, err - } - fmt.Fprintf(&b, " extra_params: %s\n", paramsJSON) - } - if dto.APIKeyRequired != nil { - fmt.Fprintf(&b, "api_key_required: %t\n", *dto.APIKeyRequired) - } - return b.Bytes(), nil - default: - return nil, fmt.Errorf("%w: %q", ErrUnknownPreparedRunFormat, outputFormat) - } -} diff --git a/internal/format/inspection_test.go b/internal/format/inspection_test.go deleted file mode 100644 index 72a4e37..0000000 --- a/internal/format/inspection_test.go +++ /dev/null @@ -1,95 +0,0 @@ -package format - -import ( - "encoding/json" - "strings" - "testing" - - "go.maximumdirect.net/promptkit" -) - -func ptr[T any](value T) *T { return &value } - -func TestPromptInspectionOptionalFieldsAndDisclosure(t *testing.T) { - inputs := []promptkit.PromptInputInspection{{Name: "first", Required: ptr(false), ContentType: ptr(""), Description: ptr("")}, {Name: "second"}} - value := &promptkit.PromptInspection{PromptID: "support/p", PromptHash: "hash", DefaultProfileID: ptr(""), Inputs: &inputs, - Messages: []promptkit.PromptMessageInspection{{Role: "user", Content: "private message"}}, Description: ptr("private description"), SessionID: ptr("private session"), - OutputContract: promptkit.PromptOutputInspection{Format: "text", ValidationMode: promptkit.ValidationNone, SchemaPath: ptr(""), RepairAttempts: ptr(0)}} - for _, format := range []OutputFormat{OutputFormatText, OutputFormatJSON} { - output, err := FormatPromptInspection(value, format) - if err != nil { - t.Fatal(err) - } - s := string(output) - for _, forbidden := range []string{"private message", "private description", "private session", "prompt_version"} { - if strings.Contains(s, forbidden) { - t.Fatalf("inspection exposed %q: %s", forbidden, s) - } - } - if strings.Index(s, "first") > strings.Index(s, "second") { - t.Fatalf("input order changed: %s", s) - } - if !strings.Contains(s, "default_profile_id") || !strings.Contains(s, "repair_attempts") || !strings.Contains(s, "required") { - t.Fatalf("present empty values missing: %s", s) - } - } - value.DefaultProfileID, value.Inputs = nil, nil - value.OutputContract.SchemaPath, value.OutputContract.RepairAttempts = nil, nil - for _, format := range []OutputFormat{OutputFormatText, OutputFormatJSON} { - output, err := FormatPromptInspection(value, format) - if err != nil { - t.Fatal(err) - } - for _, absent := range []string{"default_profile_id", "inputs", "schema_path", "repair_attempts"} { - if strings.Contains(string(output), absent) { - t.Fatalf("absent %s rendered: %s", absent, output) - } - } - } - empty := []promptkit.PromptInputInspection{} - value.Inputs = &empty - output, err := FormatPromptInspection(value, OutputFormatJSON) - if err != nil || !strings.Contains(string(output), `"inputs": []`) { - t.Fatalf("empty inputs lost: %s %v", output, err) - } - if _, err := FormatPromptInspection(nil, OutputFormatText); err == nil { - t.Fatal("nil prompt accepted") - } -} - -func TestProfileInspectionPreservesOnlyPresentValues(t *testing.T) { - params := map[string]any{} - value := &promptkit.ProfileInspection{ProfileID: "support/fast", Backend: ptr("backend"), Model: ptr(""), Temperature: ptr(0.0), MaxTokens: ptr(0), APIKeyRequired: ptr(false), ExtraParams: ¶ms} - for _, format := range []OutputFormat{OutputFormatText, OutputFormatJSON} { - output, err := FormatProfileInspection(value, format) - if err != nil { - t.Fatal(err) - } - s := string(output) - for _, key := range []string{"profile_model_params", "backend_id", "model", "temperature", "max_tokens", "api_key_required", "extra_params"} { - if !strings.Contains(s, key) { - t.Fatalf("present %s missing: %s", key, s) - } - } - for _, key := range []string{"effective_model_params", "endpoint", "api_key_env", "top_p"} { - if strings.Contains(s, key) { - t.Fatalf("absent %s appeared: %s", key, s) - } - } - } - output, err := FormatProfileInspection(value, OutputFormatJSON) - if err != nil { - t.Fatal(err) - } - var decoded map[string]any - if err := json.Unmarshal(output, &decoded); err != nil { - t.Fatal(err) - } - p := decoded["profile_model_params"].(map[string]any) - if p["backend_id"] != "backend" || p["model"] != "" || p["temperature"] != float64(0) || decoded["api_key_required"] != false || len(p["extra_params"].(map[string]any)) != 0 { - t.Fatalf("wrong values: %#v", decoded) - } - if _, err := FormatProfileInspection(nil, OutputFormatText); err == nil { - t.Fatal("nil profile accepted") - } -} diff --git a/internal/format/prepared_run.go b/internal/format/prepared_run.go deleted file mode 100644 index 960f4c0..0000000 --- a/internal/format/prepared_run.go +++ /dev/null @@ -1,210 +0,0 @@ -// Package format formats already-prepared public data for adapters. -package format - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "sort" - "strings" - - "go.maximumdirect.net/promptkit" -) - -var ErrUnknownPreparedRunFormat = errors.New("unknown prepared run format") - -// OutputFormat selects an application-owned textual or JSON representation. -type OutputFormat string - -const ( - OutputFormatText OutputFormat = "text" - OutputFormatJSON OutputFormat = "json" - - DefaultOutputFormat OutputFormat = OutputFormatText -) - -// PreparedRunOutputFormat is the output format for prepared render data. -type PreparedRunOutputFormat = OutputFormat - -const ( - PreparedRunFormatText = OutputFormatText - PreparedRunFormatJSON = OutputFormatJSON - - DefaultPreparedRunOutputFormat = DefaultOutputFormat -) - -// PreparedRunFormatter serializes a prepared run without performing use case work. -type PreparedRunFormatter interface { - Format(prepared *promptkit.PreparedRun) ([]byte, error) -} - -// ParsePreparedRunOutputFormat parses a format name. -func ParsePreparedRunOutputFormat(raw string) (PreparedRunOutputFormat, error) { - return ParseOutputFormat(raw) -} - -// ParseOutputFormat parses the shared inspection and prepared-run format names. -func ParseOutputFormat(raw string) (OutputFormat, error) { - switch OutputFormat(strings.ToLower(strings.TrimSpace(raw))) { - case "": - return DefaultOutputFormat, nil - case OutputFormatText: - return OutputFormatText, nil - case OutputFormatJSON: - return OutputFormatJSON, nil - default: - return "", fmt.Errorf("%w: %q (supported: %s, %s)", ErrUnknownPreparedRunFormat, raw, OutputFormatText, OutputFormatJSON) - } -} - -// FormatterForPreparedRun returns a formatter strategy for the chosen format. -func FormatterForPreparedRun(outputFormat PreparedRunOutputFormat) (PreparedRunFormatter, error) { - switch outputFormat { - case PreparedRunFormatText: - return textPreparedRunFormatter{}, nil - case PreparedRunFormatJSON: - return jsonPreparedRunFormatter{}, nil - default: - return nil, fmt.Errorf("%w: %q", ErrUnknownPreparedRunFormat, outputFormat) - } -} - -// FormatPreparedRun formats a prepared run using the selected format. -func FormatPreparedRun(prepared *promptkit.PreparedRun, outputFormat PreparedRunOutputFormat) ([]byte, error) { - formatter, err := FormatterForPreparedRun(outputFormat) - if err != nil { - return nil, err - } - return formatter.Format(prepared) -} - -// FormatPreparedRunByName parses a format name and formats a prepared run. -func FormatPreparedRunByName(prepared *promptkit.PreparedRun, rawFormat string) ([]byte, error) { - outputFormat, err := ParsePreparedRunOutputFormat(rawFormat) - if err != nil { - return nil, err - } - return FormatPreparedRun(prepared, outputFormat) -} - -type jsonPreparedRunFormatter struct{} - -func (jsonPreparedRunFormatter) Format(prepared *promptkit.PreparedRun) ([]byte, error) { - if prepared == nil { - return nil, errors.New("prepared run is nil") - } - return json.MarshalIndent(prepared, "", " ") -} - -type textPreparedRunFormatter struct{} - -func (textPreparedRunFormatter) Format(prepared *promptkit.PreparedRun) ([]byte, error) { - if prepared == nil { - return nil, errors.New("prepared run is nil") - } - - var b bytes.Buffer - fmt.Fprintf(&b, "prompt: %s\n", prepared.PromptID) - fmt.Fprintf(&b, "selected_profile_id: %s\n", prepared.SelectedProfileID) - if prepared.SelectedBackendID != "" { - fmt.Fprintf(&b, "selected_backend_id: %s\n", prepared.SelectedBackendID) - } - if prepared.PromptHash != "" { - fmt.Fprintf(&b, "prompt_hash: %s\n", prepared.PromptHash) - } - if prepared.SessionID != "" { - fmt.Fprintf(&b, "session_id: %s\n", prepared.SessionID) - } - fmt.Fprintf(&b, "rendered_prompt_hash: %s\n", prepared.RenderedPromptHash) - - target := prepared.EffectiveModelParams - fmt.Fprintln(&b, "effective_model_params:") - fmt.Fprintf(&b, " endpoint: %s\n", target.Endpoint) - fmt.Fprintf(&b, " model: %s\n", target.Model) - fmt.Fprintf(&b, " temperature: %g\n", target.Temperature) - fmt.Fprintf(&b, " max_tokens: %d\n", target.MaxTokens) - fmt.Fprintf(&b, " top_p: %g\n", target.TopP) - fmt.Fprintf(&b, " timeout_seconds: %d\n", target.TimeoutSeconds) - if target.ServiceTier != "" { - fmt.Fprintf(&b, " service_tier: %s\n", target.ServiceTier) - } - if target.ReasoningEffort != "" { - fmt.Fprintf(&b, " reasoning_effort: %s\n", target.ReasoningEffort) - } - if target.APIKeyEnv != "" { - fmt.Fprintf(&b, " api_key_env: %s\n", target.APIKeyEnv) - } - if len(target.ExtraParams) > 0 { - fmt.Fprintln(&b, " extra_params:") - keys := make([]string, 0, len(target.ExtraParams)) - for k := range target.ExtraParams { - keys = append(keys, k) - } - sort.Strings(keys) - for _, k := range keys { - renderedValue, err := formatExtraParamTextValue(target.ExtraParams[k]) - if err != nil { - return nil, fmt.Errorf("failed to format extra_params.%s: %w", k, err) - } - fmt.Fprintf(&b, " %s: %s\n", k, renderedValue) - } - } - - fmt.Fprintln(&b, "input_hashes:") - inputKeys := make([]string, 0, len(prepared.InputHashes)) - for k := range prepared.InputHashes { - inputKeys = append(inputKeys, k) - } - sort.Strings(inputKeys) - for _, k := range inputKeys { - fmt.Fprintf(&b, " %s: %s\n", k, prepared.InputHashes[k]) - } - - fmt.Fprintln(&b, "messages:") - roleOrder := make([]string, 0) - byRole := make(map[string][]promptkit.RenderedMessage) - for _, msg := range prepared.Messages { - if _, exists := byRole[msg.Role]; !exists { - roleOrder = append(roleOrder, msg.Role) - } - byRole[msg.Role] = append(byRole[msg.Role], msg) - } - for _, role := range roleOrder { - fmt.Fprintf(&b, " %s:\n", role) - messages := byRole[role] - for i, msg := range messages { - fmt.Fprintf(&b, " - message: %d\n", i+1) - if msg.CacheControl != nil { - fmt.Fprintf(&b, " cache_control: %s", msg.CacheControl.Type) - if msg.CacheControl.TTL != "" { - fmt.Fprintf(&b, " ttl=%s", msg.CacheControl.TTL) - } - fmt.Fprintln(&b) - } - fmt.Fprintln(&b, " content: |") - content := msg.Content - if content == "" { - fmt.Fprintln(&b, " ") - continue - } - for _, line := range strings.Split(content, "\n") { - fmt.Fprintf(&b, " %s\n", line) - } - } - } - - return b.Bytes(), nil -} - -func formatExtraParamTextValue(value any) (string, error) { - if s, ok := value.(string); ok { - return s, nil - } - - b, err := json.Marshal(value) - if err != nil { - return "", err - } - return string(b), nil -} diff --git a/internal/format/prepared_run_test.go b/internal/format/prepared_run_test.go deleted file mode 100644 index 1374746..0000000 --- a/internal/format/prepared_run_test.go +++ /dev/null @@ -1,375 +0,0 @@ -package format - -import ( - "encoding/json" - "errors" - "strings" - "testing" - - "go.maximumdirect.net/promptkit" -) - -func TestTextFormatterIncludesPreparedRunDetails(t *testing.T) { - prepared := samplePreparedRun() - - out, err := FormatPreparedRun(prepared, PreparedRunFormatText) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - s := string(out) - if strings.Contains(s, "prompt_version") { - t.Fatalf("unexpected prompt version in text output: %s", s) - } - - for _, want := range []string{ - "prompt: prompt.id", - "selected_profile_id: local-fast", - "selected_backend_id: local", - "endpoint: http://llm/v1", - "model: gpt-test", - "temperature: 0.4", - "max_tokens: 256", - "top_p: 0.8", - "timeout_seconds: 45", - "service_tier: priority", - "reasoning_effort: medium", - "api_key_env: SCRIPTORIUM_API_KEY", - "prompt_hash: prompt-hash", - "rendered_prompt_hash: rendered-hash", - "glossary: hash-glossary", - "transcript: hash-transcript", - "messages:", - " system:", - " user:", - "System guidance.", - "Summarize the transcript.", - "Include key entities.", - "Second user message.", - } { - if !strings.Contains(s, want) { - t.Fatalf("expected text output to include %q, got:\n%s", want, s) - } - } -} - -func TestTextFormatterRendersExtraParamsDeterministically(t *testing.T) { - prepared := samplePreparedRun() - prepared.EffectiveModelParams.ExtraParams = map[string]any{ - "z_string": "enabled", - "b_number": 42, - "a_object": map[string]any{ - "nested": "value", - "count": 2, - }, - "c_array": []any{"first", 3, false}, - } - - out, err := FormatPreparedRun(prepared, PreparedRunFormatText) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - s := string(out) - - want := strings.Join([]string{ - " extra_params:", - " a_object: {\"count\":2,\"nested\":\"value\"}", - " b_number: 42", - " c_array: [\"first\",3,false]", - " z_string: enabled", - }, "\n") - if !strings.Contains(s, want) { - t.Fatalf("expected deterministic extra_params block %q, got:\n%s", want, s) - } -} - -func TestTextFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) { - const secret = "super-secret-api-key" - t.Setenv("SCRIPTORIUM_API_KEY", secret) - - out, err := FormatPreparedRun(samplePreparedRun(), PreparedRunFormatText) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if strings.Contains(string(out), secret) { - t.Fatalf("text output should not include resolved api key value: %s", out) - } -} - -func TestTextFormatterDoesNotIncludeDirectAPIKeyValue(t *testing.T) { - const directKey = "direct-format-key" - // PreparedRun intentionally has no field for direct API keys. - prepared := samplePreparedRun() - out, err := FormatPreparedRun(prepared, PreparedRunFormatText) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if strings.Contains(string(out), directKey) { - t.Fatalf("text output should not include direct api key value: %s", out) - } -} - -func TestTextFormatterIncludesMessageCacheControlBeforeContent(t *testing.T) { - prepared := samplePreparedRun() - prepared.Messages = []promptkit.RenderedMessage{ - { - Role: "system", - Content: "System guidance.", - CacheControl: &promptkit.CacheControl{ - Type: promptkit.CacheControlEphemeral, - TTL: "1h", - }, - }, - {Role: "user", Content: "Summarize the transcript."}, - } - - out, err := FormatPreparedRun(prepared, PreparedRunFormatText) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - s := string(out) - if !strings.Contains(s, " system:\n - message: 1\n cache_control: ephemeral ttl=1h\n content: |") { - t.Fatalf("expected system message cache control before content, got:\n%s", s) - } - if strings.Count(s, "cache_control:") != 1 { - t.Fatalf("expected exactly one cache_control line, got:\n%s", s) - } -} - -func TestTextFormatterIncludesSessionIDWhenPresent(t *testing.T) { - prepared := samplePreparedRun() - prepared.SessionID = "session-123" - - out, err := FormatPreparedRun(prepared, PreparedRunFormatText) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if !strings.Contains(string(out), "session_id: session-123\n") { - t.Fatalf("expected session_id in text output, got:\n%s", out) - } -} - -func TestTextFormatterOmitsEmptyCacheControlTTL(t *testing.T) { - prepared := samplePreparedRun() - prepared.Messages = []promptkit.RenderedMessage{ - { - Role: "system", - Content: "System guidance.", - CacheControl: &promptkit.CacheControl{ - Type: promptkit.CacheControlEphemeral, - }, - }, - } - - out, err := FormatPreparedRun(prepared, PreparedRunFormatText) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - s := string(out) - if !strings.Contains(s, " cache_control: ephemeral\n") { - t.Fatalf("expected cache_control line without ttl, got:\n%s", s) - } - if strings.Contains(s, "ttl=") { - t.Fatalf("expected empty ttl to be omitted, got:\n%s", s) - } -} - -func TestJSONFormatterEmitsValidJSONAndIncludesPreparedRunFields(t *testing.T) { - prepared := samplePreparedRun() - prepared.SessionID = "session-123" - prepared.EffectiveModelParams.ExtraParams = map[string]any{ - "number": 42, - "nested": map[string]any{ - "enabled": true, - }, - } - - out, err := FormatPreparedRun(prepared, PreparedRunFormatJSON) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - - var decoded map[string]any - if err := json.Unmarshal(out, &decoded); err != nil { - t.Fatalf("expected valid json output, got %v", err) - } - - if decoded["prompt_id"] != "prompt.id" { - t.Fatalf("expected prompt_id in json output, got %#v", decoded["prompt_id"]) - } - if _, ok := decoded["prompt_version"]; ok { - t.Fatalf("unexpected prompt_version in json output: %#v", decoded) - } - if decoded["selected_profile_id"] != "local-fast" { - t.Fatalf("expected selected_profile_id in json output, got %#v", decoded["selected_profile_id"]) - } - if decoded["rendered_prompt_hash"] != "rendered-hash" { - t.Fatalf("expected rendered_prompt_hash in json output, got %#v", decoded["rendered_prompt_hash"]) - } - if decoded["session_id"] != "session-123" { - t.Fatalf("expected session_id in json output, got %#v", decoded["session_id"]) - } - modelParams, ok := decoded["effective_model_params"].(map[string]any) - if !ok { - t.Fatalf("expected effective_model_params in json output, got %#v", decoded) - } - extraParams, ok := modelParams["extra_params"].(map[string]any) - if !ok { - t.Fatalf("expected extra_params in json output, got %#v", modelParams["extra_params"]) - } - if extraParams["number"] != float64(42) { - t.Fatalf("unexpected numeric extra param in json output: %#v", extraParams["number"]) - } - nested, ok := extraParams["nested"].(map[string]any) - if !ok || nested["enabled"] != true { - t.Fatalf("unexpected nested extra param in json output: %#v", extraParams["nested"]) - } - if _, ok := decoded["input_hashes"]; !ok { - t.Fatalf("expected input_hashes in json output, got %#v", decoded) - } - if _, ok := decoded["messages"]; !ok { - t.Fatalf("expected messages in json output, got %#v", decoded) - } -} - -func TestJSONFormatterIncludesMessageCacheControlOnlyWhenPresent(t *testing.T) { - prepared := samplePreparedRun() - prepared.Messages = []promptkit.RenderedMessage{ - { - Role: "system", - Content: "System guidance.", - CacheControl: &promptkit.CacheControl{ - Type: promptkit.CacheControlEphemeral, - TTL: "1h", - }, - }, - {Role: "user", Content: "Summarize the transcript."}, - } - - out, err := FormatPreparedRun(prepared, PreparedRunFormatJSON) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - - var decoded struct { - Messages []map[string]any `json:"messages"` - } - if err := json.Unmarshal(out, &decoded); err != nil { - t.Fatalf("expected valid json output, got %v", err) - } - if len(decoded.Messages) != 2 { - t.Fatalf("expected 2 messages, got %d", len(decoded.Messages)) - } - - cacheControl, ok := decoded.Messages[0]["cache_control"].(map[string]any) - if !ok { - t.Fatalf("expected first message cache_control, got %#v", decoded.Messages[0]) - } - if cacheControl["type"] != string(promptkit.CacheControlEphemeral) || cacheControl["ttl"] != "1h" { - t.Fatalf("unexpected cache_control payload: %#v", cacheControl) - } - if _, ok := decoded.Messages[1]["cache_control"]; ok { - t.Fatalf("expected second message to omit cache_control, got %#v", decoded.Messages[1]) - } -} - -func TestJSONFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) { - const secret = "super-secret-api-key" - t.Setenv("SCRIPTORIUM_API_KEY", secret) - - out, err := FormatPreparedRun(samplePreparedRun(), PreparedRunFormatJSON) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if strings.Contains(string(out), secret) { - t.Fatalf("json output should not include resolved api key value: %s", out) - } -} - -func TestJSONFormatterDoesNotIncludeDirectAPIKeyValue(t *testing.T) { - const directKey = "direct-format-key" - // PreparedRun intentionally has no field for direct API keys. - prepared := samplePreparedRun() - out, err := FormatPreparedRun(prepared, PreparedRunFormatJSON) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if strings.Contains(string(out), directKey) { - t.Fatalf("json output should not include direct api key value: %s", out) - } -} - -func TestParsePreparedRunOutputFormatRecognizesSupportedNames(t *testing.T) { - tests := []struct { - name string - input string - want PreparedRunOutputFormat - }{ - {name: "default empty", input: "", want: DefaultPreparedRunOutputFormat}, - {name: "text", input: "text", want: PreparedRunFormatText}, - {name: "json", input: "json", want: PreparedRunFormatJSON}, - {name: "trim and case", input: " JSON ", want: PreparedRunFormatJSON}, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - got, err := ParsePreparedRunOutputFormat(tc.input) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if got != tc.want { - t.Fatalf("expected %q, got %q", tc.want, got) - } - }) - } -} - -func TestParsePreparedRunOutputFormatUnknownFails(t *testing.T) { - _, err := ParsePreparedRunOutputFormat("yaml") - if err == nil { - t.Fatal("expected error for unknown format") - } - if !errors.Is(err, ErrUnknownPreparedRunFormat) { - t.Fatalf("expected ErrUnknownPreparedRunFormat, got %v", err) - } -} - -func TestFormatPreparedRunByNameUnknownFailsClearly(t *testing.T) { - _, err := FormatPreparedRunByName(samplePreparedRun(), "yaml") - if err == nil { - t.Fatal("expected unknown format error") - } - if !errors.Is(err, ErrUnknownPreparedRunFormat) { - t.Fatalf("expected ErrUnknownPreparedRunFormat, got %v", err) - } -} - -func samplePreparedRun() *promptkit.PreparedRun { - return &promptkit.PreparedRun{ - PromptID: "prompt.id", - PromptHash: "prompt-hash", - SelectedProfileID: "local-fast", - SelectedBackendID: "local", - EffectiveModelParams: promptkit.ExecutionTarget{ - BackendID: "local", - Endpoint: "http://llm/v1", - Model: "gpt-test", - Temperature: 0.4, - MaxTokens: 256, - TopP: 0.8, - TimeoutSeconds: 45, - ServiceTier: "priority", - ReasoningEffort: "medium", - APIKeyEnv: "SCRIPTORIUM_API_KEY", - }, - InputHashes: map[string]string{ - "transcript": "hash-transcript", - "glossary": "hash-glossary", - }, - RenderedPromptHash: "rendered-hash", - Messages: []promptkit.RenderedMessage{ - {Role: "system", Content: "System guidance."}, - {Role: "user", Content: "Summarize the transcript.\nInclude key entities."}, - {Role: "user", Content: "Second user message."}, - }, - } -}