Add reusable PromptKit request flags
This commit is contained in:
88
docs/cli.md
88
docs/cli.md
@@ -563,8 +563,8 @@ native selection and discovery rules:
|
|||||||
|
|
||||||
```text
|
```text
|
||||||
example promptkit requirements show briefing --campaign spring
|
example promptkit requirements show briefing --campaign spring
|
||||||
example promptkit requirements show briefing --profile editorial --var mode=full
|
example promptkit requirements show briefing --llm-profile editorial --llm-var mode=full
|
||||||
example promptkit requirements show briefing --session-id session-42 --var note= --format yaml
|
example promptkit requirements show briefing --llm-session-id session-42 --llm-var note= --format yaml
|
||||||
```
|
```
|
||||||
|
|
||||||
The command accepts:
|
The command accepts:
|
||||||
@@ -574,9 +574,9 @@ The command accepts:
|
|||||||
| `--config` | path string | omitted | Select the configuration input; explicit presence is retained. |
|
| `--config` | path string | omitted | Select the configuration input; explicit presence is retained. |
|
||||||
| `--overlay` | path string | omitted | Select one overlay; explicit presence is retained. Repeated occurrences use the last value. |
|
| `--overlay` | path string | omitted | Select one overlay; explicit presence is retained. Repeated occurrences use the last value. |
|
||||||
| Suite selectors | application-defined | per selector | The flags captured from `Suite.Selectors`. |
|
| Suite selectors | application-defined | per selector | The flags captured from `Suite.Selectors`. |
|
||||||
| `--profile` | string | empty | Pass the profile ID to PromptKit; empty uses its native selection rule. |
|
| `--llm-profile` | string | empty | Pass the profile ID to PromptKit; empty uses its native selection rule. |
|
||||||
| `--session-id` | string | empty | Pass session context to PromptKit. |
|
| `--llm-session-id` | string | empty | Pass session context to PromptKit. |
|
||||||
| `--var` | repeatable `KEY=VALUE` | none | Supply caller-variable context for native discovery. |
|
| `--llm-var` | repeatable `KEY=VALUE` | none | Supply caller-variable context for native discovery. |
|
||||||
| `--format` | `text`, `yaml`, `json` | automatic | Select the requirements report format; automatic and text use YAML for this structured report. |
|
| `--format` | `text`, `yaml`, `json` | automatic | Select the requirements report format; automatic and text use YAML for this structured report. |
|
||||||
|
|
||||||
The `promptkit` namespace identifies the operation, so there is no built-in
|
The `promptkit` namespace identifies the operation, so there is no built-in
|
||||||
@@ -585,10 +585,10 @@ native empty-string meaning. Requirements inspection does not require a profile
|
|||||||
to be execution-ready. See the
|
to be execution-ready. See the
|
||||||
[PromptKit v0.18.0 consumer guide](https://forge.maximumdirect.net/base-layer/promptkit/src/commit/d9f6aad30509e538fc73ea16710df3a3ec084502/docs/consumers/pkg-promptkit.md)
|
[PromptKit v0.18.0 consumer guide](https://forge.maximumdirect.net/base-layer/promptkit/src/commit/d9f6aad30509e538fc73ea16710df3a3ec084502/docs/consumers/pkg-promptkit.md)
|
||||||
for native configuration, selection, and requirements behavior. The built-in
|
for native configuration, selection, and requirements behavior. The built-in
|
||||||
flags other than `--var` are singleton bindings; when repeated, the last value
|
flags other than `--llm-var` are singleton bindings; when repeated, the last value
|
||||||
is used. Suite selectors retain their declared binding behavior.
|
is used. Suite selectors retain their declared binding behavior.
|
||||||
|
|
||||||
Each `--var` occurrence must contain `=`. CommandKit splits at the first
|
Each `--llm-var` occurrence must contain `=`. CommandKit splits at the first
|
||||||
separator and preserves the exact key and remaining value. Empty keys and
|
separator and preserves the exact key and remaining value. Empty keys and
|
||||||
values, whitespace, commas, and additional `=` characters are retained. All
|
values, whitespace, commas, and additional `=` characters are retained. All
|
||||||
occurrences are parsed; duplicate keys use the last value, while any malformed
|
occurrences are parsed; duplicate keys use the last value, while any malformed
|
||||||
@@ -644,8 +644,8 @@ prompt ID and passes a concrete request to PromptKit's native
|
|||||||
`Engine.ValidateRequest` preparation path:
|
`Engine.ValidateRequest` preparation path:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
example promptkit requirements validate briefing --campaign spring --input topic=topic.txt
|
example promptkit requirements validate briefing --campaign spring --llm-input-file topic=topic.txt
|
||||||
example promptkit requirements validate briefing --profile editorial --session-id session-42 --var mode=full --input notes=notes.txt
|
example promptkit requirements validate briefing --llm-profile editorial --llm-session-id session-42 --llm-var mode=full --llm-input-file notes=notes.txt
|
||||||
```
|
```
|
||||||
|
|
||||||
The command accepts:
|
The command accepts:
|
||||||
@@ -655,24 +655,28 @@ The command accepts:
|
|||||||
| `--config` | path string | omitted | Select the configuration input; explicit presence is retained. |
|
| `--config` | path string | omitted | Select the configuration input; explicit presence is retained. |
|
||||||
| `--overlay` | path string | omitted | Select one overlay; explicit presence is retained. Repeated occurrences use the last value. |
|
| `--overlay` | path string | omitted | Select one overlay; explicit presence is retained. Repeated occurrences use the last value. |
|
||||||
| Suite selectors | application-defined | per selector | The flags captured from `Suite.Selectors`. |
|
| Suite selectors | application-defined | per selector | The flags captured from `Suite.Selectors`. |
|
||||||
| `--profile` | string | empty | Pass the profile ID to PromptKit. |
|
| `--llm-profile` | string | empty | Pass the profile ID to PromptKit. |
|
||||||
| `--session-id` | string | empty | Pass session context to PromptKit. |
|
| `--llm-session-id` | string | empty | Pass session context to PromptKit. |
|
||||||
| `--var` | repeatable `KEY=VALUE` | none | Supply caller variables for the concrete request. |
|
| `--llm-var` | repeatable `KEY=VALUE` | none | Supply caller variables for the concrete request. |
|
||||||
| `--input` | repeatable `NAME=PATH` | none | Bind a file-backed PromptKit artifact. |
|
| `--llm-input-text` | repeatable `NAME=TEXT` | none | Bind a literal PromptKit artifact. |
|
||||||
|
| `--llm-input-file` | repeatable `NAME=PATH` | none | Bind a file-backed PromptKit artifact. |
|
||||||
| `--format` | `text`, `yaml`, `json` | automatic | Select the success confirmation format; automatic and text use YAML for this structured report. |
|
| `--format` | `text`, `yaml`, `json` | automatic | Select the success confirmation format; automatic and text use YAML for this structured report. |
|
||||||
|
|
||||||
The built-in flags other than repeatable `--var` and `--input` are singleton
|
Validation also accepts the execution controls from `--llm-endpoint` through
|
||||||
bindings; when repeated, the last value is used. Explicit empty `--config` or
|
`--llm-extra-params` listed under [smoke testing](#promptkit-smoke-testing).
|
||||||
|
The built-in flags other than
|
||||||
|
`--llm-var`, `--llm-input-text`, and `--llm-input-file` are singleton bindings;
|
||||||
|
when repeated, the last value is used. Explicit empty `--config` or
|
||||||
`--overlay` values keep their supplied-presence information. The profile and
|
`--overlay` values keep their supplied-presence information. The profile and
|
||||||
session fields use native empty-string semantics. PromptKit owns their
|
session fields use native empty-string semantics. PromptKit owns their
|
||||||
interpretation and selection errors.
|
interpretation and selection errors.
|
||||||
|
|
||||||
Each `--var` and `--input` occurrence must contain `=`. CommandKit splits at the
|
Each `--llm-var`, `--llm-input-text`, and `--llm-input-file` occurrence must
|
||||||
first separator and preserves the exact key/name and remaining value/path.
|
contain `=`. CommandKit splits at the first separator and preserves the exact
|
||||||
Empty variable keys and values and empty input names and paths are passed to
|
key/name and remaining value. Empty names and values are passed to native
|
||||||
native validation. File paths become `promptkit.File(PATH)` references without
|
validation. Text becomes a `promptkit.Inline` reference; file paths become
|
||||||
being read, normalized, expanded, or rebased during argument binding. Duplicate
|
`promptkit.File` references without being read, normalized, expanded, or
|
||||||
keys or names use the last value, but every occurrence is parsed.
|
rebased during argument binding. Every occurrence is parsed.
|
||||||
|
|
||||||
On success, the confirmation contains only the requested prompt ID and
|
On success, the confirmation contains only the requested prompt ID and
|
||||||
`valid: true`:
|
`valid: true`:
|
||||||
@@ -765,33 +769,39 @@ each aggregate call changes the seed kind before invoking it. The command
|
|||||||
accepts:
|
accepts:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
example promptkit test text_inference --campaign spring --prompt briefing --profile spring
|
example promptkit test text_inference --campaign spring --llm-prompt briefing --llm-profile spring
|
||||||
example promptkit test all --campaign spring --prompt briefing --profile spring
|
example promptkit test all --campaign spring --llm-prompt briefing --llm-profile spring
|
||||||
example promptkit test authentication --backend local
|
example promptkit test authentication --llm-backend local
|
||||||
```
|
```
|
||||||
|
|
||||||
| Flag | Values | Default | Meaning |
|
| Flag | Values | Default | Meaning |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| `--config`, `--overlay` | path strings | omitted | Select PromptKit configuration; supplied presence is retained. |
|
| `--config`, `--overlay` | path strings | omitted | Select PromptKit configuration; supplied presence is retained. |
|
||||||
| Suite selectors | application-defined | per selector | Captured flags from `Suite.Selectors`. |
|
| Suite selectors | application-defined | per selector | Captured flags from `Suite.Selectors`. |
|
||||||
| `--backend` | string | empty | Native backend ID. |
|
| `--llm-backend` | string | empty | Native backend ID. |
|
||||||
| `--prompt` | string | empty | Native prompt ID. |
|
| `--llm-prompt` | string | empty | Native prompt ID. |
|
||||||
| `--profile` | string | empty | Native profile ID. |
|
| `--llm-profile` | string | empty | Native profile ID. |
|
||||||
| `--endpoint` | string | omitted | Override the execution endpoint. |
|
| `--llm-endpoint` | string | omitted | Override the execution endpoint. |
|
||||||
| `--model` | string | omitted | Override the execution model. |
|
| `--llm-model` | string | omitted | Override the execution model. |
|
||||||
| `--temperature` | float | omitted | Override execution temperature. |
|
| `--llm-temperature` | float | omitted | Override execution temperature. |
|
||||||
| `--max-tokens` | integer | omitted | Override execution maximum tokens. |
|
| `--llm-max-tokens` | integer | omitted | Override execution maximum tokens. |
|
||||||
| `--top-p` | float | omitted | Override execution top-p. |
|
| `--llm-top-p` | float | omitted | Override execution top-p. |
|
||||||
| `--timeout-seconds` | integer | omitted | Override the execution timeout. |
|
| `--llm-timeout-seconds` | integer | omitted | Override the execution timeout. |
|
||||||
| `--service-tier` | string | omitted | Override the execution service tier. |
|
| `--llm-service-tier` | string | omitted | Override the execution service tier. |
|
||||||
| `--reasoning-effort` | string | omitted | Override execution reasoning effort. |
|
| `--llm-reasoning-effort` | string | omitted | Override execution reasoning effort. |
|
||||||
| `--api-key-env` | string | omitted | Supply an API-key environment-variable name to native configuration. |
|
| `--llm-api-key-env` | string | omitted | Supply an API-key environment-variable name to native configuration. |
|
||||||
| `--transient-retries` | integer | omitted | Set the native inference transient retry count. |
|
| `--llm-transient-retries` | integer | omitted | Set the native inference transient retry count. |
|
||||||
|
| `--llm-extra-params` | JSON object | omitted | Supply provider parameters as a whole map. |
|
||||||
| `--format` | `text`, `yaml`, `json` | automatic | Select the report format; automatic and text use YAML for these structured reports. |
|
| `--format` | `text`, `yaml`, `json` | automatic | Select the report format; automatic and text use YAML for these structured reports. |
|
||||||
|
|
||||||
The execution flags populate PromptKit's native execution override. The
|
The execution flags populate PromptKit's native execution override. The
|
||||||
`--api-key-env` value is a variable name, not a credential value. Application
|
`--llm-api-key-env` value is a variable name, not a credential value. The
|
||||||
selectors, loader policy, credentials, and client wiring remain application
|
extra-parameters flag accepts one JSON object, including an empty object;
|
||||||
|
PromptKit owns its native replacement semantics and validation. Integer JSON
|
||||||
|
numbers use signed or unsigned 64-bit integers where representable; decimal
|
||||||
|
and exponent numbers use floating point.
|
||||||
|
|
||||||
|
Application selectors, loader policy, credentials, and client wiring remain application
|
||||||
owned. The `WithSmokeTesting` helper accepts an optional typed builder. A nil
|
owned. The `WithSmokeTesting` helper accepts an optional typed builder. A nil
|
||||||
builder forwards the seed; a supplied builder's complete native request is
|
builder forwards the seed; a supplied builder's complete native request is
|
||||||
authoritative. The helper retains a native report when PromptKit returns both a
|
authoritative. The helper retains a native report when PromptKit returns both a
|
||||||
|
|||||||
@@ -326,6 +326,12 @@ and state. `FromEngine` uses PromptKit's public listing and inspection methods,
|
|||||||
including `Engine.InspectRequirements`; validation must be opted into separately
|
including `Engine.InspectRequirements`; validation must be opted into separately
|
||||||
and uses `Engine.ValidateRequest`.
|
and uses `Engine.ValidateRequest`.
|
||||||
|
|
||||||
|
Application-owned commands can use `ckprompt.NewExecutionFlags`,
|
||||||
|
`ckprompt.NewResourceFlags`, and `ckprompt.NewPromptInputFlags` to bind the
|
||||||
|
same canonical request flags into their own input types. Each collection exposes
|
||||||
|
individual handles and an `All` method, so a command can choose its applicable
|
||||||
|
subset. The application decides how to apply the resulting values to its request.
|
||||||
|
|
||||||
Catalog, named definition inspection, and requirements discovery do not render
|
Catalog, named definition inspection, and requirements discovery do not render
|
||||||
messages or load request artifacts. Concrete request validation follows native
|
messages or load request artifacts. Concrete request validation follows native
|
||||||
PromptKit preparation, which can resolve prompt/profile resources, load
|
PromptKit preparation, which can resolve prompt/profile resources, load
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
# Shared reporting and PromptKit v0.20.0 implementation plan
|
# Shared reporting and PromptKit v0.20.0 implementation plan
|
||||||
|
|
||||||
Status: implementation in progress; Stages 1–9 complete.
|
Status: implementation in progress; Stages 1–10 complete.
|
||||||
|
|
||||||
## Scope, authority, and execution
|
## Scope, authority, and execution
|
||||||
|
|
||||||
@@ -350,6 +350,11 @@ for the converted commands.
|
|||||||
|
|
||||||
## Stage 10: Reusable execution and resource flag collections
|
## Stage 10: Reusable execution and resource flag collections
|
||||||
|
|
||||||
|
**Complete.** Reusable request flag collections feed requirements discovery,
|
||||||
|
validation, and smoke requests; the next stage supplies ordered text/file
|
||||||
|
precedence and prepared-request preview. The root, ckconfig, and ckprompt
|
||||||
|
workspace suites passed with Go 1.27.0, including collection and builder tests.
|
||||||
|
|
||||||
Implement CLI-specific collections in ckprompt using ordinary public core flags.
|
Implement CLI-specific collections in ckprompt using ordinary public core flags.
|
||||||
Use typed flag-set structs exposing named original `Flag[I]` handles and an
|
Use typed flag-set structs exposing named original `Flag[I]` handles and an
|
||||||
`All() []Flag[I]` convenience method; callers select subsets with ordinary slices.
|
`All() []Flag[I]` convenience method; callers select subsets with ordinary slices.
|
||||||
|
|||||||
@@ -31,6 +31,8 @@ type catalogInput[I any] struct {
|
|||||||
type requirementsInput[I any] struct {
|
type requirementsInput[I any] struct {
|
||||||
selection ckconfig.Selection
|
selection ckconfig.Selection
|
||||||
inspection promptkit.RequirementInspectionRequest
|
inspection promptkit.RequirementInspectionRequest
|
||||||
|
resources ResourceInput
|
||||||
|
inputs PromptInputs
|
||||||
captured commandkit.FlagValues[I]
|
captured commandkit.FlagValues[I]
|
||||||
selected I
|
selected I
|
||||||
}
|
}
|
||||||
@@ -281,24 +283,12 @@ func newRequirementsCommand[I, S any](suite Suite[I, S]) commandkit.Command {
|
|||||||
func(input **requirementsInput[I], value commandkit.Supplied[string]) {
|
func(input **requirementsInput[I], value commandkit.Supplied[string]) {
|
||||||
(*input).selection.Overlay = value
|
(*input).selection.Overlay = value
|
||||||
}),
|
}),
|
||||||
commandkit.ValueFlag(commandkit.FlagMeta{Name: "profile", Usage: "Profile identity for discovery."}, stringDefault(""), parseDefinitionString,
|
|
||||||
func(input **requirementsInput[I], value commandkit.Supplied[string]) {
|
|
||||||
(*input).inspection.ProfileID = value.Value
|
|
||||||
}),
|
|
||||||
commandkit.ValueFlag(commandkit.FlagMeta{Name: "session-id", Usage: "Session context for discovery."}, stringDefault(""), parseDefinitionString,
|
|
||||||
func(input **requirementsInput[I], value commandkit.Supplied[string]) {
|
|
||||||
(*input).inspection.SessionID = value.Value
|
|
||||||
}),
|
|
||||||
commandkit.RepeatFlag(commandkit.FlagMeta{Name: "var", Usage: "Caller variable assignment KEY=VALUE."}, []variableAssignment(nil), parseVariableAssignment,
|
|
||||||
func(input **requirementsInput[I], value commandkit.Supplied[[]variableAssignment]) {
|
|
||||||
variables := make(map[string]string, len(value.Value))
|
|
||||||
for _, assignment := range value.Value {
|
|
||||||
variables[assignment.key] = assignment.value
|
|
||||||
}
|
|
||||||
(*input).inspection.Vars = variables
|
|
||||||
}),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
resources := NewResourceFlags(func(input **requirementsInput[I]) *ResourceInput { return &(*input).resources })
|
||||||
|
inputs := NewPromptInputFlags(func(input **requirementsInput[I]) *PromptInputs { return &(*input).inputs })
|
||||||
|
flags = append(flags, resources.Profile, resources.SessionID, inputs.Variable)
|
||||||
|
|
||||||
spec := commandkit.CommandSpec[*requirementsInput[I]]{
|
spec := commandkit.CommandSpec[*requirementsInput[I]]{
|
||||||
Name: "show",
|
Name: "show",
|
||||||
Summary: "Inspect requirements for a named PromptKit prompt.",
|
Summary: "Inspect requirements for a named PromptKit prompt.",
|
||||||
@@ -313,6 +303,9 @@ func newRequirementsCommand[I, S any](suite Suite[I, S]) commandkit.Command {
|
|||||||
}),
|
}),
|
||||||
},
|
},
|
||||||
CheckInput: func(input *requirementsInput[I]) error {
|
CheckInput: func(input *requirementsInput[I]) error {
|
||||||
|
input.inspection.ProfileID = input.resources.Profile.Value
|
||||||
|
input.inspection.SessionID = input.resources.SessionID.Value
|
||||||
|
input.inspection.Vars = input.inputs.Vars
|
||||||
selected, err := prepareSuiteInput(suite, input.selection, input.captured)
|
selected, err := prepareSuiteInput(suite, input.selection, input.captured)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
|||||||
@@ -141,7 +141,7 @@ func TestRequirementsCommandCapturesContextAndVariableAssignments(t *testing.T)
|
|||||||
if familyOutcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(familyHelp.String(), "requirements") || strings.Contains(familyHelp.String(), "\n show ") || strings.Contains(familyHelp.String(), "\n list ") {
|
if familyOutcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(familyHelp.String(), "requirements") || strings.Contains(familyHelp.String(), "\n show ") || strings.Contains(familyHelp.String(), "\n list ") {
|
||||||
t.Fatalf("requirements-only family help = (%#v, %q)", familyOutcome, familyHelp.String())
|
t.Fatalf("requirements-only family help = (%#v, %q)", familyOutcome, familyHelp.String())
|
||||||
}
|
}
|
||||||
for _, flag := range []string{"--config", "--overlay", "--profile", "--session-id", "--var", "--format", "--campaign"} {
|
for _, flag := range []string{"--config", "--overlay", "--llm-profile", "--llm-session-id", "--llm-var", "--format", "--campaign"} {
|
||||||
if !strings.Contains(help.String(), flag) {
|
if !strings.Contains(help.String(), flag) {
|
||||||
t.Errorf("requirements help omits %s: %q", flag, help.String())
|
t.Errorf("requirements help omits %s: %q", flag, help.String())
|
||||||
}
|
}
|
||||||
@@ -154,10 +154,10 @@ func TestRequirementsCommandCapturesContextAndVariableAssignments(t *testing.T)
|
|||||||
args := []string{
|
args := []string{
|
||||||
"promptkit", "requirements", "show", "briefing", "--campaign", "spring",
|
"promptkit", "requirements", "show", "briefing", "--campaign", "spring",
|
||||||
"--config", "before.yml", "--config=", "--overlay", "base.yml", "--overlay=",
|
"--config", "before.yml", "--config=", "--overlay", "base.yml", "--overlay=",
|
||||||
"--profile", "first", "--profile", "editorial",
|
"--llm-profile", "first", "--llm-profile", "editorial",
|
||||||
"--session-id", "old-session", "--session-id", "session-42", "--format", "text", "--format", "yaml",
|
"--llm-session-id", "old-session", "--llm-session-id", "session-42", "--format", "text", "--format", "yaml",
|
||||||
"--var", "note=hello, world", "--var", "empty=", "--var", "token=a=b", "--var", "=value",
|
"--llm-var", "note=hello, world", "--llm-var", "empty=", "--llm-var", "token=a=b", "--llm-var", "=value",
|
||||||
"--var", "mode=brief", "--var", "mode=full",
|
"--llm-var", "mode=brief", "--llm-var", "mode=full",
|
||||||
}
|
}
|
||||||
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: io.Discard})
|
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: io.Discard})
|
||||||
if outcome.Kind != commandkit.OutcomeSuccess || outcome.Status != 0 {
|
if outcome.Kind != commandkit.OutcomeSuccess || outcome.Status != 0 {
|
||||||
@@ -203,9 +203,9 @@ func TestRequirementsCommandUsageFailuresPrecedeApplicationPreparation(t *testin
|
|||||||
{name: "missing operand", args: []string{"promptkit", "requirements", "show"}},
|
{name: "missing operand", args: []string{"promptkit", "requirements", "show"}},
|
||||||
{name: "extra operand", args: []string{"promptkit", "requirements", "show", "briefing", "extra"}},
|
{name: "extra operand", args: []string{"promptkit", "requirements", "show", "briefing", "extra"}},
|
||||||
{name: "unsupported format", args: []string{"promptkit", "requirements", "show", "briefing", "--format", "invalid"}},
|
{name: "unsupported format", args: []string{"promptkit", "requirements", "show", "briefing", "--format", "invalid"}},
|
||||||
{name: "malformed variable followed by valid assignment", args: []string{"promptkit", "requirements", "show", "briefing", "--var", "broken", "--var", "mode=full"}},
|
{name: "malformed variable followed by valid assignment", args: []string{"promptkit", "requirements", "show", "briefing", "--llm-var", "broken", "--llm-var", "mode=full"}},
|
||||||
{name: "no built-in scope", args: []string{"promptkit", "requirements", "show", "briefing", "--scope", "all"}},
|
{name: "no built-in scope", args: []string{"promptkit", "requirements", "show", "briefing", "--scope", "all"}},
|
||||||
{name: "malformed variable on help", args: []string{"help", "promptkit", "requirements", "show", "--var", "broken"}},
|
{name: "malformed variable on help", args: []string{"help", "promptkit", "requirements", "show", "--llm-var", "broken"}},
|
||||||
} {
|
} {
|
||||||
t.Run(test.name, func(t *testing.T) {
|
t.Run(test.name, func(t *testing.T) {
|
||||||
newInputCalls, checkInputCalls, newStateCalls, providerCalls := 0, 0, 0, 0
|
newInputCalls, checkInputCalls, newStateCalls, providerCalls := 0, 0, 0, 0
|
||||||
|
|||||||
@@ -25,13 +25,13 @@ go run . promptkit show prompt briefing --campaign spring --format yaml
|
|||||||
go run . pk show prompt briefing --campaign spring --fmt yaml
|
go run . pk show prompt briefing --campaign spring --fmt yaml
|
||||||
go run . promptkit show profile spring --campaign spring
|
go run . promptkit show profile spring --campaign spring
|
||||||
go run . promptkit requirements show briefing --campaign spring
|
go run . promptkit requirements show briefing --campaign spring
|
||||||
go run . promptkit requirements show briefing --campaign spring --var mode=full --format yaml
|
go run . promptkit requirements show briefing --campaign spring --llm-var mode=full --format yaml
|
||||||
go run . promptkit requirements validate briefing --campaign spring --input topic=fixtures/topic.txt
|
go run . promptkit requirements validate briefing --campaign spring --llm-input-file topic=fixtures/topic.txt
|
||||||
go run . promptkit requirements validate briefing --campaign spring --input topic=fixtures/topic.txt --var mode=full --input notes=fixtures/notes.txt --format yaml
|
go run . promptkit requirements validate briefing --campaign spring --llm-input-file topic=fixtures/topic.txt --llm-var mode=full --llm-input-file notes=fixtures/notes.txt --format yaml
|
||||||
go run . promptkit requirements validate briefing --campaign spring
|
go run . promptkit requirements validate briefing --campaign spring
|
||||||
go run . promptkit test text_inference --campaign spring --prompt briefing --profile spring
|
go run . promptkit test text_inference --campaign spring --llm-prompt briefing --llm-profile spring
|
||||||
go run . smoke text_inference --campaign spring --prompt briefing --profile spring
|
go run . smoke text_inference --campaign spring --llm-prompt briefing --llm-profile spring
|
||||||
go run . promptkit test all --campaign spring --prompt briefing --profile spring
|
go run . promptkit test all --campaign spring --llm-prompt briefing --llm-profile spring
|
||||||
go run . pk --hlep
|
go run . pk --hlep
|
||||||
go run . session restore session-42 --campaign spring
|
go run . session restore session-42 --campaign spring
|
||||||
```
|
```
|
||||||
|
|||||||
@@ -164,7 +164,7 @@ func TestComposedRequirementsDiscoveryUsesSelectedCampaign(t *testing.T) {
|
|||||||
args []string
|
args []string
|
||||||
}{
|
}{
|
||||||
{name: "without variables", args: []string{"promptkit", "requirements", "show", "briefing", "--campaign", "spring"}},
|
{name: "without variables", args: []string{"promptkit", "requirements", "show", "briefing", "--campaign", "spring"}},
|
||||||
{name: "with caller variable", args: []string{"promptkit", "requirements", "show", "briefing", "--campaign", "spring", "--var", "mode=full", "--format", "yaml"}},
|
{name: "with caller variable", args: []string{"promptkit", "requirements", "show", "briefing", "--campaign", "spring", "--llm-var", "mode=full", "--format", "yaml"}},
|
||||||
} {
|
} {
|
||||||
t.Run(test.name, func(t *testing.T) {
|
t.Run(test.name, func(t *testing.T) {
|
||||||
var output bytes.Buffer
|
var output bytes.Buffer
|
||||||
@@ -204,11 +204,11 @@ func TestComposedRequestValidationUsesFileBindings(t *testing.T) {
|
|||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "default mode",
|
name: "default mode",
|
||||||
args: []string{"promptkit", "requirements", "validate", "briefing", "--campaign", "spring", "--input", "topic=fixtures/topic.txt"},
|
args: []string{"promptkit", "requirements", "validate", "briefing", "--campaign", "spring", "--llm-input-file", "topic=fixtures/topic.txt"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "conditional notes",
|
name: "conditional notes",
|
||||||
args: []string{"promptkit", "requirements", "validate", "briefing", "--campaign", "spring", "--input", "topic=fixtures/topic.txt", "--var", "mode=full", "--input", "notes=fixtures/notes.txt", "--format", "yaml"},
|
args: []string{"promptkit", "requirements", "validate", "briefing", "--campaign", "spring", "--llm-input-file", "topic=fixtures/topic.txt", "--llm-var", "mode=full", "--llm-input-file", "notes=fixtures/notes.txt", "--format", "yaml"},
|
||||||
},
|
},
|
||||||
} {
|
} {
|
||||||
t.Run(test.name, func(t *testing.T) {
|
t.Run(test.name, func(t *testing.T) {
|
||||||
@@ -393,7 +393,7 @@ func TestComposedCatalogsAndSmokeUseInjectedClientOnlyForInference(t *testing.T)
|
|||||||
}
|
}
|
||||||
|
|
||||||
var output bytes.Buffer
|
var output bytes.Buffer
|
||||||
outcome := app.Execute(context.Background(), []string{"promptkit", "test", "all", "--campaign", "spring", "--prompt", "briefing", "--profile", "spring"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
outcome := app.Execute(context.Background(), []string{"promptkit", "test", "all", "--campaign", "spring", "--llm-prompt", "briefing", "--llm-profile", "spring"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
||||||
if output.Len() == 0 || !bytes.Contains(output.Bytes(), []byte("tests:")) {
|
if output.Len() == 0 || !bytes.Contains(output.Bytes(), []byte("tests:")) {
|
||||||
t.Fatalf("aggregate smoke report missing: outcome=%#v output=%q", outcome, output.String())
|
t.Fatalf("aggregate smoke report missing: outcome=%#v output=%q", outcome, output.String())
|
||||||
}
|
}
|
||||||
@@ -453,7 +453,7 @@ func TestComposedSmokeReportSurvivesInferenceFailure(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
var output, diagnostics bytes.Buffer
|
var output, diagnostics bytes.Buffer
|
||||||
outcome := app.Execute(context.Background(), []string{"promptkit", "test", "text_inference", "--campaign", "spring", "--prompt", "briefing", "--profile", "spring", "--verbose"}, commandkit.Streams{Out: &output, Err: &diagnostics})
|
outcome := app.Execute(context.Background(), []string{"promptkit", "test", "text_inference", "--campaign", "spring", "--llm-prompt", "briefing", "--llm-profile", "spring", "--verbose"}, commandkit.Streams{Out: &output, Err: &diagnostics})
|
||||||
if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || !errors.Is(outcome.Cause, wantCause) {
|
if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || !errors.Is(outcome.Cause, wantCause) {
|
||||||
t.Fatalf("smoke failure outcome = %#v; report %q; diagnostics %q", outcome, output.String(), diagnostics.String())
|
t.Fatalf("smoke failure outcome = %#v; report %q; diagnostics %q", outcome, output.String(), diagnostics.String())
|
||||||
}
|
}
|
||||||
@@ -543,7 +543,7 @@ func TestComposedAliasesReachCanonicalCommands(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
output.Reset()
|
output.Reset()
|
||||||
outcome = app.Execute(context.Background(), []string{"smoke", "text_inference", "--campaign", "spring", "--prompt", "briefing", "--profile", "spring"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
outcome = app.Execute(context.Background(), []string{"smoke", "text_inference", "--campaign", "spring", "--llm-prompt", "briefing", "--llm-profile", "spring"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
||||||
if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(output.String(), "kind: text_inference") || !strings.Contains(output.String(), "completed: true") || clientCalls != 1 {
|
if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(output.String(), "kind: text_inference") || !strings.Contains(output.String(), "completed: true") || clientCalls != 1 {
|
||||||
t.Fatalf("smoke shortcut outcome/report/client calls = (%#v, %q, %d)", outcome, output.String(), clientCalls)
|
t.Fatalf("smoke shortcut outcome/report/client calls = (%#v, %q, %d)", outcome, output.String(), clientCalls)
|
||||||
}
|
}
|
||||||
|
|||||||
196
integrations/ckprompt/request_flags.go
Normal file
196
integrations/ckprompt/request_flags.go
Normal file
@@ -0,0 +1,196 @@
|
|||||||
|
package ckprompt
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strconv"
|
||||||
|
|
||||||
|
commandkit "go.maximumdirect.net/commandkit"
|
||||||
|
promptkit "go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
// requestFlagDefinition keeps the canonical spellings, help text, and optional
|
||||||
|
// command-scoped aliases together for consumers of the standard flag sets.
|
||||||
|
type requestFlagDefinition struct {
|
||||||
|
meta commandkit.FlagMeta
|
||||||
|
aliases []string
|
||||||
|
}
|
||||||
|
|
||||||
|
var requestFlags = struct {
|
||||||
|
Endpoint requestFlagDefinition
|
||||||
|
Model requestFlagDefinition
|
||||||
|
Temperature requestFlagDefinition
|
||||||
|
MaxTokens requestFlagDefinition
|
||||||
|
TopP requestFlagDefinition
|
||||||
|
TimeoutSeconds requestFlagDefinition
|
||||||
|
ServiceTier requestFlagDefinition
|
||||||
|
ReasoningEffort requestFlagDefinition
|
||||||
|
APIKeyEnv requestFlagDefinition
|
||||||
|
TransientRetries requestFlagDefinition
|
||||||
|
ExtraParams requestFlagDefinition
|
||||||
|
Prompt requestFlagDefinition
|
||||||
|
Profile requestFlagDefinition
|
||||||
|
Backend requestFlagDefinition
|
||||||
|
SessionID requestFlagDefinition
|
||||||
|
Var requestFlagDefinition
|
||||||
|
InputText requestFlagDefinition
|
||||||
|
InputFile requestFlagDefinition
|
||||||
|
}{
|
||||||
|
Endpoint: requestFlagDefinition{commandkit.FlagMeta{Name: "llm-endpoint", Usage: "Execution endpoint override."}, []string{"endpoint"}},
|
||||||
|
Model: requestFlagDefinition{commandkit.FlagMeta{Name: "llm-model", Usage: "Execution model override."}, []string{"model"}},
|
||||||
|
Temperature: requestFlagDefinition{commandkit.FlagMeta{Name: "llm-temperature", Usage: "Execution temperature override."}, []string{"temperature"}},
|
||||||
|
MaxTokens: requestFlagDefinition{commandkit.FlagMeta{Name: "llm-max-tokens", Usage: "Execution maximum tokens override."}, []string{"max-tokens"}},
|
||||||
|
TopP: requestFlagDefinition{commandkit.FlagMeta{Name: "llm-top-p", Usage: "Execution top-p override."}, []string{"top-p"}},
|
||||||
|
TimeoutSeconds: requestFlagDefinition{commandkit.FlagMeta{Name: "llm-timeout-seconds", Usage: "Execution timeout override in seconds."}, []string{"timeout-seconds"}},
|
||||||
|
ServiceTier: requestFlagDefinition{commandkit.FlagMeta{Name: "llm-service-tier", Usage: "Execution service tier."}, []string{"service-tier"}},
|
||||||
|
ReasoningEffort: requestFlagDefinition{commandkit.FlagMeta{Name: "llm-reasoning-effort", Usage: "Execution reasoning effort override."}, []string{"reasoning-effort"}},
|
||||||
|
APIKeyEnv: requestFlagDefinition{commandkit.FlagMeta{Name: "llm-api-key-env", Usage: "Execution API key environment variable."}, []string{"api-key-env"}},
|
||||||
|
TransientRetries: requestFlagDefinition{commandkit.FlagMeta{Name: "llm-transient-retries", Usage: "Inference transient retry count."}, []string{"transient-retries"}},
|
||||||
|
ExtraParams: requestFlagDefinition{commandkit.FlagMeta{Name: "llm-extra-params", Usage: "Execution provider parameters as a JSON object."}, []string{"extra-params"}},
|
||||||
|
Prompt: requestFlagDefinition{commandkit.FlagMeta{Name: "llm-prompt", Usage: "PromptKit prompt ID."}, []string{"prompt"}},
|
||||||
|
Profile: requestFlagDefinition{commandkit.FlagMeta{Name: "llm-profile", Usage: "PromptKit profile ID."}, []string{"profile"}},
|
||||||
|
Backend: requestFlagDefinition{commandkit.FlagMeta{Name: "llm-backend", Usage: "PromptKit backend ID."}, []string{"backend"}},
|
||||||
|
SessionID: requestFlagDefinition{commandkit.FlagMeta{Name: "llm-session-id", Usage: "PromptKit session ID."}, []string{"session-id"}},
|
||||||
|
Var: requestFlagDefinition{commandkit.FlagMeta{Name: "llm-var", Usage: "Caller variable assignment KEY=VALUE."}, []string{"var"}},
|
||||||
|
InputText: requestFlagDefinition{commandkit.FlagMeta{Name: "llm-input-text", Usage: "Literal prompt input assignment NAME=TEXT."}, []string{"input-text", "llm-input", "input"}},
|
||||||
|
InputFile: requestFlagDefinition{commandkit.FlagMeta{Name: "llm-input-file", Usage: "Prompt input file assignment NAME=PATH."}, []string{"input-file"}},
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExecutionInput contains request-scoped execution controls supplied on the CLI.
|
||||||
|
// A nil override or retry count leaves the corresponding native default intact.
|
||||||
|
type ExecutionInput struct {
|
||||||
|
Execution *promptkit.ExecutionTargetOverride
|
||||||
|
TransientRetries *int
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExecutionFlags exposes independently selectable execution-control bindings.
|
||||||
|
type ExecutionFlags[I any] struct {
|
||||||
|
Endpoint, Model, Temperature, MaxTokens, TopP, TimeoutSeconds commandkit.Flag[I]
|
||||||
|
ServiceTier, ReasoningEffort, APIKeyEnv, TransientRetries, ExtraParams commandkit.Flag[I]
|
||||||
|
}
|
||||||
|
|
||||||
|
// All returns all execution controls.
|
||||||
|
func (f ExecutionFlags[I]) All() []commandkit.Flag[I] {
|
||||||
|
return []commandkit.Flag[I]{f.Endpoint, f.Model, f.Temperature, f.MaxTokens, f.TopP, f.TimeoutSeconds, f.ServiceTier, f.ReasoningEffort, f.APIKeyEnv, f.TransientRetries, f.ExtraParams}
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewExecutionFlags binds execution controls to a field of the caller's input.
|
||||||
|
func NewExecutionFlags[I any](destination func(*I) *ExecutionInput) ExecutionFlags[I] {
|
||||||
|
target := func(input *I) *promptkit.ExecutionTargetOverride {
|
||||||
|
value := destination(input)
|
||||||
|
if value.Execution == nil {
|
||||||
|
value.Execution = &promptkit.ExecutionTargetOverride{}
|
||||||
|
}
|
||||||
|
return value.Execution
|
||||||
|
}
|
||||||
|
return ExecutionFlags[I]{
|
||||||
|
Endpoint: executionFlag(requestFlags.Endpoint.meta, parseDefinitionString, func(i *I, v string) { target(i).Endpoint = v }),
|
||||||
|
Model: executionFlag(requestFlags.Model.meta, parseDefinitionString, func(i *I, v string) { target(i).Model = v }),
|
||||||
|
Temperature: executionFlag(requestFlags.Temperature.meta, parseExecutionFloat, func(i *I, v float64) { target(i).Temperature = &v }),
|
||||||
|
MaxTokens: executionFlag(requestFlags.MaxTokens.meta, strconv.Atoi, func(i *I, v int) { target(i).MaxTokens = &v }),
|
||||||
|
TopP: executionFlag(requestFlags.TopP.meta, parseExecutionFloat, func(i *I, v float64) { target(i).TopP = &v }),
|
||||||
|
TimeoutSeconds: executionFlag(requestFlags.TimeoutSeconds.meta, strconv.Atoi, func(i *I, v int) { target(i).TimeoutSeconds = &v }),
|
||||||
|
ServiceTier: executionFlag(requestFlags.ServiceTier.meta, parseDefinitionString, func(i *I, v string) { target(i).ServiceTier = v }),
|
||||||
|
ReasoningEffort: executionFlag(requestFlags.ReasoningEffort.meta, parseDefinitionString, func(i *I, v string) { target(i).ReasoningEffort = &v }),
|
||||||
|
APIKeyEnv: executionFlag(requestFlags.APIKeyEnv.meta, parseDefinitionString, func(i *I, v string) { target(i).APIKeyEnv = v }),
|
||||||
|
TransientRetries: executionFlag(requestFlags.TransientRetries.meta, strconv.Atoi, func(i *I, v int) { destination(i).TransientRetries = &v }),
|
||||||
|
ExtraParams: executionFlag(requestFlags.ExtraParams.meta, parseExtraParams, func(i *I, v map[string]any) { target(i).ExtraParams = v }),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func executionFlag[I, V any](meta commandkit.FlagMeta, parse func(string) (V, error), set func(*I, V)) commandkit.Flag[I] {
|
||||||
|
return commandkit.ValueFlag(meta, (*V)(nil), parse, func(i *I, value commandkit.Supplied[V]) {
|
||||||
|
if value.Set {
|
||||||
|
set(i, value.Value)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseExtraParams(raw string) (map[string]any, error) {
|
||||||
|
value, err := commandkit.ParseCLIJSON(raw)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
object, ok := value.(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("provider parameters must be a JSON object")
|
||||||
|
}
|
||||||
|
return object, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ResourceInput retains whether each resource identifier was explicitly supplied.
|
||||||
|
type ResourceInput struct {
|
||||||
|
Prompt, Profile, Backend, SessionID commandkit.Supplied[string]
|
||||||
|
}
|
||||||
|
|
||||||
|
// ResourceFlags exposes independently selectable resource and session bindings.
|
||||||
|
type ResourceFlags[I any] struct {
|
||||||
|
Prompt, Profile, Backend, SessionID commandkit.Flag[I]
|
||||||
|
}
|
||||||
|
|
||||||
|
// All returns all resource bindings.
|
||||||
|
func (f ResourceFlags[I]) All() []commandkit.Flag[I] {
|
||||||
|
return []commandkit.Flag[I]{f.Prompt, f.Profile, f.Backend, f.SessionID}
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewResourceFlags binds resource identifiers to a field of the caller's input.
|
||||||
|
func NewResourceFlags[I any](destination func(*I) *ResourceInput) ResourceFlags[I] {
|
||||||
|
flag := func(meta commandkit.FlagMeta, selectField func(*ResourceInput) *commandkit.Supplied[string]) commandkit.Flag[I] {
|
||||||
|
return commandkit.ValueFlag(meta, (*string)(nil), parseDefinitionString, func(i *I, v commandkit.Supplied[string]) { *selectField(destination(i)) = v })
|
||||||
|
}
|
||||||
|
return ResourceFlags[I]{
|
||||||
|
Prompt: flag(requestFlags.Prompt.meta, func(v *ResourceInput) *commandkit.Supplied[string] { return &v.Prompt }),
|
||||||
|
Profile: flag(requestFlags.Profile.meta, func(v *ResourceInput) *commandkit.Supplied[string] { return &v.Profile }),
|
||||||
|
Backend: flag(requestFlags.Backend.meta, func(v *ResourceInput) *commandkit.Supplied[string] { return &v.Backend }),
|
||||||
|
SessionID: flag(requestFlags.SessionID.meta, func(v *ResourceInput) *commandkit.Supplied[string] { return &v.SessionID }),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// PromptInputs contains caller variables and native artifact references.
|
||||||
|
type PromptInputs struct {
|
||||||
|
Vars map[string]string
|
||||||
|
Inputs map[string]promptkit.ArtifactRef
|
||||||
|
}
|
||||||
|
|
||||||
|
// PromptInputFlags exposes variable and named artifact input bindings.
|
||||||
|
type PromptInputFlags[I any] struct {
|
||||||
|
Variable, Text, File commandkit.Flag[I]
|
||||||
|
}
|
||||||
|
|
||||||
|
// All returns all variable and input bindings.
|
||||||
|
func (f PromptInputFlags[I]) All() []commandkit.Flag[I] {
|
||||||
|
return []commandkit.Flag[I]{f.Variable, f.Text, f.File}
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewPromptInputFlags binds caller variables and input references to a field of
|
||||||
|
// the caller's input.
|
||||||
|
func NewPromptInputFlags[I any](destination func(*I) *PromptInputs) PromptInputFlags[I] {
|
||||||
|
variable := commandkit.RepeatFlag(requestFlags.Var.meta, []variableAssignment(nil), parseVariableAssignment, func(i *I, v commandkit.Supplied[[]variableAssignment]) {
|
||||||
|
if !v.Set {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
values := make(map[string]string, len(v.Value))
|
||||||
|
for _, assignment := range v.Value {
|
||||||
|
values[assignment.key] = assignment.value
|
||||||
|
}
|
||||||
|
destination(i).Vars = values
|
||||||
|
})
|
||||||
|
inputFlag := func(meta commandkit.FlagMeta, makeRef func(string) promptkit.ArtifactRef) commandkit.Flag[I] {
|
||||||
|
return commandkit.RepeatFlag(meta, []inputAssignment(nil), parseInputAssignment, func(i *I, v commandkit.Supplied[[]inputAssignment]) {
|
||||||
|
if !v.Set {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
input := destination(i)
|
||||||
|
if input.Inputs == nil {
|
||||||
|
input.Inputs = make(map[string]promptkit.ArtifactRef)
|
||||||
|
}
|
||||||
|
for _, assignment := range v.Value {
|
||||||
|
input.Inputs[assignment.name] = makeRef(assignment.path)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return PromptInputFlags[I]{Variable: variable, Text: inputFlag(requestFlags.InputText.meta, promptkit.Inline), File: inputFlag(requestFlags.InputFile.meta, promptkit.File)}
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseExecutionFloat(value string) (float64, error) {
|
||||||
|
return strconv.ParseFloat(value, 64)
|
||||||
|
}
|
||||||
78
integrations/ckprompt/request_flags_test.go
Normal file
78
integrations/ckprompt/request_flags_test.go
Normal file
@@ -0,0 +1,78 @@
|
|||||||
|
package ckprompt_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"io"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/ckprompt"
|
||||||
|
commandkit "go.maximumdirect.net/commandkit"
|
||||||
|
promptkit "go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestReusableRequestFlagsOnApplicationCommand(t *testing.T) {
|
||||||
|
type input struct {
|
||||||
|
execution ckprompt.ExecutionInput
|
||||||
|
resources ckprompt.ResourceInput
|
||||||
|
prompt ckprompt.PromptInputs
|
||||||
|
}
|
||||||
|
execution := ckprompt.NewExecutionFlags(func(i *input) *ckprompt.ExecutionInput { return &i.execution })
|
||||||
|
resources := ckprompt.NewResourceFlags(func(i *input) *ckprompt.ResourceInput { return &i.resources })
|
||||||
|
inputs := ckprompt.NewPromptInputFlags(func(i *input) *ckprompt.PromptInputs { return &i.prompt })
|
||||||
|
flags := append(execution.All(), resources.Profile, inputs.Variable)
|
||||||
|
var received input
|
||||||
|
calls := 0
|
||||||
|
command := commandkit.NewCommand(commandkit.CommandSpec[input]{
|
||||||
|
Name: "run", Flags: flags,
|
||||||
|
Run: func(_ commandkit.Invocation, i input) commandkit.Outcome {
|
||||||
|
calls++
|
||||||
|
received = i
|
||||||
|
return commandkit.Success()
|
||||||
|
},
|
||||||
|
})
|
||||||
|
app, err := commandkit.New(commandkit.Identity{Name: "app"}, command)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
run := func(args ...string) commandkit.Outcome {
|
||||||
|
return app.Execute(context.Background(), append([]string{"run"}, args...), commandkit.Streams{Out: io.Discard, Err: io.Discard})
|
||||||
|
}
|
||||||
|
if result := run(); result.Kind != commandkit.OutcomeSuccess || received.execution.Execution != nil || received.execution.TransientRetries != nil || received.resources.Profile.Set {
|
||||||
|
t.Fatalf("absent values: %#v %#v", result, received)
|
||||||
|
}
|
||||||
|
if result := run("--llm-profile=", "--llm-temperature=0", "--llm-reasoning-effort=", "--llm-transient-retries=0", "--llm-extra-params={\"n\":1,\"d\":1.5}", "--llm-var=x="); result.Kind != commandkit.OutcomeSuccess {
|
||||||
|
t.Fatal(result)
|
||||||
|
}
|
||||||
|
e := received.execution
|
||||||
|
if !received.resources.Profile.Set || received.resources.Profile.Value != "" || e.Execution == nil || e.Execution.Temperature == nil || *e.Execution.Temperature != 0 || e.Execution.ReasoningEffort == nil || *e.Execution.ReasoningEffort != "" || e.TransientRetries == nil || *e.TransientRetries != 0 || e.Execution.ExtraParams["n"] != int64(1) || e.Execution.ExtraParams["d"] != float64(1.5) || received.prompt.Vars["x"] != "" {
|
||||||
|
t.Fatalf("explicit values: %#v", received)
|
||||||
|
}
|
||||||
|
if result := run("--llm-extra-params={}"); result.Kind != commandkit.OutcomeSuccess || received.execution.Execution == nil || received.execution.Execution.ExtraParams == nil || len(received.execution.Execution.ExtraParams) != 0 {
|
||||||
|
t.Fatalf("empty object: %#v %#v", result, received)
|
||||||
|
}
|
||||||
|
for _, raw := range []string{"null", "[]", "1", "{invalid}"} {
|
||||||
|
before := calls
|
||||||
|
if result := run("--llm-extra-params=" + raw); result.Kind != commandkit.OutcomeUsage || calls != before {
|
||||||
|
t.Fatalf("invalid object %q: %#v", raw, result)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if result := run("--llm-backend=unused"); result.Kind != commandkit.OutcomeUsage {
|
||||||
|
t.Fatalf("unselected flag: %#v", result)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidationForwardsExecutionControls(t *testing.T) {
|
||||||
|
var received promptkit.RunRequest
|
||||||
|
suite := ckprompt.Suite[struct{}, struct{}]{ValidateRequest: func(_ context.Context, req ckprompt.ValidationRequest, _, _ struct{}) (commandkit.Acquired[struct{}], error) {
|
||||||
|
received = req.Request
|
||||||
|
return commandkit.Acquired[struct{}]{}, nil
|
||||||
|
}}
|
||||||
|
app, err := commandkit.New(commandkit.Identity{Name: "app"}, ckprompt.Command(suite))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
result := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing", "--llm-profile=", "--llm-max-tokens=0", "--llm-extra-params={}", "--llm-transient-retries=0"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
|
||||||
|
if result.Kind != commandkit.OutcomeSuccess || received.Execution == nil || received.Execution.MaxTokens == nil || *received.Execution.MaxTokens != 0 || received.Execution.ExtraParams == nil || received.TransientRetries == nil || *received.TransientRetries != 0 {
|
||||||
|
t.Fatalf("result/request = %#v %#v", result, received)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -108,17 +108,17 @@ func TestRequirementsCommandForwardsNativeRequestAndProjectsInspection(t *testin
|
|||||||
{
|
{
|
||||||
name: "conditional discovery",
|
name: "conditional discovery",
|
||||||
request: promptkit.RequirementInspectionRequest{PromptID: "briefing", ProfileID: "editorial"},
|
request: promptkit.RequirementInspectionRequest{PromptID: "briefing", ProfileID: "editorial"},
|
||||||
args: []string{"promptkit", "requirements", "show", "briefing", "--profile", "editorial"},
|
args: []string{"promptkit", "requirements", "show", "briefing", "--llm-profile", "editorial"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "supplied branch variable",
|
name: "supplied branch variable",
|
||||||
request: promptkit.RequirementInspectionRequest{PromptID: "briefing", ProfileID: "editorial", Vars: map[string]string{"mode": "full"}},
|
request: promptkit.RequirementInspectionRequest{PromptID: "briefing", ProfileID: "editorial", Vars: map[string]string{"mode": "full"}},
|
||||||
args: []string{"promptkit", "requirements", "show", "briefing", "--profile", "editorial", "--var", "mode=full", "--format", "yaml"},
|
args: []string{"promptkit", "requirements", "show", "briefing", "--llm-profile", "editorial", "--llm-var", "mode=full", "--format", "yaml"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "prompt with a required variable",
|
name: "prompt with a required variable",
|
||||||
request: promptkit.RequirementInspectionRequest{PromptID: "required-variable", ProfileID: "editorial"},
|
request: promptkit.RequirementInspectionRequest{PromptID: "required-variable", ProfileID: "editorial"},
|
||||||
args: []string{"promptkit", "requirements", "show", "required-variable", "--profile", "editorial"},
|
args: []string{"promptkit", "requirements", "show", "required-variable", "--llm-profile", "editorial"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "default profile selection",
|
name: "default profile selection",
|
||||||
@@ -128,22 +128,22 @@ func TestRequirementsCommandForwardsNativeRequestAndProjectsInspection(t *testin
|
|||||||
{
|
{
|
||||||
name: "first distinct prompt ID",
|
name: "first distinct prompt ID",
|
||||||
request: promptkit.RequirementInspectionRequest{PromptID: "ambiguous-first", ProfileID: "editorial"},
|
request: promptkit.RequirementInspectionRequest{PromptID: "ambiguous-first", ProfileID: "editorial"},
|
||||||
args: []string{"promptkit", "requirements", "show", "ambiguous-first", "--profile", "editorial"},
|
args: []string{"promptkit", "requirements", "show", "ambiguous-first", "--llm-profile", "editorial"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "second distinct prompt ID",
|
name: "second distinct prompt ID",
|
||||||
request: promptkit.RequirementInspectionRequest{PromptID: "ambiguous-second", ProfileID: "editorial"},
|
request: promptkit.RequirementInspectionRequest{PromptID: "ambiguous-second", ProfileID: "editorial"},
|
||||||
args: []string{"promptkit", "requirements", "show", "ambiguous-second", "--profile", "editorial"},
|
args: []string{"promptkit", "requirements", "show", "ambiguous-second", "--llm-profile", "editorial"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "session template context",
|
name: "session template context",
|
||||||
request: promptkit.RequirementInspectionRequest{PromptID: "briefing", ProfileID: "editorial"},
|
request: promptkit.RequirementInspectionRequest{PromptID: "briefing", ProfileID: "editorial"},
|
||||||
args: []string{"promptkit", "requirements", "show", "briefing", "--profile", "editorial"},
|
args: []string{"promptkit", "requirements", "show", "briefing", "--llm-profile", "editorial"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "session ID context",
|
name: "session ID context",
|
||||||
request: promptkit.RequirementInspectionRequest{PromptID: "briefing", ProfileID: "editorial", SessionID: "session-42"},
|
request: promptkit.RequirementInspectionRequest{PromptID: "briefing", ProfileID: "editorial", SessionID: "session-42"},
|
||||||
args: []string{"promptkit", "requirements", "show", "briefing", "--profile", "editorial", "--session-id", "session-42"},
|
args: []string{"promptkit", "requirements", "show", "briefing", "--llm-profile", "editorial", "--llm-session-id", "session-42"},
|
||||||
},
|
},
|
||||||
} {
|
} {
|
||||||
t.Run(test.name, func(t *testing.T) {
|
t.Run(test.name, func(t *testing.T) {
|
||||||
@@ -164,17 +164,17 @@ func TestRequirementsCommandPreservesNativeSelectionErrors(t *testing.T) {
|
|||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "missing prompt",
|
name: "missing prompt",
|
||||||
args: []string{"promptkit", "requirements", "show", "missing", "--profile", "editorial"},
|
args: []string{"promptkit", "requirements", "show", "missing", "--llm-profile", "editorial"},
|
||||||
cause: promptkit.ErrPromptNotFound,
|
cause: promptkit.ErrPromptNotFound,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "missing profile",
|
name: "missing profile",
|
||||||
args: []string{"promptkit", "requirements", "show", "briefing", "--profile", "missing"},
|
args: []string{"promptkit", "requirements", "show", "briefing", "--llm-profile", "missing"},
|
||||||
cause: promptkit.ErrProfileNotFound,
|
cause: promptkit.ErrProfileNotFound,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "removed prompt version flag",
|
name: "removed prompt version flag",
|
||||||
args: []string{"promptkit", "requirements", "show", "briefing", "--version", "2", "--profile", "editorial"},
|
args: []string{"promptkit", "requirements", "show", "briefing", "--version", "2", "--llm-profile", "editorial"},
|
||||||
cause: nil,
|
cause: nil,
|
||||||
},
|
},
|
||||||
} {
|
} {
|
||||||
|
|||||||
@@ -72,7 +72,7 @@ func TestSmokeAggregateCoordinatesConcreteRequestsAndRetainsErrors(t *testing.T)
|
|||||||
app := newPromptTestApp(t, Command(suite))
|
app := newPromptTestApp(t, Command(suite))
|
||||||
var output bytes.Buffer
|
var output bytes.Buffer
|
||||||
outcome := app.Execute(context.Background(), []string{
|
outcome := app.Execute(context.Background(), []string{
|
||||||
"promptkit", "test", "all", "--model=base-model", "--temperature=0.25", "--transient-retries=3",
|
"promptkit", "test", "all", "--llm-model=base-model", "--llm-temperature=0.25", "--llm-transient-retries=3",
|
||||||
}, commandkit.Streams{Out: &output, Err: io.Discard})
|
}, commandkit.Streams{Out: &output, Err: io.Discard})
|
||||||
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, nativeErr) {
|
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, nativeErr) {
|
||||||
t.Fatalf("aggregate outcome = %#v", outcome)
|
t.Fatalf("aggregate outcome = %#v", outcome)
|
||||||
|
|||||||
@@ -4,7 +4,6 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"strconv"
|
|
||||||
|
|
||||||
"go.maximumdirect.net/ckconfig"
|
"go.maximumdirect.net/ckconfig"
|
||||||
commandkit "go.maximumdirect.net/commandkit"
|
commandkit "go.maximumdirect.net/commandkit"
|
||||||
@@ -15,6 +14,8 @@ type smokeCommandInput[I any] struct {
|
|||||||
selection ckconfig.Selection
|
selection ckconfig.Selection
|
||||||
kind string
|
kind string
|
||||||
request promptkit.SmokeTestRequest
|
request promptkit.SmokeTestRequest
|
||||||
|
execution ExecutionInput
|
||||||
|
resources ResourceInput
|
||||||
captured commandkit.FlagValues[I]
|
captured commandkit.FlagValues[I]
|
||||||
selected I
|
selected I
|
||||||
}
|
}
|
||||||
@@ -35,98 +36,13 @@ func newSmokeTestCommand[I, S any](suite Suite[I, S]) commandkit.Command {
|
|||||||
func(input **smokeCommandInput[I], value commandkit.Supplied[string]) {
|
func(input **smokeCommandInput[I], value commandkit.Supplied[string]) {
|
||||||
(*input).selection.Overlay = value
|
(*input).selection.Overlay = value
|
||||||
}),
|
}),
|
||||||
commandkit.ValueFlag(commandkit.FlagMeta{Name: "backend", Usage: "PromptKit backend ID."}, (*string)(nil), parseDefinitionString,
|
|
||||||
func(input **smokeCommandInput[I], value commandkit.Supplied[string]) {
|
|
||||||
(*input).request.BackendID = value.Value
|
|
||||||
}),
|
|
||||||
commandkit.ValueFlag(commandkit.FlagMeta{Name: "prompt", Usage: "PromptKit prompt ID."}, (*string)(nil), parseDefinitionString,
|
|
||||||
func(input **smokeCommandInput[I], value commandkit.Supplied[string]) {
|
|
||||||
(*input).request.PromptID = value.Value
|
|
||||||
}),
|
|
||||||
commandkit.ValueFlag(commandkit.FlagMeta{Name: "profile", Usage: "PromptKit profile ID."}, (*string)(nil), parseDefinitionString,
|
|
||||||
func(input **smokeCommandInput[I], value commandkit.Supplied[string]) {
|
|
||||||
(*input).request.ProfileID = value.Value
|
|
||||||
}),
|
|
||||||
commandkit.ValueFlag(commandkit.FlagMeta{Name: "endpoint", Usage: "Execution endpoint override."}, (*string)(nil), parseDefinitionString,
|
|
||||||
func(input **smokeCommandInput[I], value commandkit.Supplied[string]) {
|
|
||||||
if !value.Set {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
execution := smokeExecution(input)
|
|
||||||
execution.Endpoint = value.Value
|
|
||||||
}),
|
|
||||||
commandkit.ValueFlag(commandkit.FlagMeta{Name: "model", Usage: "Execution model override."}, (*string)(nil), parseDefinitionString,
|
|
||||||
func(input **smokeCommandInput[I], value commandkit.Supplied[string]) {
|
|
||||||
if !value.Set {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
execution := smokeExecution(input)
|
|
||||||
execution.Model = value.Value
|
|
||||||
}),
|
|
||||||
commandkit.ValueFlag(commandkit.FlagMeta{Name: "temperature", Usage: "Execution temperature override."}, (*float64)(nil), parseSmokeFloat,
|
|
||||||
func(input **smokeCommandInput[I], value commandkit.Supplied[float64]) {
|
|
||||||
if !value.Set {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
execution := smokeExecution(input)
|
|
||||||
execution.Temperature = &value.Value
|
|
||||||
}),
|
|
||||||
commandkit.ValueFlag(commandkit.FlagMeta{Name: "max-tokens", Usage: "Execution maximum tokens override."}, (*int)(nil), strconv.Atoi,
|
|
||||||
func(input **smokeCommandInput[I], value commandkit.Supplied[int]) {
|
|
||||||
if !value.Set {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
execution := smokeExecution(input)
|
|
||||||
execution.MaxTokens = &value.Value
|
|
||||||
}),
|
|
||||||
commandkit.ValueFlag(commandkit.FlagMeta{Name: "top-p", Usage: "Execution top-p override."}, (*float64)(nil), parseSmokeFloat,
|
|
||||||
func(input **smokeCommandInput[I], value commandkit.Supplied[float64]) {
|
|
||||||
if !value.Set {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
execution := smokeExecution(input)
|
|
||||||
execution.TopP = &value.Value
|
|
||||||
}),
|
|
||||||
commandkit.ValueFlag(commandkit.FlagMeta{Name: "timeout-seconds", Usage: "Execution timeout override."}, (*int)(nil), strconv.Atoi,
|
|
||||||
func(input **smokeCommandInput[I], value commandkit.Supplied[int]) {
|
|
||||||
if !value.Set {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
execution := smokeExecution(input)
|
|
||||||
execution.TimeoutSeconds = &value.Value
|
|
||||||
}),
|
|
||||||
commandkit.ValueFlag(commandkit.FlagMeta{Name: "service-tier", Usage: "Execution service tier."}, (*string)(nil), parseDefinitionString,
|
|
||||||
func(input **smokeCommandInput[I], value commandkit.Supplied[string]) {
|
|
||||||
if !value.Set {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
execution := smokeExecution(input)
|
|
||||||
execution.ServiceTier = value.Value
|
|
||||||
}),
|
|
||||||
commandkit.ValueFlag(commandkit.FlagMeta{Name: "reasoning-effort", Usage: "Execution reasoning effort override."}, (*string)(nil), parseDefinitionString,
|
|
||||||
func(input **smokeCommandInput[I], value commandkit.Supplied[string]) {
|
|
||||||
if !value.Set {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
execution := smokeExecution(input)
|
|
||||||
execution.ReasoningEffort = &value.Value
|
|
||||||
}),
|
|
||||||
commandkit.ValueFlag(commandkit.FlagMeta{Name: "api-key-env", Usage: "Execution API key environment variable."}, (*string)(nil), parseDefinitionString,
|
|
||||||
func(input **smokeCommandInput[I], value commandkit.Supplied[string]) {
|
|
||||||
if !value.Set {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
execution := smokeExecution(input)
|
|
||||||
execution.APIKeyEnv = value.Value
|
|
||||||
}),
|
|
||||||
commandkit.ValueFlag(commandkit.FlagMeta{Name: "transient-retries", Usage: "Inference transient retry count."}, (*int)(nil), strconv.Atoi,
|
|
||||||
func(input **smokeCommandInput[I], value commandkit.Supplied[int]) {
|
|
||||||
if value.Set {
|
|
||||||
(*input).request.TransientRetries = &value.Value
|
|
||||||
}
|
|
||||||
}),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
resources := NewResourceFlags(func(input **smokeCommandInput[I]) *ResourceInput { return &(*input).resources })
|
||||||
|
execution := NewExecutionFlags(func(input **smokeCommandInput[I]) *ExecutionInput { return &(*input).execution })
|
||||||
|
flags = append(flags, resources.Backend, resources.Prompt, resources.Profile)
|
||||||
|
flags = append(flags, execution.All()...)
|
||||||
|
|
||||||
return commandkit.NewCommand(commandkit.CommandSpec[*smokeCommandInput[I]]{
|
return commandkit.NewCommand(commandkit.CommandSpec[*smokeCommandInput[I]]{
|
||||||
Name: "test",
|
Name: "test",
|
||||||
Summary: "Run one PromptKit smoke test.",
|
Summary: "Run one PromptKit smoke test.",
|
||||||
@@ -142,6 +58,11 @@ func newSmokeTestCommand[I, S any](suite Suite[I, S]) commandkit.Command {
|
|||||||
if input == nil {
|
if input == nil {
|
||||||
return errors.New("PromptKit smoke-test input must not be nil")
|
return errors.New("PromptKit smoke-test input must not be nil")
|
||||||
}
|
}
|
||||||
|
input.request.BackendID = input.resources.Backend.Value
|
||||||
|
input.request.PromptID = input.resources.Prompt.Value
|
||||||
|
input.request.ProfileID = input.resources.Profile.Value
|
||||||
|
input.request.Execution = input.execution.Execution
|
||||||
|
input.request.TransientRetries = input.execution.TransientRetries
|
||||||
if input.kind != "all" {
|
if input.kind != "all" {
|
||||||
input.request.Kind = promptkit.SmokeTestKind(input.kind)
|
input.request.Kind = promptkit.SmokeTestKind(input.kind)
|
||||||
}
|
}
|
||||||
@@ -158,17 +79,6 @@ func newSmokeTestCommand[I, S any](suite Suite[I, S]) commandkit.Command {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func smokeExecution[I any](input **smokeCommandInput[I]) *promptkit.ExecutionTargetOverride {
|
|
||||||
if (*input).request.Execution == nil {
|
|
||||||
(*input).request.Execution = &promptkit.ExecutionTargetOverride{}
|
|
||||||
}
|
|
||||||
return (*input).request.Execution
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseSmokeFloat(value string) (float64, error) {
|
|
||||||
return strconv.ParseFloat(value, 64)
|
|
||||||
}
|
|
||||||
|
|
||||||
func runSmokeTest[I, S any](invocation commandkit.Invocation, suite Suite[I, S], input *smokeCommandInput[I]) commandkit.Outcome {
|
func runSmokeTest[I, S any](invocation commandkit.Invocation, suite Suite[I, S], input *smokeCommandInput[I]) commandkit.Outcome {
|
||||||
if err := promptKitContextError(invocation.Context); err != nil {
|
if err := promptKitContextError(invocation.Context); err != nil {
|
||||||
return definitionFailure("PromptKit smoke testing was canceled.", commandkit.Explanation{}, err)
|
return definitionFailure("PromptKit smoke testing was canceled.", commandkit.Explanation{}, err)
|
||||||
|
|||||||
@@ -53,9 +53,9 @@ func TestSmokeTestCommandBindsRequestAndRendersFindings(t *testing.T) {
|
|||||||
var output bytes.Buffer
|
var output bytes.Buffer
|
||||||
outcome = app.Execute(context.Background(), []string{
|
outcome = app.Execute(context.Background(), []string{
|
||||||
"promptkit", "test", "authentication", "--config=config.yml", "--overlay=overlay.yml", "--campaign=spring",
|
"promptkit", "test", "authentication", "--config=config.yml", "--overlay=overlay.yml", "--campaign=spring",
|
||||||
"--backend=local", "--prompt=briefing", "--profile=editorial", "--endpoint=https://example.invalid",
|
"--llm-backend=local", "--llm-prompt=briefing", "--llm-profile=editorial", "--llm-endpoint=https://example.invalid",
|
||||||
"--model=fixture", "--temperature=0", "--max-tokens=0", "--top-p=0", "--timeout-seconds=0",
|
"--llm-model=fixture", "--llm-temperature=0", "--llm-max-tokens=0", "--llm-top-p=0", "--llm-timeout-seconds=0",
|
||||||
"--service-tier=", "--reasoning-effort=", "--api-key-env=TOKEN", "--transient-retries=0",
|
"--llm-service-tier=", "--llm-reasoning-effort=", "--llm-api-key-env=TOKEN", "--llm-transient-retries=0",
|
||||||
}, commandkit.Streams{Out: &output, Err: io.Discard})
|
}, commandkit.Streams{Out: &output, Err: io.Discard})
|
||||||
if outcome.Kind != commandkit.OutcomeSuccess || providerCalls != 1 {
|
if outcome.Kind != commandkit.OutcomeSuccess || providerCalls != 1 {
|
||||||
t.Fatalf("smoke outcome = (%#v, %q), calls=%d", outcome, output.String(), providerCalls)
|
t.Fatalf("smoke outcome = (%#v, %q), calls=%d", outcome, output.String(), providerCalls)
|
||||||
|
|||||||
@@ -37,9 +37,9 @@ func TestSmokeTestCommandRetainsSingletonValuesAndResetsInvocations(t *testing.T
|
|||||||
first := app.Execute(context.Background(), []string{
|
first := app.Execute(context.Background(), []string{
|
||||||
"promptkit", "test", "authentication", "--config=old.yml", "--config=final.yml",
|
"promptkit", "test", "authentication", "--config=old.yml", "--config=final.yml",
|
||||||
"--overlay=old-overlay.yml", "--overlay=final-overlay.yml", "--campaign=old", "--campaign=final",
|
"--overlay=old-overlay.yml", "--overlay=final-overlay.yml", "--campaign=old", "--campaign=final",
|
||||||
"--backend=old-backend", "--backend=final-backend", "--prompt=old-prompt", "--prompt=final-prompt",
|
"--llm-backend=old-backend", "--llm-backend=final-backend", "--llm-prompt=old-prompt", "--llm-prompt=final-prompt",
|
||||||
"--profile=old-profile", "--profile=final-profile", "--model=old-model", "--model=final-model",
|
"--llm-profile=old-profile", "--llm-profile=final-profile", "--llm-model=old-model", "--llm-model=final-model",
|
||||||
"--temperature=0", "--transient-retries=0",
|
"--llm-temperature=0", "--llm-transient-retries=0",
|
||||||
}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
|
}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
|
||||||
if first.Kind != commandkit.OutcomeSuccess || len(requests) != 1 {
|
if first.Kind != commandkit.OutcomeSuccess || len(requests) != 1 {
|
||||||
t.Fatalf("first smoke invocation = %#v, requests=%#v", first, requests)
|
t.Fatalf("first smoke invocation = %#v, requests=%#v", first, requests)
|
||||||
@@ -73,11 +73,11 @@ func TestSmokeTestCommandUsageAndInputFailuresStopProviderWork(t *testing.T) {
|
|||||||
}{
|
}{
|
||||||
{name: "missing kind", args: []string{"promptkit", "test"}},
|
{name: "missing kind", args: []string{"promptkit", "test"}},
|
||||||
{name: "extra operand", args: []string{"promptkit", "test", "authentication", "extra"}},
|
{name: "extra operand", args: []string{"promptkit", "test", "authentication", "extra"}},
|
||||||
{name: "malformed temperature", args: []string{"promptkit", "test", "authentication", "--temperature=warm"}},
|
{name: "malformed temperature", args: []string{"promptkit", "test", "authentication", "--llm-temperature=warm"}},
|
||||||
{name: "malformed token count", args: []string{"promptkit", "test", "authentication", "--max-tokens=many"}},
|
{name: "malformed token count", args: []string{"promptkit", "test", "authentication", "--llm-max-tokens=many"}},
|
||||||
{name: "malformed top-p", args: []string{"promptkit", "test", "authentication", "--top-p=high"}},
|
{name: "malformed top-p", args: []string{"promptkit", "test", "authentication", "--llm-top-p=high"}},
|
||||||
{name: "malformed timeout", args: []string{"promptkit", "test", "authentication", "--timeout-seconds=soon"}},
|
{name: "malformed timeout", args: []string{"promptkit", "test", "authentication", "--llm-timeout-seconds=soon"}},
|
||||||
{name: "malformed retries", args: []string{"promptkit", "test", "authentication", "--transient-retries=several"}},
|
{name: "malformed retries", args: []string{"promptkit", "test", "authentication", "--llm-transient-retries=several"}},
|
||||||
{name: "unsupported format", args: []string{"promptkit", "test", "authentication", "--format=invalid"}},
|
{name: "unsupported format", args: []string{"promptkit", "test", "authentication", "--format=invalid"}},
|
||||||
{name: "unknown flag", args: []string{"promptkit", "test", "authentication", "--unknown=value"}},
|
{name: "unknown flag", args: []string{"promptkit", "test", "authentication", "--unknown=value"}},
|
||||||
} {
|
} {
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package ckprompt
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
"io"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"testing/fstest"
|
"testing/fstest"
|
||||||
@@ -331,3 +332,25 @@ func TestWithRequestValidationCancellationBoundaries(t *testing.T) {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestValidationBuilderReplacesCLIExecutionControls(t *testing.T) {
|
||||||
|
engine := adapterEngine(t)
|
||||||
|
calls := 0
|
||||||
|
suite := WithRequestValidation(Suite[struct{}, struct{}]{},
|
||||||
|
func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) {
|
||||||
|
return engine, nil
|
||||||
|
},
|
||||||
|
func(_ context.Context, seed ValidationRequest, _, _ struct{}) (promptkit.RunRequest, error) {
|
||||||
|
calls++
|
||||||
|
if seed.Request.Execution == nil || seed.Request.Execution.MaxTokens == nil || *seed.Request.Execution.MaxTokens != -1 {
|
||||||
|
t.Fatalf("builder seed = %#v", seed.Request)
|
||||||
|
}
|
||||||
|
return promptkit.RunRequest{PromptID: "empty", ProfileID: "default"}, nil
|
||||||
|
},
|
||||||
|
)
|
||||||
|
app := newPromptTestApp(t, Command(suite))
|
||||||
|
result := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "empty", "--llm-max-tokens=-1"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
|
||||||
|
if result.Kind != commandkit.OutcomeSuccess || calls != 1 {
|
||||||
|
t.Fatalf("builder replacement = %#v, calls = %d", result, calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -5,12 +5,14 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
commandkit "go.maximumdirect.net/commandkit"
|
commandkit "go.maximumdirect.net/commandkit"
|
||||||
promptkit "go.maximumdirect.net/promptkit"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type validationCommandInput[I any] struct {
|
type validationCommandInput[I any] struct {
|
||||||
promptID string
|
promptID string
|
||||||
request ValidationRequest
|
request ValidationRequest
|
||||||
|
execution ExecutionInput
|
||||||
|
resources ResourceInput
|
||||||
|
inputs PromptInputs
|
||||||
captured commandkit.FlagValues[I]
|
captured commandkit.FlagValues[I]
|
||||||
selected I
|
selected I
|
||||||
}
|
}
|
||||||
@@ -36,32 +38,14 @@ func newValidationCommand[I, S any](suite Suite[I, S]) commandkit.Command {
|
|||||||
func(input **validationCommandInput[I], value commandkit.Supplied[string]) {
|
func(input **validationCommandInput[I], value commandkit.Supplied[string]) {
|
||||||
(*input).request.Selection.Overlay = value
|
(*input).request.Selection.Overlay = value
|
||||||
}),
|
}),
|
||||||
commandkit.ValueFlag(commandkit.FlagMeta{Name: "profile", Usage: "Profile identity for validation."}, stringDefault(""), parseDefinitionString,
|
|
||||||
func(input **validationCommandInput[I], value commandkit.Supplied[string]) {
|
|
||||||
(*input).request.Request.ProfileID = value.Value
|
|
||||||
}),
|
|
||||||
commandkit.ValueFlag(commandkit.FlagMeta{Name: "session-id", Usage: "Session context for validation."}, stringDefault(""), parseDefinitionString,
|
|
||||||
func(input **validationCommandInput[I], value commandkit.Supplied[string]) {
|
|
||||||
(*input).request.Request.SessionID = value.Value
|
|
||||||
}),
|
|
||||||
commandkit.RepeatFlag(commandkit.FlagMeta{Name: "var", Usage: "Caller variable assignment KEY=VALUE."}, []variableAssignment(nil), parseVariableAssignment,
|
|
||||||
func(input **validationCommandInput[I], value commandkit.Supplied[[]variableAssignment]) {
|
|
||||||
variables := make(map[string]string, len(value.Value))
|
|
||||||
for _, assignment := range value.Value {
|
|
||||||
variables[assignment.key] = assignment.value
|
|
||||||
}
|
|
||||||
(*input).request.Request.Vars = variables
|
|
||||||
}),
|
|
||||||
commandkit.RepeatFlag(commandkit.FlagMeta{Name: "input", Usage: "Prompt input file assignment NAME=PATH."}, []inputAssignment(nil), parseInputAssignment,
|
|
||||||
func(input **validationCommandInput[I], value commandkit.Supplied[[]inputAssignment]) {
|
|
||||||
inputs := make(map[string]promptkit.ArtifactRef, len(value.Value))
|
|
||||||
for _, assignment := range value.Value {
|
|
||||||
inputs[assignment.name] = promptkit.File(assignment.path)
|
|
||||||
}
|
|
||||||
(*input).request.Request.Inputs = inputs
|
|
||||||
}),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
resources := NewResourceFlags(func(input **validationCommandInput[I]) *ResourceInput { return &(*input).resources })
|
||||||
|
execution := NewExecutionFlags(func(input **validationCommandInput[I]) *ExecutionInput { return &(*input).execution })
|
||||||
|
inputs := NewPromptInputFlags(func(input **validationCommandInput[I]) *PromptInputs { return &(*input).inputs })
|
||||||
|
flags = append(flags, resources.Profile, resources.SessionID, inputs.Variable, inputs.Text, inputs.File)
|
||||||
|
flags = append(flags, execution.All()...)
|
||||||
|
|
||||||
return commandkit.NewCommand(commandkit.CommandSpec[*validationCommandInput[I]]{
|
return commandkit.NewCommand(commandkit.CommandSpec[*validationCommandInput[I]]{
|
||||||
Name: "validate",
|
Name: "validate",
|
||||||
Summary: "Validate a concrete PromptKit request without executing it.",
|
Summary: "Validate a concrete PromptKit request without executing it.",
|
||||||
@@ -80,6 +64,12 @@ func newValidationCommand[I, S any](suite Suite[I, S]) commandkit.Command {
|
|||||||
if input == nil {
|
if input == nil {
|
||||||
return errors.New("PromptKit validation input must not be nil")
|
return errors.New("PromptKit validation input must not be nil")
|
||||||
}
|
}
|
||||||
|
input.request.Request.ProfileID = input.resources.Profile.Value
|
||||||
|
input.request.Request.SessionID = input.resources.SessionID.Value
|
||||||
|
input.request.Request.Vars = input.inputs.Vars
|
||||||
|
input.request.Request.Inputs = input.inputs.Inputs
|
||||||
|
input.request.Request.Execution = input.execution.Execution
|
||||||
|
input.request.Request.TransientRetries = input.execution.TransientRetries
|
||||||
selected, err := prepareSuiteInput(suite, input.request.Selection, input.captured)
|
selected, err := prepareSuiteInput(suite, input.request.Selection, input.captured)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
|||||||
@@ -64,8 +64,8 @@ func TestValidationCommandComposesWithRequirementsShow(t *testing.T) {
|
|||||||
outcome = app.Execute(context.Background(), []string{
|
outcome = app.Execute(context.Background(), []string{
|
||||||
"promptkit", "requirements", "validate", "briefing",
|
"promptkit", "requirements", "validate", "briefing",
|
||||||
"--config", "config.yml", "--overlay", "overlay.yml", "--campaign", "spring",
|
"--config", "config.yml", "--overlay", "overlay.yml", "--campaign", "spring",
|
||||||
"--profile", "editorial", "--session-id", "session-1",
|
"--llm-profile", "editorial", "--llm-session-id", "session-1",
|
||||||
"--var", "topic=roadmap", "--input", "notes=fixtures/notes=final.txt", "--format", "yaml",
|
"--llm-var", "topic=roadmap", "--llm-input-file", "notes=fixtures/notes=final.txt", "--format", "yaml",
|
||||||
}, commandkit.Streams{Out: &output, Err: io.Discard})
|
}, commandkit.Streams{Out: &output, Err: io.Discard})
|
||||||
if outcome.Kind != commandkit.OutcomeSuccess || outcome.Status != 0 || validateCalls != 1 {
|
if outcome.Kind != commandkit.OutcomeSuccess || outcome.Status != 0 || validateCalls != 1 {
|
||||||
t.Fatalf("validation outcome = (%#v, %q), calls=%d", outcome, output.String(), validateCalls)
|
t.Fatalf("validation outcome = (%#v, %q), calls=%d", outcome, output.String(), validateCalls)
|
||||||
@@ -168,7 +168,7 @@ func TestValidationCommandConstructionAndCaptureLifecycle(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var output bytes.Buffer
|
var output bytes.Buffer
|
||||||
outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "empty", "--profile", "default", "--campaign", "spring"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "empty", "--llm-profile", "default", "--campaign", "spring"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
||||||
if outcome.Kind != commandkit.OutcomeSuccess || newInputCalls != 1 || newStateCalls != 1 || checkCalls != 1 || buildCalls != 1 || loadCalls != 1 {
|
if outcome.Kind != commandkit.OutcomeSuccess || newInputCalls != 1 || newStateCalls != 1 || checkCalls != 1 || buildCalls != 1 || loadCalls != 1 {
|
||||||
t.Fatalf("captured command = %#v, callback counts input/state/check/build/load=%d/%d/%d/%d/%d", outcome, newInputCalls, newStateCalls, checkCalls, buildCalls, loadCalls)
|
t.Fatalf("captured command = %#v, callback counts input/state/check/build/load=%d/%d/%d/%d/%d", outcome, newInputCalls, newStateCalls, checkCalls, buildCalls, loadCalls)
|
||||||
}
|
}
|
||||||
@@ -197,9 +197,9 @@ func TestValidationCommandPreservesPresenceAndRepeatedAssignmentsPerInvocation(t
|
|||||||
|
|
||||||
firstArgs := []string{
|
firstArgs := []string{
|
||||||
"promptkit", "requirements", "validate", "briefing", "--config", "", "--overlay", "",
|
"promptkit", "requirements", "validate", "briefing", "--config", "", "--overlay", "",
|
||||||
"--profile", "", "--session-id", "", "--campaign", "spring",
|
"--llm-profile", "", "--llm-session-id", "", "--campaign", "spring",
|
||||||
"--var", "=", "--var", "phrase= two words, three ", "--var", "duplicate=first", "--var", "duplicate=last=value",
|
"--llm-var", "=", "--llm-var", "phrase= two words, three ", "--llm-var", "duplicate=first", "--llm-var", "duplicate=last=value",
|
||||||
"--input", "=", "--input", "file= path, with commas=and=equals ", "--input", "duplicate=first.txt", "--input", "duplicate=last=path.txt",
|
"--llm-input-file", "=", "--llm-input-file", "file= path, with commas=and=equals ", "--llm-input-file", "duplicate=first.txt", "--llm-input-file", "duplicate=last=path.txt",
|
||||||
}
|
}
|
||||||
outcome := app.Execute(context.Background(), firstArgs, commandkit.Streams{Out: io.Discard, Err: io.Discard})
|
outcome := app.Execute(context.Background(), firstArgs, commandkit.Streams{Out: io.Discard, Err: io.Discard})
|
||||||
if outcome.Kind != commandkit.OutcomeSuccess || len(requests) != 1 {
|
if outcome.Kind != commandkit.OutcomeSuccess || len(requests) != 1 {
|
||||||
@@ -257,8 +257,8 @@ func TestValidationCommandUsageAndInputFailuresStopProviderWork(t *testing.T) {
|
|||||||
{name: "unsupported format", args: []string{"promptkit", "requirements", "validate", "briefing", "--campaign", "spring", "--format", "invalid"}},
|
{name: "unsupported format", args: []string{"promptkit", "requirements", "validate", "briefing", "--campaign", "spring", "--format", "invalid"}},
|
||||||
{name: "unknown flag", args: []string{"promptkit", "requirements", "validate", "briefing", "--campaign", "spring", "--unknown", "value"}},
|
{name: "unknown flag", args: []string{"promptkit", "requirements", "validate", "briefing", "--campaign", "spring", "--unknown", "value"}},
|
||||||
{name: "removed prompt version flag", args: []string{"promptkit", "requirements", "validate", "briefing", "--campaign", "spring", "--version", "2"}},
|
{name: "removed prompt version flag", args: []string{"promptkit", "requirements", "validate", "briefing", "--campaign", "spring", "--version", "2"}},
|
||||||
{name: "malformed variable before valid assignment", args: []string{"promptkit", "requirements", "validate", "briefing", "--campaign", "spring", "--var", "broken", "--var", "mode=full"}},
|
{name: "malformed variable before valid assignment", args: []string{"promptkit", "requirements", "validate", "briefing", "--campaign", "spring", "--llm-var", "broken", "--llm-var", "mode=full"}},
|
||||||
{name: "malformed input before valid assignment", args: []string{"promptkit", "requirements", "validate", "briefing", "--campaign", "spring", "--input", "broken", "--input", "notes=file.txt"}},
|
{name: "malformed input before valid assignment", args: []string{"promptkit", "requirements", "validate", "briefing", "--campaign", "spring", "--llm-input-file", "broken", "--llm-input-file", "notes=file.txt"}},
|
||||||
{name: "missing required selector", args: []string{"promptkit", "requirements", "validate", "briefing"}},
|
{name: "missing required selector", args: []string{"promptkit", "requirements", "validate", "briefing"}},
|
||||||
} {
|
} {
|
||||||
t.Run(test.name, func(t *testing.T) {
|
t.Run(test.name, func(t *testing.T) {
|
||||||
@@ -325,7 +325,7 @@ func TestValidationCommandUsesNativeOutcomesAndMinimalConfirmation(t *testing.T)
|
|||||||
app := newPromptTestApp(t, Command(suite))
|
app := newPromptTestApp(t, Command(suite))
|
||||||
|
|
||||||
var output bytes.Buffer
|
var output bytes.Buffer
|
||||||
outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "empty", "--profile", "default"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "empty", "--llm-profile", "default"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
||||||
if outcome.Kind != commandkit.OutcomeSuccess || outcome.Status != 0 {
|
if outcome.Kind != commandkit.OutcomeSuccess || outcome.Status != 0 {
|
||||||
t.Fatalf("native request validation = %#v, output=%q", outcome, output.String())
|
t.Fatalf("native request validation = %#v, output=%q", outcome, output.String())
|
||||||
}
|
}
|
||||||
@@ -342,8 +342,8 @@ func TestValidationCommandUsesNativeOutcomesAndMinimalConfirmation(t *testing.T)
|
|||||||
args []string
|
args []string
|
||||||
want error
|
want error
|
||||||
}{
|
}{
|
||||||
{name: "missing required input", args: []string{"promptkit", "requirements", "validate", "briefing", "--profile", "default"}, want: promptkit.ErrPromptRender},
|
{name: "missing required input", args: []string{"promptkit", "requirements", "validate", "briefing", "--llm-profile", "default"}, want: promptkit.ErrPromptRender},
|
||||||
{name: "invalid profile selection", args: []string{"promptkit", "requirements", "validate", "briefing", "--profile", "missing", "--input", "topic=unused.txt"}, want: promptkit.ErrProfileNotFound},
|
{name: "invalid profile selection", args: []string{"promptkit", "requirements", "validate", "briefing", "--llm-profile", "missing", "--llm-input-file", "topic=unused.txt"}, want: promptkit.ErrProfileNotFound},
|
||||||
} {
|
} {
|
||||||
t.Run(test.name, func(t *testing.T) {
|
t.Run(test.name, func(t *testing.T) {
|
||||||
output.Reset()
|
output.Reset()
|
||||||
|
|||||||
Reference in New Issue
Block a user