Suggest PromptKit smoke test kinds

This commit is contained in:
2026-11-03 13:13:01 +00:00
parent 8964ad02ad
commit 5227049de6
4 changed files with 29 additions and 8 deletions

View File

@@ -137,7 +137,9 @@ The transport and public metadata findings above remain open.
### 6. Supply the existing PromptKit smoke kinds as operand hints ### 6. Supply the existing PromptKit smoke kinds as operand hints
Priority: P3. Status: open. Priority: P3. Status: complete.
- [x] Completed: The smoke operand suggests the existing native kinds and all. Composed tests verify canonical and alias hints without loading an engine or calling smoke or inference; the ckprompt suite passes.
The [smoke command](../../integrations/ckprompt/smoke_command.go#L141) still The [smoke command](../../integrations/ckprompt/smoke_command.go#L141) still
declares `KIND` without a hint. In the maintained composed consumer, queries declares `KIND` without a hint. In the maintained composed consumer, queries

View File

@@ -9,3 +9,12 @@ func completionChoices(values ...string) commandkit.CompletionHint {
} }
return commandkit.CompleteChoices(choices...) return commandkit.CompleteChoices(choices...)
} }
func smokeKindCompletion() commandkit.CompletionHint {
kinds := aggregateSmokeKinds()
values := make([]string, 0, len(kinds)+1)
for _, kind := range kinds {
values = append(values, string(kind))
}
return completionChoices(append(values, "all")...)
}

View File

@@ -353,12 +353,16 @@ func TestComposedCatalogsAndSmokeUseInjectedClientOnlyForInference(t *testing.T)
} }
func TestComposedCompletionUsesHintsWithoutLoadingEngine(t *testing.T) { func TestComposedCompletionUsesHintsWithoutLoadingEngine(t *testing.T) {
var loads, calls int var loads, calls, smokeCalls int
loader := func(ctx context.Context, selection ckconfig.Selection, input *inspectionInput, cache *runCache) (*promptkit.Engine, error) { loader := func(ctx context.Context, selection ckconfig.Selection, input *inspectionInput, cache *runCache) (*promptkit.Engine, error) {
loads++ loads++
return loadPromptEngine(ctx, selection, input, cache) return loadPromptEngine(ctx, selection, input, cache)
} }
app, err := newAppWithEngineLoader(&exampleLLMClient{calls: &calls}, loader, nil) smoke := func(context.Context, ckprompt.SmokeRequest, *inspectionInput, *runCache) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
smokeCalls++
return commandkit.Acquired[*promptkit.SmokeTestReport]{}, nil
}
app, err := newAppWithEngineLoader(&exampleLLMClient{calls: &calls}, loader, smoke)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -366,6 +370,8 @@ func TestComposedCompletionUsesHintsWithoutLoadingEngine(t *testing.T) {
args []string args []string
want string want string
}{ }{
{[]string{"__complete", "promptkit", "test", "re"}, "reachability\n"},
{[]string{"__complete", "smoke", "al"}, "all\n"},
{[]string{"__complete", "pk", "show", "prompt", "briefing", "--fmt=y"}, "yaml\n"}, {[]string{"__complete", "pk", "show", "prompt", "briefing", "--fmt=y"}, "yaml\n"},
{[]string{"__complete", "pk", "show", "prompt", "briefing", "--config=fixtures/"}, ":0\n"}, {[]string{"__complete", "pk", "show", "prompt", "briefing", "--config=fixtures/"}, ":0\n"},
} { } {
@@ -375,8 +381,8 @@ func TestComposedCompletionUsesHintsWithoutLoadingEngine(t *testing.T) {
t.Fatalf("query %q outcome=%#v output=%q", tc.args, outcome, output.String()) t.Fatalf("query %q outcome=%#v output=%q", tc.args, outcome, output.String())
} }
} }
if loads != 0 || calls != 0 { if loads != 0 || calls != 0 || smokeCalls != 0 {
t.Fatalf("completion loaded engine %d times and invoked client %d times", loads, calls) t.Fatalf("completion loaded engine %d times and invoked client %d times, smoke calls=%d", loads, calls, smokeCalls)
} }
} }

View File

@@ -138,7 +138,7 @@ func newSmokeTestCommand[I, S any](suite Suite[I, S]) commandkit.Command {
}, },
Flags: flags, Flags: flags,
Arguments: []commandkit.Argument[*smokeCommandInput[I]]{ Arguments: []commandkit.Argument[*smokeCommandInput[I]]{
commandkit.ValueArgument(commandkit.ArgumentMeta{Name: "kind", Usage: "Native smoke-test kind."}, true, parseDefinitionString, commandkit.ValueArgument(commandkit.ArgumentMeta{Name: "kind", Usage: "Native smoke-test kind.", Completion: smokeKindCompletion()}, true, parseDefinitionString,
func(input **smokeCommandInput[I], value commandkit.Supplied[string]) { (*input).kind = value.Value }), func(input **smokeCommandInput[I], value commandkit.Supplied[string]) { (*input).kind = value.Value }),
}, },
CheckInput: func(input *smokeCommandInput[I]) error { CheckInput: func(input *smokeCommandInput[I]) error {
@@ -228,13 +228,17 @@ func runSmokeTest[I, S any](invocation commandkit.Invocation, suite Suite[I, S],
return commandkit.Success() return commandkit.Success()
} }
func runSmokeAggregate[I, S any](invocation commandkit.Invocation, suite Suite[I, S], input *smokeCommandInput[I], state S) commandkit.Outcome { func aggregateSmokeKinds() [4]promptkit.SmokeTestKind {
kinds := [...]promptkit.SmokeTestKind{ return [...]promptkit.SmokeTestKind{
promptkit.SmokeTestReachability, promptkit.SmokeTestReachability,
promptkit.SmokeTestAuthentication, promptkit.SmokeTestAuthentication,
promptkit.SmokeTestTextInference, promptkit.SmokeTestTextInference,
promptkit.SmokeTestStructuredOutput, promptkit.SmokeTestStructuredOutput,
} }
}
func runSmokeAggregate[I, S any](invocation commandkit.Invocation, suite Suite[I, S], input *smokeCommandInput[I], state S) commandkit.Outcome {
kinds := aggregateSmokeKinds()
results := make([]smokeResult, 0, len(kinds)) results := make([]smokeResult, 0, len(kinds))
var operationErrors []error var operationErrors []error
var explanations []commandkit.Explanation var explanations []commandkit.Explanation