base-layer/commandkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Update the composed PromptKit consumer

Browse Source
This commit is contained in:
Eric Rakestraw
2026-11-07 05:01:30 +00:00
parent 42af1da142
commit 1289868ed0
4 changed files with 236 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

169
integrations/ckprompt/examples/composed/main_test.go
View File

@@ -3,6 +3,7 @@ package main
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
@@ -101,6 +102,159 @@ func TestApplicationCacheDoesNotRetainFailedLoads(t *testing.T) {
}
}
func TestApplicationCacheDistinguishesOverrides(t *testing.T) {
cache := &runCache{}
selection := func(value string) ckconfig.Selection {
return ckconfig.Selection{Overrides: []configkit.Override{{Path: configkit.Keys("campaigns", "spring", "model"), Value: value, Flag: "--set"}}}
}
left, err := loadApplicationConfig(context.Background(), selection("left"), nil, cache)
if err != nil {
t.Fatal(err)
}
right, err := loadApplicationConfig(context.Background(), selection("right"), nil, cache)
if err != nil {
t.Fatal(err)
}
again, err := loadApplicationConfig(context.Background(), selection("left"), nil, cache)
if err != nil {
t.Fatal(err)
}
if left == right || left != again || left.Config.Campaigns["spring"].Model != "left" || right.Config.Campaigns["spring"].Model != "right" {
t.Fatalf("override selections reused the wrong result: left=%#v right=%#v", left.Config, right.Config)
}
}
func TestComposedIndependentOverridesAndFailureEvidence(t *testing.T) {
app, err := newApp()
if err != nil {
t.Fatal(err)
}
var output bytes.Buffer
args := []string{"config", "diff", "--scope", "app", "--campaign", "spring", "--limit=0", "--against-campaign", "autumn", "--against-limit=0", "--set", `/campaigns/spring/model="left"`, "--against-set", `/campaigns/spring/model="right"`, "--details", "--format", "json"}
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: io.Discard})
if outcome.Kind != commandkit.OutcomeSuccess {
t.Fatalf("comparison: %#v, %s", outcome, output.String())
}
var report struct {
Differences []struct {
Path string `json:"path"`
Before struct {
Value any `json:"value"`
} `json:"before"`
After struct {
Value any `json:"value"`
} `json:"after"`
} `json:"differences"`
}
if err := json.Unmarshal(output.Bytes(), &report); err != nil {
t.Fatal(err)
}
changes := map[string][2]any{}
for _, difference := range report.Differences {
changes[difference.Path] = [2]any{difference.Before.Value, difference.After.Value}
}
if changes["/campaigns/spring/model"] != [2]any{"left", "right"} || changes["/default_campaign"] != [2]any{"spring", "autumn"} {
t.Fatalf("independent comparison: %#v", changes)
}
output.Reset()
outcome = app.Execute(context.Background(), []string{"config", "sources", "--scope", "app", "--config", "fixtures/missing.yml", "--format", "json"}, commandkit.Streams{Out: &output, Err: io.Discard})
var evidence struct {
Sources []any `json:"sources"`
}
if err := json.Unmarshal(output.Bytes(), &evidence); err != nil {
t.Fatal(err)
}
if outcome.Kind != commandkit.OutcomeOperational || outcome.Cause == nil || len(evidence.Sources) == 0 {
t.Fatalf("failed load evidence: %#v, %s", outcome, output.String())
}
outcome = app.Execute(context.Background(), []string{"config", "validate", "--scope", "app", "--campaign", "spring", "--limit=0", "--set", `/default_campaign="autumn"`}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
if outcome.Kind != commandkit.OutcomeOperational || outcome.Cause == nil {
t.Fatalf("overlapping native overrides: %#v", outcome)
}
loads := 0
loader := func(ctx context.Context, selection ckconfig.Selection, input *inspectionInput, cache *runCache) (*promptkit.Engine, error) {
loads++
return loadPromptEngine(ctx, selection, input, cache)
}
app, err = newAppWithEngineLoader(&exampleLLMClient{}, loader, nil)
if err != nil {
t.Fatal(err)
}
outcome = app.Execute(context.Background(), []string{"config", "validate", "--scope", "all", "--set", `/default_campaign="spring"`}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
if outcome.Kind != commandkit.OutcomeUsage || loads != 0 {
t.Fatalf("unsupported aggregate override: %#v", outcome)
}
}
func TestComposedPreviewAndReusableFlags(t *testing.T) {
calls := 0
app, err := newAppWithLLMClient(&exampleLLMClient{calls: &calls})
if err != nil {
t.Fatal(err)
}
var output bytes.Buffer
args := []string{"promptkit", "show", "prepared", "briefing", "--campaign", "spring", "--llm-input-file", "topic=fixtures/topic.txt", "--input", "topic=sprint", "--profile", "spring", "--extra-params", `{"campaign":"preview"}`, "--format", "json"}
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: io.Discard})
if outcome.Kind != commandkit.OutcomeSuccess {
t.Fatalf("preview: %#v, %s", outcome, output.String())
}
var report promptkit.PreparedRun
if err := json.Unmarshal(output.Bytes(), &report); err != nil {
t.Fatal(err)
}
input := &inspectionInput{Campaign: commandkit.Supplied[string]{Value: "spring", Set: true}}
engine, err := loadPromptEngine(context.Background(), ckconfig.Selection{}, input, nil)
if err != nil {
t.Fatal(err)
}
request, err := buildValidationRequest(context.Background(), ckprompt.ValidationRequest{Request: promptkit.RunRequest{PromptID: "briefing", ProfileID: "spring", Inputs: map[string]promptkit.ArtifactRef{"topic": promptkit.Inline("sprint")}, Execution: &promptkit.ExecutionTargetOverride{ExtraParams: map[string]any{"campaign": "preview"}}}}, input, nil)
if err != nil {
t.Fatal(err)
}
native, err := engine.Prepare(context.Background(), request)
if err != nil {
t.Fatal(err)
}
if report.PromptID != native.PromptID || !reflect.DeepEqual(report.Messages, native.Messages) || !reflect.DeepEqual(report.EffectiveModelParams, native.EffectiveModelParams) || calls != 0 {
t.Fatalf("preview diverged from native result or called model: %#v", report)
}
output.Reset()
outcome = app.Execute(context.Background(), []string{"promptkit", "show", "prompt", "briefing", "--campaign", "spring", "--prompt-namespace", "field-notes", "--format", "json"}, commandkit.Streams{Out: &output, Err: io.Discard})
var namespaced promptkit.PromptInspection
if err := json.Unmarshal(output.Bytes(), &namespaced); err != nil {
t.Fatal(err)
}
view, err := engine.WithNamespaces(promptkit.Namespaces{Prompt: "field-notes"})
if err != nil {
t.Fatal(err)
}
wantPrompt, err := view.InspectPrompt(context.Background(), "briefing")
if err != nil {
t.Fatal(err)
}
if outcome.Kind != commandkit.OutcomeSuccess || namespaced.PromptID != wantPrompt.PromptID {
t.Fatalf("namespace selection: %#v, %#v", outcome, namespaced)
}
output.Reset()
args = []string{"request", "inspect", "--llm-prompt", "briefing", "--llm-profile=", "--llm-model", "sample", "--llm-input-text", "topic=hello", "--format", "json"}
outcome = app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: io.Discard})
var owned requestInputs
if err := json.Unmarshal(output.Bytes(), &owned); err != nil {
t.Fatal(err)
}
if outcome.Kind != commandkit.OutcomeSuccess || !owned.Resources.Profile.Set || owned.Resources.Profile.Value != "" || owned.Execution.Execution.Model != "sample" || owned.Inputs.Inputs["topic"] != promptkit.Inline("hello") {
t.Fatalf("application flags: %#v, %#v", outcome, owned)
}
outcome = app.Execute(context.Background(), []string{"request", "inspect", "--model", "sample"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
if outcome.Kind != commandkit.OutcomeUsage {
t.Fatalf("unexpected application alias: %#v", outcome)
}
}
func TestComposedPromptKitAndDomainCommandsRemainAvailable(t *testing.T) {
app, err := newApp()
if err != nil {
@@ -266,15 +420,19 @@ func TestBuildValidationRequestDefaultsOnlyAbsentVariables(t *testing.T) {
if !reflect.DeepEqual(got.Inputs, inputs) || !reflect.DeepEqual(seed.Request.Vars, explicit) {
t.Fatalf("seed bindings changed: built=%#v seed=%#v", got.Inputs, explicit)
}
key := applicationCacheKey{config: seed.Selection.Config, overlay: seed.Selection.Overlay, campaign: input.Campaign}
cached := cache.applications[key]
var cached *configkit.Result[appConfiguration]
for _, entry := range cache.applications {
if reflect.DeepEqual(entry.selection, seed.Selection) && entry.campaign == input.Campaign {
cached = entry.result
}
}
if cached == nil {
t.Fatal("builder did not load application configuration into invocation state")
}
if _, err := loadPromptEngine(context.Background(), seed.Selection, input, cache); err != nil {
t.Fatal(err)
}
if cache.applications[key] != cached {
if len(cache.applications) != 1 || cache.applications[0].result != cached {
t.Fatal("engine loader did not reuse the builder's application configuration")
}
@@ -518,6 +676,11 @@ func TestComposedAliasesReachCanonicalCommands(t *testing.T) {
if outcome.Kind != commandkit.OutcomeSuccess || output.Len() != 0 || engineLoads != 0 || clientCalls != 0 {
t.Fatalf("quiet aliased help outcome/output/engine loads/client calls = (%#v, %q, %d, %d)", outcome, output.String(), engineLoads, clientCalls)
}
output.Reset()
outcome = app.Execute(context.Background(), []string{"version"}, commandkit.Streams{Out: &output, Err: io.Discard})
if outcome.Kind != commandkit.OutcomeSuccess || engineLoads != 0 || clientCalls != 0 {
t.Fatalf("version outcome/engine loads/client calls = (%#v, %d, %d)", outcome, engineLoads, clientCalls)
}
for _, args := range [][]string{
{"list", "collections", "--campaign", "spring"},