base-layer/commandkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Migrate maintained examples to invocation reporting

Browse Source
This commit is contained in:
Eric Rakestraw
2026-10-26 03:44:19 +00:00
parent b6f22462d3
commit 6264a87183
8 changed files with 204 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
integrations/ckprompt/examples/composed/main.go
View File

@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"io"
"os"
"strconv"
@@ -65,6 +66,8 @@ type session struct {
Summary string
}
var errSessionNotFound = errors.New("saved session was not found")
func campaignSelector() commandkit.Flag[*inspectionInput] {
return commandkit.ValueFlag(commandkit.FlagMeta{Name: "campaign", Usage: "Select campaign inputs."}, (*string)(nil), parseString,
func(input **inspectionInput, value commandkit.Supplied[string]) { (*input).Campaign = value })
@@ -200,10 +203,14 @@ func newApp() (*commandkit.App, error) {
}
func newAppWithLLMClient(client promptkit.LLMClient) (*commandkit.App, error) {
return newAppWithEngineLoader(client, loadPromptEngine)
return newAppWithEngineLoader(client, loadPromptEngine, nil)
}
func newAppWithEngineLoader(client promptkit.LLMClient, engineLoader func(context.Context, ckconfig.Selection, *inspectionInput, *runCache) (*promptkit.Engine, error)) (*commandkit.App, error) {
func newAppWithEngineLoader(
client promptkit.LLMClient,
engineLoader func(context.Context, ckconfig.Selection, *inspectionInput, *runCache) (*promptkit.Engine, error),
smokeTest func(context.Context, ckprompt.SmokeRequest, *inspectionInput, *runCache) (commandkit.Acquired[*promptkit.SmokeTestReport], error),
) (*commandkit.App, error) {
if client == nil {
client = &exampleLLMClient{}
}
@@ -237,6 +244,9 @@ func newAppWithEngineLoader(client promptkit.LLMClient, engineLoader func(contex
definitions.Selectors = []commandkit.Flag[*inspectionInput]{campaignSelector()}
definitions = ckprompt.WithRequestValidation(definitions, engineLoader, buildValidationRequest)
definitions = ckprompt.WithSmokeTesting(definitions, engineLoader, nil)
if smokeTest != nil {
definitions.SmokeTest = smokeTest
}
definitionCommand := ckprompt.Command(definitions)
store := map[string]session{
@@ -250,27 +260,30 @@ func newAppWithEngineLoader(client promptkit.LLMClient, engineLoader func(contex
Arguments: []commandkit.Argument[restoreInput]{commandkit.ValueArgument(commandkit.ArgumentMeta{Name: "ID", Usage: "Session identifier."}, true, parseString,
func(input *restoreInput, value commandkit.Supplied[string]) { input.Session = value.Value })},
Run: func(invocation commandkit.Invocation, input restoreInput) commandkit.Outcome {
reporter := invocation.Reporter()
if err := invocation.Context.Err(); err != nil {
return commandkit.Failure(err)
return commandkit.Failure("Session restore was canceled.").WithCause(err)
}
application, err := loadApplicationConfig(invocation.Context, ckconfig.Selection{}, nil, nil)
if err != nil {
return commandkit.Failure(err)
return commandkit.Failure("Could not load application configuration.").WithCause(err)
}
name, _, err := selectedCampaign(application.Config, input.Campaign)
if err != nil {
return commandkit.Failure(err)
return commandkit.Failure(err.Error()).WithCause(err)
}
record, ok := store[input.Session]
if !ok {
return commandkit.Outcome{Kind: commandkit.OutcomeDomain, Status: 4, Domain: "session_not_found", Cause: errors.New("saved session was not found")}
return commandkit.Outcome{Kind: commandkit.OutcomeDomain, Status: 4, Domain: "session_not_found", Cause: errSessionNotFound, Explanation: commandkit.Explanation{Message: "Saved session was not found."}}
}
if record.Campaign != name {
return commandkit.Outcome{Kind: commandkit.OutcomeDomain, Status: 3, Domain: "campaign_mismatch", Cause: fmt.Errorf("session belongs to campaign %q", record.Campaign)}
}
if _, err := fmt.Fprintf(invocation.Streams.Out, "restored %s: %s\n", record.ID, record.Summary); err != nil {
return commandkit.Failure(err)
cause := fmt.Errorf("session belongs to campaign %q", record.Campaign)
return commandkit.Outcome{Kind: commandkit.OutcomeDomain, Status: 3, Domain: "campaign_mismatch", Cause: cause, Explanation: commandkit.Explanation{Message: "Session campaign does not match the selected campaign."}}
}
_ = reporter.RenderOutput(func(writer io.Writer) error {
_, err := fmt.Fprintf(writer, "restored %s: %s\n", record.ID, record.Summary)
return err
})
return commandkit.Success()
},
})

68
integrations/ckprompt/examples/composed/main_test.go
View File

@@ -3,6 +3,7 @@ package main
import (
"bytes"
"context"
"errors"
"io"
"reflect"
"strings"
@@ -351,6 +352,66 @@ func TestComposedCatalogsAndSmokeUseInjectedClientOnlyForInference(t *testing.T)
}
}
func TestComposedSmokeReportSurvivesInferenceFailure(t *testing.T) {
wantCause := errors.New("local example inference failed")
smokeTest := func(_ context.Context, request ckprompt.SmokeRequest, _ *inspectionInput, _ *runCache) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
return commandkit.Acquired[*promptkit.SmokeTestReport]{
Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind, Completed: true},
Diagnostics: []commandkit.Diagnostic{commandkit.MessageDiagnostic(commandkit.SeverityWarning, "independent smoke warning")},
Explanation: commandkit.Explanation{Details: []commandkit.Detail{func(writer io.Writer) error {
_, err := io.WriteString(writer, "Native text inference context.\n")
return err
}}},
}, wantCause
}
app, err := newAppWithEngineLoader(&exampleLLMClient{}, loadPromptEngine, smokeTest)
if err != nil {
t.Fatal(err)
}
var output, diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), []string{"promptkit", "test", "text_inference", "--campaign", "spring", "--prompt", "briefing", "--profile", "spring", "--verbose"}, commandkit.Streams{Out: &output, Err: &diagnostics})
if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || !errors.Is(outcome.Cause, wantCause) {
t.Fatalf("smoke failure outcome = %#v; report %q; diagnostics %q", outcome, output.String(), diagnostics.String())
}
if !strings.Contains(output.String(), "kind: text_inference") || !strings.Contains(output.String(), "completed: true") {
t.Fatalf("native smoke report was not delivered: %q", output.String())
}
warningAt := strings.Index(diagnostics.String(), "independent smoke warning")
contextAt := strings.Index(diagnostics.String(), "Native text inference context")
if warningAt < 0 || contextAt < 0 || warningAt >= contextAt || strings.Contains(diagnostics.String(), wantCause.Error()) {
t.Fatalf("terminal explanation/diagnostics = %q / %q", outcome.Explanation.Message, diagnostics.String())
}
if outcome.Explanation.Message != "" {
t.Errorf("unexpected synthesized terminal explanation %q", outcome.Explanation.Message)
}
}
func TestComposedCustomRestoreUsesSharedReportingAndTerminalFailure(t *testing.T) {
app, err := newApp()
if err != nil {
t.Fatal(err)
}
var output, diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), []string{"session", "restore", "session-42", "--campaign", "spring", "--quiet"}, commandkit.Streams{Out: &output, Err: &diagnostics})
if outcome.Kind != commandkit.OutcomeSuccess || output.Len() != 0 || diagnostics.Len() != 0 {
t.Fatalf("quiet custom command outcome/output/diagnostics = %#v / %q / %q", outcome, output.String(), diagnostics.String())
}
output.Reset()
diagnostics.Reset()
outcome = app.Execute(context.Background(), []string{"session", "restore", "missing", "--quiet"}, commandkit.Streams{Out: &output, Err: &diagnostics})
if outcome.Kind != commandkit.OutcomeDomain || outcome.Status != 4 || outcome.Domain != "session_not_found" || !errors.Is(outcome.Cause, errSessionNotFound) {
t.Fatalf("missing session outcome = %#v", outcome)
}
if output.Len() != 0 || outcome.Explanation.Message == "" || !strings.Contains(diagnostics.String(), outcome.Explanation.Message) {
t.Fatalf("terminal failure output/explanation/diagnostics = %q / %q / %q", output.String(), outcome.Explanation.Message, diagnostics.String())
}
if strings.Contains(diagnostics.String(), errSessionNotFound.Error()) {
t.Errorf("terminal presentation exposed the underlying cause: %q", diagnostics.String())
}
}
func TestComposedAliasesReachCanonicalCommands(t *testing.T) {
clientCalls := 0
engineLoads := 0
@@ -358,7 +419,7 @@ func TestComposedAliasesReachCanonicalCommands(t *testing.T) {
engineLoads++
return loadPromptEngine(ctx, selection, input, cache)
}
app, err := newAppWithEngineLoader(&exampleLLMClient{calls: &clientCalls}, engineLoader)
app, err := newAppWithEngineLoader(&exampleLLMClient{calls: &clientCalls}, engineLoader, nil)
if err != nil {
t.Fatal(err)
}
@@ -368,6 +429,11 @@ func TestComposedAliasesReachCanonicalCommands(t *testing.T) {
if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(output.String(), "--hlep → --help") || strings.Contains(output.String(), "field-notes") || engineLoads != 0 || clientCalls != 0 {
t.Fatalf("group help alias outcome/output/engine loads/client calls = (%#v, %q, %d, %d)", outcome, output.String(), engineLoads, clientCalls)
}
output.Reset()
outcome = app.Execute(context.Background(), []string{"pk", "--hlep", "--quiet"}, commandkit.Streams{Out: &output, Err: io.Discard})
if outcome.Kind != commandkit.OutcomeSuccess || output.Len() != 0 || engineLoads != 0 || clientCalls != 0 {
t.Fatalf("quiet aliased help outcome/output/engine loads/client calls = (%#v, %q, %d, %d)", outcome, output.String(), engineLoads, clientCalls)
}
for _, args := range [][]string{
{"list", "collections", "--campaign", "spring"},