base-layer/commandkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add PromptKit configuration comparison

Browse Source
This commit is contained in:
Eric Rakestraw
2026-10-12 02:01:10 +00:00
parent c6b4a42287
commit ddd14b70a8
3 changed files with 204 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
integrations/ckprompt/adapters.go
View File

@@ -69,7 +69,7 @@ func ConfigScope[I, S any](load func(context.Context, ckconfig.Selection, I, S)
scope := ckconfig.Scope[I, S]{
Name: "promptkit",
Summary: "PromptKit engine configuration.",
Operations: []ckconfig.Operation{ckconfig.Validate, ckconfig.Show, ckconfig.Sources, ckconfig.Explain},
Operations: []ckconfig.Operation{ckconfig.Validate, ckconfig.Show, ckconfig.Sources, ckconfig.Explain, ckconfig.Diff},
}
if load == nil {
return ckconfig.WithScopeError(scope, errors.New("PromptKit configuration loader is nil"))
@@ -89,9 +89,47 @@ func ConfigScope[I, S any](load func(context.Context, ckconfig.Selection, I, S)
Diagnostics: diagnostics,
}, nil
}
scope.Compare = func(ctx context.Context, request ckconfig.ComparisonRequest[I], state S) (commandkit.Acquired[[]configkit.Difference], error) {
if err := promptKitContextError(ctx); err != nil {
return commandkit.Acquired[[]configkit.Difference]{}, fmt.Errorf("PromptKit scope %q comparison before left load: %w", scope.Name, err)
}
_, leftInspection, leftDiagnostics, err := loadPromptKitEngine(ctx, request.Left.Selection, request.Left.Input, state, "left", load)
diagnostics := append([]commandkit.Diagnostic(nil), leftDiagnostics...)
if err != nil {
return commandkit.Acquired[[]configkit.Difference]{Diagnostics: diagnostics}, fmt.Errorf("PromptKit scope %q left comparison load: %w", scope.Name, err)
}
if err := promptKitContextError(ctx); err != nil {
return commandkit.Acquired[[]configkit.Difference]{Diagnostics: diagnostics}, fmt.Errorf("PromptKit scope %q comparison after left load: %w", scope.Name, err)
}
_, rightInspection, rightDiagnostics, err := loadPromptKitEngine(ctx, request.Right.Selection, request.Right.Input, state, "right", load)
diagnostics = append(diagnostics, rightDiagnostics...)
if err != nil {
return commandkit.Acquired[[]configkit.Difference]{Diagnostics: diagnostics}, fmt.Errorf("PromptKit scope %q right comparison load: %w", scope.Name, err)
}
if err := promptKitContextError(ctx); err != nil {
return commandkit.Acquired[[]configkit.Difference]{Diagnostics: diagnostics}, fmt.Errorf("PromptKit scope %q comparison before native diff: %w", scope.Name, err)
}
differences, err := leftInspection.Diff(rightInspection)
if err != nil {
return commandkit.Acquired[[]configkit.Difference]{Diagnostics: diagnostics}, fmt.Errorf("PromptKit scope %q native configuration diff: %w", scope.Name, err)
}
return commandkit.Acquired[[]configkit.Difference]{Value: differences, Diagnostics: diagnostics}, nil
}
return scope
}
func promptKitContextError(ctx context.Context) error {
if ctx == nil {
return nil
}
return ctx.Err()
}
// FromEngine creates named prompt and profile inspection callbacks backed by a
// PromptKit engine loader. It does not call the loader during construction. Each
// selected inspection passes its context, supplied selection, initialized input,

145
integrations/ckprompt/adapters_test.go
View File

@@ -5,6 +5,7 @@ import (
"context"
"errors"
"io"
"reflect"
"strings"
"testing"
"testing/fstest"
@@ -65,9 +66,13 @@ profiles:
`
func adapterEngine(t *testing.T) *promptkit.Engine {
return adapterEngineFrom(t, "adapter fixture", adapterPrompt)
}
func adapterEngineFrom(t *testing.T, label, configuration string) *promptkit.Engine {
t.Helper()
inputs, err := promptkit.FilesystemInputs("adapter fixture", fstest.MapFS{
"config/prompt.yml": &fstest.MapFile{Data: []byte(adapterPrompt)},
inputs, err := promptkit.FilesystemInputs(label, fstest.MapFS{
"config/prompt.yml": &fstest.MapFile{Data: []byte(configuration)},
}, "config")
if err != nil {
t.Fatal(err)
@@ -98,7 +103,7 @@ func TestConfigScopeUsesNativeInspectionFacetsAndInvocationValues(t *testing.T)
}
return engine, nil
})
if scope.Name != "promptkit" || len(scope.Operations) != 4 || scope.Acquire == nil {
if scope.Name != "promptkit" || len(scope.Operations) != 5 || scope.Acquire == nil {
t.Fatalf("configuration scope = %#v", scope)
}
acquired, err := scope.Acquire(wantContext, ckconfig.Request{
@@ -136,6 +141,140 @@ func TestConfigScopeRetainsNilLoaderIssueAfterCapabilitiesAreCleared(t *testing.
}
}
func TestConfigScopeCompareUsesNativeDiffAndLoadsBothEqualSelections(t *testing.T) {
leftEngine := adapterEngine(t)
rightConfig := strings.Replace(adapterPrompt, `content: "Hello"`, `content: "Goodbye"`, 1)
if rightConfig == adapterPrompt {
t.Fatal("comparison fixture did not change the prompt content")
}
rightEngine := adapterEngineFrom(t, "right comparison fixture", rightConfig)
type input struct{ label string }
type call struct {
selection ckconfig.Selection
input input
state *int
}
var calls []call
scope := ConfigScope(func(_ context.Context, selection ckconfig.Selection, gotInput input, state *int) (*promptkit.Engine, error) {
calls = append(calls, call{selection: selection, input: gotInput, state: state})
if gotInput.label == "before" {
return leftEngine, nil
}
return rightEngine, nil
})
if scope.Compare == nil || len(scope.Operations) != 5 || scope.Operations[4] != ckconfig.Diff {
t.Fatalf("configuration scope does not declare comparison: %#v", scope)
}
selection := ckconfig.Selection{Config: commandkit.Supplied[string]{Value: "same-selection.yml", Set: true}}
state := new(int)
request := ckconfig.ComparisonRequest[input]{
Left: ckconfig.ComparisonSide[input]{Selection: selection, Input: input{label: "before"}},
Right: ckconfig.ComparisonSide[input]{Selection: selection, Input: input{label: "after"}},
}
leftInspection, err := leftEngine.InspectConfiguration()
if err != nil {
t.Fatal(err)
}
rightInspection, err := rightEngine.InspectConfiguration()
if err != nil {
t.Fatal(err)
}
want, err := leftInspection.Diff(rightInspection)
if err != nil {
t.Fatal(err)
}
if len(want) == 0 {
t.Fatal("native PromptKit fixture comparison produced no differences")
}
got, err := scope.Compare(context.Background(), request, state)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(got.Value, want) {
t.Fatalf("adapter differences = %#v, want native PromptKit result %#v", got.Value, want)
}
if len(calls) != 2 || calls[0].selection != selection || calls[1].selection != selection || calls[0].input.label != "before" || calls[1].input.label != "after" || calls[0].state != state || calls[1].state != state {
t.Fatalf("loader calls = %#v", calls)
}
}
func TestConfigScopeCompareStopsAfterLeftLoadFailure(t *testing.T) {
loadErr := errors.New("left engine load failed")
rightCalled := false
scope := ConfigScope(func(_ context.Context, selection ckconfig.Selection, _ struct{}, _ struct{}) (*promptkit.Engine, error) {
if selection.Config.Value == "left" {
return nil, loadErr
}
rightCalled = true
return adapterEngine(t), nil
})
request := ckconfig.ComparisonRequest[struct{}]{
Left: ckconfig.ComparisonSide[struct{}]{Selection: ckconfig.Selection{Config: commandkit.Supplied[string]{Value: "left", Set: true}}},
Right: ckconfig.ComparisonSide[struct{}]{Selection: ckconfig.Selection{Config: commandkit.Supplied[string]{Value: "right", Set: true}}},
}
acquired, err := scope.Compare(context.Background(), request, struct{}{})
if !errors.Is(err, loadErr) || rightCalled || acquired.Value != nil {
t.Fatalf("left failure = (%#v, %v), right called=%t", acquired, err, rightCalled)
}
}
func TestConfigScopeCompareRetainsRightLoadDiagnosticsAndErrorIdentity(t *testing.T) {
_, loadErr := configkit.Load[map[string]any](configkit.Bytes("invalid right prompt config", []byte("broken: [")))
if loadErr == nil {
t.Fatal("invalid right configuration unexpectedly loaded")
}
leftEngine := adapterEngine(t)
var calls []string
scope := ConfigScope(func(_ context.Context, selection ckconfig.Selection, _ struct{}, _ struct{}) (*promptkit.Engine, error) {
calls = append(calls, selection.Config.Value)
if selection.Config.Value == "left" {
return leftEngine, nil
}
return nil, loadErr
})
request := ckconfig.ComparisonRequest[struct{}]{
Left: ckconfig.ComparisonSide[struct{}]{Selection: ckconfig.Selection{Config: commandkit.Supplied[string]{Value: "left", Set: true}}},
Right: ckconfig.ComparisonSide[struct{}]{Selection: ckconfig.Selection{Config: commandkit.Supplied[string]{Value: "right", Set: true}}},
}
acquired, err := scope.Compare(context.Background(), request, struct{}{})
var retained *configkit.LoadError
if !errors.Is(err, loadErr) || !errors.As(err, &retained) || retained != loadErr || len(calls) != 2 || len(acquired.Diagnostics) == 0 {
t.Fatalf("right failure = (%#v, %v), calls=%#v", acquired, err, calls)
}
var output bytes.Buffer
if writeErr := acquired.Diagnostics[0].Write(&output); writeErr != nil || !strings.Contains(output.String(), `Configuration scope "promptkit" (right):`) {
t.Fatalf("right diagnostic = %q, %v", output.String(), writeErr)
}
}
func TestConfigScopeCompareCancellationAfterLeftLoadStopsRight(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
leftEngine := adapterEngine(t)
rightCalled := false
scope := ConfigScope(func(_ context.Context, selection ckconfig.Selection, _ struct{}, _ struct{}) (*promptkit.Engine, error) {
if selection.Config.Value == "left" {
cancel()
return leftEngine, nil
}
rightCalled = true
return adapterEngine(t), nil
})
request := ckconfig.ComparisonRequest[struct{}]{
Left: ckconfig.ComparisonSide[struct{}]{Selection: ckconfig.Selection{Config: commandkit.Supplied[string]{Value: "left", Set: true}}},
Right: ckconfig.ComparisonSide[struct{}]{Selection: ckconfig.Selection{Config: commandkit.Supplied[string]{Value: "right", Set: true}}},
}
acquired, err := scope.Compare(ctx, request, struct{}{})
if !errors.Is(err, context.Canceled) || rightCalled || acquired.Value != nil {
t.Fatalf("cancellation = (%#v, %v), right called=%t", acquired, err, rightCalled)
}
}
func TestFromEnginePassesRequestsAndRetainsNilLoaderIssue(t *testing.T) {
type input struct{ value string }
type state struct{ value string }