Verify smoke and composed configuration outcomes
This commit is contained in:
@@ -252,7 +252,13 @@ packages using the aligned workspace.
|
|||||||
|
|
||||||
## Stage 4 — Verify smoke aggregation and configuration composition
|
## Stage 4 — Verify smoke aggregation and configuration composition
|
||||||
|
|
||||||
Status — pending.
|
Status — complete. Verified individual and aggregate smoke outcomes, mixed
|
||||||
|
failures and cancellation with retained reports and causes, and all five
|
||||||
|
operations through the composed PromptKit configuration scope, including the
|
||||||
|
right comparison side and pre-canceled invocations. Updated the existing
|
||||||
|
comparison and composed-example expectations for input diagnostics. With Go
|
||||||
|
1.27.0, `GOTOOLCHAIN=local GOPROXY=off go test ./integrations/ckprompt/...`
|
||||||
|
passed in the aligned workspace using a temporary Go build cache.
|
||||||
|
|
||||||
In `integrations/ckprompt/smoke_command.go`, complete the context argument updates
|
In `integrations/ckprompt/smoke_command.go`, complete the context argument updates
|
||||||
and apply the shared classification to individual failures and the final joined
|
and apply the shared classification to individual failures and the final joined
|
||||||
|
|||||||
@@ -154,7 +154,7 @@ func TestPublicConfigDiffUsesPromptKitPairAndPreparedInputs(t *testing.T) {
|
|||||||
|
|
||||||
var diagnostics bytes.Buffer
|
var diagnostics bytes.Buffer
|
||||||
failure := app.Execute(context.Background(), []string{"config", "diff", "--scope", "promptkit", "--config", "bad-left.yml", "--against-config", "right.yml", "--campaign", "spring", "--against-campaign", "spring"}, commandkit.Streams{Out: io.Discard, Err: &diagnostics})
|
failure := app.Execute(context.Background(), []string{"config", "diff", "--scope", "promptkit", "--config", "bad-left.yml", "--against-config", "right.yml", "--campaign", "spring", "--against-campaign", "spring"}, commandkit.Streams{Out: io.Discard, Err: &diagnostics})
|
||||||
if failure.Kind != commandkit.OutcomeOperational || !strings.Contains(diagnostics.String(), `Configuration scope "promptkit" (left):`) {
|
if failure.Kind != commandkit.OutcomeUsage || !strings.Contains(diagnostics.String(), `Configuration scope "promptkit" (left):`) {
|
||||||
t.Fatalf("installed load failure = %#v, diagnostics %q", failure, diagnostics.String())
|
t.Fatalf("installed load failure = %#v, diagnostics %q", failure, diagnostics.String())
|
||||||
}
|
}
|
||||||
var loadFailure *configkit.LoadError
|
var loadFailure *configkit.LoadError
|
||||||
|
|||||||
100
integrations/ckprompt/configuration_composition_outcomes_test.go
Normal file
100
integrations/ckprompt/configuration_composition_outcomes_test.go
Normal file
@@ -0,0 +1,100 @@
|
|||||||
|
package ckprompt
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/ckconfig"
|
||||||
|
"go.maximumdirect.net/commandkit"
|
||||||
|
"go.maximumdirect.net/configkit"
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestComposedConfigurationInputOutcomes(t *testing.T) {
|
||||||
|
_, loadErr := configkit.Load[map[string]any](configkit.Bytes("prompt-invalid", []byte("broken: [")))
|
||||||
|
engine := adapterEngine(t)
|
||||||
|
calls := 0
|
||||||
|
scope := ConfigScope(func(_ context.Context, selection ckconfig.Selection, _, _ struct{}) (*promptkit.Engine, error) {
|
||||||
|
calls++
|
||||||
|
if selection.Config.Value == "good" {
|
||||||
|
return engine, nil
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("wrapped: %w", loadErr)
|
||||||
|
})
|
||||||
|
app := newPromptTestApp(t, ckconfig.Command(ckconfig.Suite[struct{}, struct{}]{Scopes: []ckconfig.Scope[struct{}, struct{}]{scope}}))
|
||||||
|
cases := []struct {
|
||||||
|
args []string
|
||||||
|
report string
|
||||||
|
side string
|
||||||
|
}{
|
||||||
|
{[]string{"config", "show"}, "", ""},
|
||||||
|
{[]string{"config", "sources", "--format=json"}, `"sources"`, ""},
|
||||||
|
{[]string{"config", "validate", "--details", "--format=json"}, `"code": "yaml.invalid"`, ""},
|
||||||
|
{[]string{"config", "explain", "--list"}, "", ""},
|
||||||
|
{[]string{"config", "diff"}, "", "left"},
|
||||||
|
{[]string{"config", "diff", "--config=good", "--against-config=bad"}, "", "right"},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(strings.Join(tc.args, "_"), func(t *testing.T) {
|
||||||
|
var out, diag bytes.Buffer
|
||||||
|
args := append(append([]string{}, tc.args...), "--scope=promptkit")
|
||||||
|
result := app.Execute(context.Background(), args, commandkit.Streams{Out: &out, Err: &diag})
|
||||||
|
var native *configkit.LoadError
|
||||||
|
if result.Status != 2 || !errors.Is(result.Cause, loadErr) || !errors.As(result.Cause, &native) || !strings.Contains(diag.String(), "prompt-invalid") || (tc.report != "" && !strings.Contains(out.String(), tc.report)) || (tc.side != "" && !strings.Contains(diag.String(), "("+tc.side+")")) {
|
||||||
|
t.Fatalf("result=%#v output=%q diagnostics=%q", result, out.String(), diag.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
before := calls
|
||||||
|
var out bytes.Buffer
|
||||||
|
args := append(append([]string{}, tc.args...), "--scope=promptkit")
|
||||||
|
result := app.Execute(ctx, args, commandkit.Streams{Out: &out, Err: io.Discard})
|
||||||
|
if result.Status != 1 || !errors.Is(result.Cause, context.Canceled) || calls != before || out.Len() != 0 {
|
||||||
|
t.Fatalf("canceled %v: %#v output=%q", tc.args, result, out.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSmokeConfigurationFailureOutcomes(t *testing.T) {
|
||||||
|
_, first := configkit.Load[map[string]any](configkit.Bytes("first", []byte("broken: [")))
|
||||||
|
_, second := configkit.Load[map[string]any](configkit.Bytes("second", []byte("other: [")))
|
||||||
|
operational := errors.New("provider failure")
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name, command string
|
||||||
|
failures []error
|
||||||
|
cancel bool
|
||||||
|
want int
|
||||||
|
}{
|
||||||
|
{"single", "reachability", []error{first}, false, 2},
|
||||||
|
{"all input", "all", []error{first, second, first, second}, false, 2},
|
||||||
|
{"mixed", "all", []error{first, operational, second, first}, false, 1},
|
||||||
|
{"cancellation", "all", []error{first}, true, 1},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
calls := 0
|
||||||
|
app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{SmokeTest: func(_ context.Context, req SmokeRequest, _, _ struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
|
||||||
|
err := tc.failures[calls]
|
||||||
|
calls++
|
||||||
|
if tc.cancel {
|
||||||
|
cancel()
|
||||||
|
}
|
||||||
|
return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: req.Request.Kind}, Explanation: commandkit.Explanation{Message: fmt.Sprintf("context-%d", calls)}}, err
|
||||||
|
}}))
|
||||||
|
var out, diag bytes.Buffer
|
||||||
|
result := app.Execute(ctx, []string{"promptkit", "test", tc.command, "--format=json"}, commandkit.Streams{Out: &out, Err: &diag})
|
||||||
|
if result.Status != tc.want || calls != len(tc.failures) || !errors.Is(result.Cause, first) || (tc.name == "all input" && !errors.Is(result.Cause, second)) || (tc.name == "mixed" && (!errors.Is(result.Cause, second) || !errors.Is(result.Cause, operational))) || (tc.cancel && !errors.Is(result.Cause, context.Canceled)) || out.Len() == 0 || !strings.Contains(diag.String(), "context-1") {
|
||||||
|
t.Fatalf("outcome=%#v calls=%d report=%q diagnostics=%q", result, calls, out.String(), diag.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -170,7 +170,7 @@ func TestComposedIndependentOverridesAndFailureEvidence(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
outcome = app.Execute(context.Background(), []string{"config", "validate", "--scope", "app", "--campaign", "spring", "--limit=0", "--set", `/default_campaign="autumn"`}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
|
outcome = app.Execute(context.Background(), []string{"config", "validate", "--scope", "app", "--campaign", "spring", "--limit=0", "--set", `/default_campaign="autumn"`}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
|
||||||
if outcome.Kind != commandkit.OutcomeOperational || outcome.Cause == nil {
|
if outcome.Kind != commandkit.OutcomeUsage || outcome.Cause == nil {
|
||||||
t.Fatalf("overlapping native overrides: %#v", outcome)
|
t.Fatalf("overlapping native overrides: %#v", outcome)
|
||||||
}
|
}
|
||||||
loads := 0
|
loads := 0
|
||||||
|
|||||||
Reference in New Issue
Block a user