Keep aggregate smoke tests focused on behavior
This commit is contained in:
@@ -1,6 +1,6 @@
|
|||||||
# Dependency upgrade and PromptKit smoke review remediation
|
# Dependency upgrade and PromptKit smoke review remediation
|
||||||
|
|
||||||
Status: in progress; R-1 and R-2 complete; R-3 and R-4 remain open.
|
Status: in progress; R-1 through R-3 complete; R-4 remains open.
|
||||||
|
|
||||||
## Review scope and evidence
|
## Review scope and evidence
|
||||||
|
|
||||||
@@ -127,7 +127,7 @@ Production code is unchanged.
|
|||||||
|
|
||||||
## R-3: Remove incidental loader-count and pointer-identity contracts
|
## R-3: Remove incidental loader-count and pointer-identity contracts
|
||||||
|
|
||||||
Status: open. Priority: P3. Category: unnecessary implementation constraints.
|
Status: complete. Priority: P3. Category: unnecessary implementation constraints.
|
||||||
|
|
||||||
**Locations:**
|
**Locations:**
|
||||||
|
|
||||||
@@ -167,6 +167,14 @@ pass. The tests continue to detect cross-test value contamination and missing
|
|||||||
operations, while accepting equivalent acquisition and storage implementations.
|
operations, while accepting equivalent acquisition and storage implementations.
|
||||||
No production-code change is required.
|
No production-code change is required.
|
||||||
|
|
||||||
|
**Completed:** 2026-10-18. Removed the aggregate loader counter and pairwise
|
||||||
|
pointer-address assertions/bookkeeping. The tests retain ordered concrete
|
||||||
|
builder/callback inputs, shared input/state, mutation-isolated seed values,
|
||||||
|
continuation, cancellation, reports, and error identity. Focused aggregate,
|
||||||
|
builder, help/version, and builder-failure tests and the complete ckprompt suite,
|
||||||
|
including the composed consumer, passed with Go 1.26.8 and offline dependency
|
||||||
|
resolution. Production code is unchanged.
|
||||||
|
|
||||||
## R-4: Reconcile the feature roadmap with completed implementation
|
## R-4: Reconcile the feature roadmap with completed implementation
|
||||||
|
|
||||||
Status: open. Priority: P3. Category: stale implementation status.
|
Status: open. Priority: P3. Category: stale implementation status.
|
||||||
|
|||||||
@@ -28,9 +28,6 @@ func TestSmokeAggregateCoordinatesConcreteRequestsAndRetainsErrors(t *testing.T)
|
|||||||
nativeErr := errors.New("authentication smoke failed")
|
nativeErr := errors.New("authentication smoke failed")
|
||||||
newInputCalls, checkCalls, newStateCalls := 0, 0, 0
|
newInputCalls, checkCalls, newStateCalls := 0, 0, 0
|
||||||
var gotKinds []promptkit.SmokeTestKind
|
var gotKinds []promptkit.SmokeTestKind
|
||||||
var executionPointers []*promptkit.ExecutionTargetOverride
|
|
||||||
var temperaturePointers []*float64
|
|
||||||
var retryPointers []*int
|
|
||||||
var firstInput *input
|
var firstInput *input
|
||||||
var sharedState *state
|
var sharedState *state
|
||||||
suite := Suite[*input, *state]{
|
suite := Suite[*input, *state]{
|
||||||
@@ -60,9 +57,6 @@ func TestSmokeAggregateCoordinatesConcreteRequestsAndRetainsErrors(t *testing.T)
|
|||||||
if request.Request.Execution == nil || request.Request.Execution.Model != "base-model" || request.Request.Execution.Temperature == nil || *request.Request.Execution.Temperature != 0.25 || request.Request.TransientRetries == nil || *request.Request.TransientRetries != 3 {
|
if request.Request.Execution == nil || request.Request.Execution.Model != "base-model" || request.Request.Execution.Temperature == nil || *request.Request.Execution.Temperature != 0.25 || request.Request.TransientRetries == nil || *request.Request.TransientRetries != 3 {
|
||||||
t.Errorf("seed for %s = %#v", request.Request.Kind, request.Request)
|
t.Errorf("seed for %s = %#v", request.Request.Kind, request.Request)
|
||||||
}
|
}
|
||||||
executionPointers = append(executionPointers, request.Request.Execution)
|
|
||||||
temperaturePointers = append(temperaturePointers, request.Request.Execution.Temperature)
|
|
||||||
retryPointers = append(retryPointers, request.Request.TransientRetries)
|
|
||||||
if len(gotKinds) == 1 {
|
if len(gotKinds) == 1 {
|
||||||
request.Request.Execution.Model = "builder mutation"
|
request.Request.Execution.Model = "builder mutation"
|
||||||
*request.Request.Execution.Temperature = 0.9
|
*request.Request.Execution.Temperature = 0.9
|
||||||
@@ -86,13 +80,6 @@ func TestSmokeAggregateCoordinatesConcreteRequestsAndRetainsErrors(t *testing.T)
|
|||||||
if newInputCalls != 1 || checkCalls != 1 || newStateCalls != 1 || !reflect.DeepEqual(gotKinds, wantKinds) {
|
if newInputCalls != 1 || checkCalls != 1 || newStateCalls != 1 || !reflect.DeepEqual(gotKinds, wantKinds) {
|
||||||
t.Fatalf("invocation calls = input %d, check %d, state %d, kinds %#v", newInputCalls, checkCalls, newStateCalls, gotKinds)
|
t.Fatalf("invocation calls = input %d, check %d, state %d, kinds %#v", newInputCalls, checkCalls, newStateCalls, gotKinds)
|
||||||
}
|
}
|
||||||
for index := range executionPointers {
|
|
||||||
for other := index + 1; other < len(executionPointers); other++ {
|
|
||||||
if executionPointers[index] == executionPointers[other] || temperaturePointers[index] == temperaturePointers[other] || retryPointers[index] == retryPointers[other] {
|
|
||||||
t.Fatalf("request pointers were reused between operations %d and %d", index, other)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
var document struct {
|
var document struct {
|
||||||
Tests []struct {
|
Tests []struct {
|
||||||
|
|||||||
@@ -198,9 +198,7 @@ func TestSmokeTestBuilderReceivesConcreteAggregateKinds(t *testing.T) {
|
|||||||
}
|
}
|
||||||
var builtKinds []promptkit.SmokeTestKind
|
var builtKinds []promptkit.SmokeTestKind
|
||||||
loadErr := errors.New("engine unavailable")
|
loadErr := errors.New("engine unavailable")
|
||||||
loads := 0
|
|
||||||
suite := WithSmokeTesting(Suite[struct{}, struct{}]{}, func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) {
|
suite := WithSmokeTesting(Suite[struct{}, struct{}]{}, func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) {
|
||||||
loads++
|
|
||||||
return nil, loadErr
|
return nil, loadErr
|
||||||
}, func(_ context.Context, request SmokeRequest, _ struct{}, _ struct{}) (promptkit.SmokeTestRequest, error) {
|
}, func(_ context.Context, request SmokeRequest, _ struct{}, _ struct{}) (promptkit.SmokeTestRequest, error) {
|
||||||
builtKinds = append(builtKinds, request.Request.Kind)
|
builtKinds = append(builtKinds, request.Request.Kind)
|
||||||
@@ -208,8 +206,8 @@ func TestSmokeTestBuilderReceivesConcreteAggregateKinds(t *testing.T) {
|
|||||||
})
|
})
|
||||||
app := newPromptTestApp(t, Command(suite))
|
app := newPromptTestApp(t, Command(suite))
|
||||||
outcome := app.Execute(context.Background(), []string{"promptkit", "test", "all"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
|
outcome := app.Execute(context.Background(), []string{"promptkit", "test", "all"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
|
||||||
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, loadErr) || loads != 4 || !reflect.DeepEqual(builtKinds, wantKinds) {
|
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, loadErr) || !reflect.DeepEqual(builtKinds, wantKinds) {
|
||||||
t.Fatalf("aggregate builder lifecycle = %#v, built=%#v loads=%d", outcome, builtKinds, loads)
|
t.Fatalf("aggregate builder lifecycle = %#v, built=%#v", outcome, builtKinds)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user