base-layer/commandkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Keep aggregate smoke tests focused on behavior

Browse Source
This commit is contained in:
Eric Rakestraw
2026-10-18 13:30:02 +00:00
parent 94c352dbd6
commit 114c7da7be
3 changed files with 12 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
docs/roadmap/remediation.md
View File

@@ -1,6 +1,6 @@
# Dependency upgrade and PromptKit smoke review remediation
Status: in progress; R-1 and R-2 complete; R-3 and R-4 remain open.
Status: in progress; R-1 through R-3 complete; R-4 remains open.
## Review scope and evidence
@@ -127,7 +127,7 @@ Production code is unchanged.
## R-3: Remove incidental loader-count and pointer-identity contracts
Status: open. Priority: P3. Category: unnecessary implementation constraints.
Status: complete. Priority: P3. Category: unnecessary implementation constraints.
**Locations:**
@@ -167,6 +167,14 @@ pass. The tests continue to detect cross-test value contamination and missing
operations, while accepting equivalent acquisition and storage implementations.
No production-code change is required.
**Completed:** 2026-10-18. Removed the aggregate loader counter and pairwise
pointer-address assertions/bookkeeping. The tests retain ordered concrete
builder/callback inputs, shared input/state, mutation-isolated seed values,
continuation, cancellation, reports, and error identity. Focused aggregate,
builder, help/version, and builder-failure tests and the complete ckprompt suite,
including the composed consumer, passed with Go 1.26.8 and offline dependency
resolution. Production code is unchanged.
## R-4: Reconcile the feature roadmap with completed implementation
Status: open. Priority: P3. Category: stale implementation status.

13
integrations/ckprompt/smoke_aggregate_test.go
View File

@@ -28,9 +28,6 @@ func TestSmokeAggregateCoordinatesConcreteRequestsAndRetainsErrors(t *testing.T)
nativeErr := errors.New("authentication smoke failed")
newInputCalls, checkCalls, newStateCalls := 0, 0, 0
var gotKinds []promptkit.SmokeTestKind
var executionPointers []*promptkit.ExecutionTargetOverride
var temperaturePointers []*float64
var retryPointers []*int
var firstInput *input
var sharedState *state
suite := Suite[*input, *state]{
@@ -60,9 +57,6 @@ func TestSmokeAggregateCoordinatesConcreteRequestsAndRetainsErrors(t *testing.T)
if request.Request.Execution == nil || request.Request.Execution.Model != "base-model" || request.Request.Execution.Temperature == nil || *request.Request.Execution.Temperature != 0.25 || request.Request.TransientRetries == nil || *request.Request.TransientRetries != 3 {
t.Errorf("seed for %s = %#v", request.Request.Kind, request.Request)
}
executionPointers = append(executionPointers, request.Request.Execution)
temperaturePointers = append(temperaturePointers, request.Request.Execution.Temperature)
retryPointers = append(retryPointers, request.Request.TransientRetries)
if len(gotKinds) == 1 {
request.Request.Execution.Model = "builder mutation"
*request.Request.Execution.Temperature = 0.9
@@ -86,13 +80,6 @@ func TestSmokeAggregateCoordinatesConcreteRequestsAndRetainsErrors(t *testing.T)
if newInputCalls != 1 || checkCalls != 1 || newStateCalls != 1 || !reflect.DeepEqual(gotKinds, wantKinds) {
t.Fatalf("invocation calls = input %d, check %d, state %d, kinds %#v", newInputCalls, checkCalls, newStateCalls, gotKinds)
}
for index := range executionPointers {
for other := index + 1; other < len(executionPointers); other++ {
if executionPointers[index] == executionPointers[other] || temperaturePointers[index] == temperaturePointers[other] || retryPointers[index] == retryPointers[other] {
t.Fatalf("request pointers were reused between operations %d and %d", index, other)
}
}
}
var document struct {
Tests []struct {

6
integrations/ckprompt/smoke_lifecycle_test.go
View File

@@ -198,9 +198,7 @@ func TestSmokeTestBuilderReceivesConcreteAggregateKinds(t *testing.T) {
}
var builtKinds []promptkit.SmokeTestKind
loadErr := errors.New("engine unavailable")
loads := 0
suite := WithSmokeTesting(Suite[struct{}, struct{}]{}, func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) {
loads++
return nil, loadErr
}, func(_ context.Context, request SmokeRequest, _ struct{}, _ struct{}) (promptkit.SmokeTestRequest, error) {
builtKinds = append(builtKinds, request.Request.Kind)
@@ -208,8 +206,8 @@ func TestSmokeTestBuilderReceivesConcreteAggregateKinds(t *testing.T) {
})
app := newPromptTestApp(t, Command(suite))
outcome := app.Execute(context.Background(), []string{"promptkit", "test", "all"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, loadErr) || loads != 4 || !reflect.DeepEqual(builtKinds, wantKinds) {
t.Fatalf("aggregate builder lifecycle = %#v, built=%#v loads=%d", outcome, builtKinds, loads)
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, loadErr) || !reflect.DeepEqual(builtKinds, wantKinds) {
t.Fatalf("aggregate builder lifecycle = %#v, built=%#v", outcome, builtKinds)
}
}