Coordinate aggregate PromptKit smoke tests

integrations/ckprompt/smoke_aggregate_test.go

@@ -0,0 +1,159 @@
+package ckprompt
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"io"
+	"reflect"
+	"testing"
+
+	commandkit "gitea.maximumdirect.net/eric/CommandKit"
+	"gitea.maximumdirect.net/eric/CommandKit/integrations/ckconfig"
+	promptkit "gitea.maximumdirect.net/eric/promptkit"
+	"go.yaml.in/yaml/v4"
+)
+
+func TestSmokeAggregateCoordinatesConcreteRequestsAndRetainsErrors(t *testing.T) {
+	type input struct{ identity *int }
+	type state struct{ identity *int }
+
+	inputIdentity, stateIdentity := 1, 2
+	wantKinds := []promptkit.SmokeTestKind{
+		promptkit.SmokeTestReachability,
+		promptkit.SmokeTestAuthentication,
+		promptkit.SmokeTestTextInference,
+		promptkit.SmokeTestStructuredOutput,
+	}
+	nativeErr := errors.New("authentication smoke failed")
+	newInputCalls, checkCalls, newStateCalls := 0, 0, 0
+	var gotKinds []promptkit.SmokeTestKind
+	var executionPointers []*promptkit.ExecutionTargetOverride
+	var temperaturePointers []*float64
+	var retryPointers []*int
+	var firstInput *input
+	var sharedState *state
+	suite := Suite[*input, *state]{
+		NewInput: func() *input {
+			newInputCalls++
+			return &input{identity: &inputIdentity}
+		},
+		NewState: func() *state {
+			newStateCalls++
+			return &state{identity: &stateIdentity}
+		},
+		CheckInput: func(_ ckconfig.Selection, value *input) error {
+			checkCalls++
+			firstInput = value
+			return nil
+		},
+		SmokeTest: func(_ context.Context, request SmokeRequest, value *input, shared *state) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
+			gotKinds = append(gotKinds, request.Request.Kind)
+			if firstInput != value || value.identity != &inputIdentity || shared.identity != &stateIdentity {
+				t.Errorf("input/state were not shared: input=%p state=%#v", value, shared)
+			}
+			if sharedState == nil {
+				sharedState = shared
+			} else if sharedState != shared {
+				t.Errorf("state pointer changed between smoke tests: first=%p current=%p", sharedState, shared)
+			}
+			if request.Request.Execution == nil || request.Request.Execution.Model != "base-model" || request.Request.Execution.Temperature == nil || *request.Request.Execution.Temperature != 0.25 || request.Request.TransientRetries == nil || *request.Request.TransientRetries != 3 {
+				t.Errorf("seed for %s = %#v", request.Request.Kind, request.Request)
+			}
+			executionPointers = append(executionPointers, request.Request.Execution)
+			temperaturePointers = append(temperaturePointers, request.Request.Execution.Temperature)
+			retryPointers = append(retryPointers, request.Request.TransientRetries)
+			if len(gotKinds) == 1 {
+				request.Request.Execution.Model = "builder mutation"
+				*request.Request.Execution.Temperature = 0.9
+				*request.Request.TransientRetries = 11
+			}
+			result := &promptkit.SmokeTestReport{Kind: promptkit.SmokeTestAuthentication, Supported: true, Completed: true}
+			if request.Request.Kind == promptkit.SmokeTestAuthentication {
+				return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: result}, nativeErr
+			}
+			return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: result}, nil
+		},
+	}
+	app := newPromptTestApp(t, Command(suite))
+	var output bytes.Buffer
+	outcome := app.Execute(context.Background(), []string{
+		"promptkit", "test", "all", "--model=base-model", "--temperature=0.25", "--transient-retries=3",
+	}, commandkit.Streams{Out: &output, Err: io.Discard})
+	if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, nativeErr) {
+		t.Fatalf("aggregate outcome = %#v", outcome)
+	}
+	if newInputCalls != 1 || checkCalls != 1 || newStateCalls != 1 || !reflect.DeepEqual(gotKinds, wantKinds) {
+		t.Fatalf("invocation calls = input %d, check %d, state %d, kinds %#v", newInputCalls, checkCalls, newStateCalls, gotKinds)
+	}
+	for index := range executionPointers {
+		for other := index + 1; other < len(executionPointers); other++ {
+			if executionPointers[index] == executionPointers[other] || temperaturePointers[index] == temperaturePointers[other] || retryPointers[index] == retryPointers[other] {
+				t.Fatalf("request pointers were reused between operations %d and %d", index, other)
+			}
+		}
+	}
+
+	var document struct {
+		Tests []struct {
+			Kind   string `yaml:"kind"`
+			Report *struct {
+				Kind string `yaml:"kind"`
+			} `yaml:"report"`
+			Error string `yaml:"error"`
+		} `yaml:"tests"`
+	}
+	if err := yaml.Unmarshal(output.Bytes(), &document); err != nil {
+		t.Fatalf("decode aggregate report %q: %v", output.String(), err)
+	}
+	if len(document.Tests) != len(wantKinds) {
+		t.Fatalf("aggregate entries = %#v", document.Tests)
+	}
+	for index, test := range document.Tests {
+		if test.Kind != string(wantKinds[index]) || test.Report == nil || test.Report.Kind != string(promptkit.SmokeTestAuthentication) {
+			t.Errorf("aggregate entry %d = %#v", index, test)
+		}
+		if index == 1 && test.Error != nativeErr.Error() {
+			t.Errorf("authentication entry error = %q", test.Error)
+		}
+		if index != 1 && test.Error != "" {
+			t.Errorf("unexpected entry error at %d: %q", index, test.Error)
+		}
+	}
+}
+
+func TestSmokeAggregateCancellationRetainsPartialResults(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	t.Cleanup(cancel)
+	operationErr := errors.New("second smoke operation failed")
+	var gotKinds []promptkit.SmokeTestKind
+	app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{
+		SmokeTest: func(_ context.Context, request SmokeRequest, _ struct{}, _ struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
+			gotKinds = append(gotKinds, request.Request.Kind)
+			if len(gotKinds) == 2 {
+				cancel()
+				return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind}}, operationErr
+			}
+			return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind}}, nil
+		},
+	}))
+	var output bytes.Buffer
+	outcome := app.Execute(ctx, []string{"promptkit", "test", "all"}, commandkit.Streams{Out: &output, Err: io.Discard})
+	if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, operationErr) || !errors.Is(outcome.Cause, context.Canceled) {
+		t.Fatalf("partial aggregate outcome = %#v", outcome)
+	}
+	if !reflect.DeepEqual(gotKinds, []promptkit.SmokeTestKind{promptkit.SmokeTestReachability, promptkit.SmokeTestAuthentication}) {
+		t.Fatalf("callbacks after cancellation = %#v", gotKinds)
+	}
+	var document struct {
+		Tests []struct {
+			Kind string `yaml:"kind"`
+		} `yaml:"tests"`
+	}
+	if err := yaml.Unmarshal(output.Bytes(), &document); err != nil {
+		t.Fatalf("decode partial aggregate report %q: %v", output.String(), err)
+	}
+	if len(document.Tests) != 2 || document.Tests[0].Kind != string(promptkit.SmokeTestReachability) || document.Tests[1].Kind != string(promptkit.SmokeTestAuthentication) {
+		t.Fatalf("partial aggregate entries = %#v", document.Tests)
+	}
+}

integrations/ckprompt/smoke_command.go

@@ -148,10 +148,9 @@ func newSmokeTestCommand[I, S any](suite Suite[I, S]) commandkit.Command {
 			if err := checkSmokeFormat(input.format); err != nil {
 				return err
 			}
-			if input.kind == "all" {
-				return errors.New("PromptKit aggregate smoke tests are not available")
+			if input.kind != "all" {
+				input.request.Kind = promptkit.SmokeTestKind(input.kind)
 			}
-			input.request.Kind = promptkit.SmokeTestKind(input.kind)
 			selected, err := prepareSuiteInput(suite, input.selection, input.captured)
 			if err != nil {
 				return err
@@ -194,6 +193,9 @@ func runSmokeTest[I, S any](invocation commandkit.Invocation, suite Suite[I, S],
 	if err := promptKitContextError(invocation.Context); err != nil {
 		return definitionFailure(err, nil)
 	}
+	if input.kind == "all" {
+		return runSmokeAggregate(invocation, suite, input, state)
+	}
 
 	request := SmokeRequest{Selection: input.selection, Request: input.request}
 	acquired, operationErr := suite.SmokeTest(invocation.Context, request, input.selected, state)
@@ -218,6 +220,99 @@ func runSmokeTest[I, S any](invocation commandkit.Invocation, suite Suite[I, S],
 	return writeSmokeReport(invocation, output, acquired.Diagnostics, operationErr)
 }
 
+func runSmokeAggregate[I, S any](invocation commandkit.Invocation, suite Suite[I, S], input *smokeCommandInput[I], state S) commandkit.Outcome {
+	kinds := [...]promptkit.SmokeTestKind{
+		promptkit.SmokeTestReachability,
+		promptkit.SmokeTestAuthentication,
+		promptkit.SmokeTestTextInference,
+		promptkit.SmokeTestStructuredOutput,
+	}
+	results := make([]smokeResult, 0, len(kinds))
+	var diagnostics []commandkit.Diagnostic
+	var operationErrors []error
+
+	for _, kind := range kinds {
+		if err := promptKitContextError(invocation.Context); err != nil {
+			if len(results) == 0 {
+				return definitionFailure(err, diagnostics)
+			}
+			operationErrors = append(operationErrors, err)
+			break
+		}
+
+		seed := smokeRequestForKind(input.request, kind)
+		acquired, operationErr := suite.SmokeTest(invocation.Context, SmokeRequest{Selection: input.selection, Request: seed}, input.selected, state)
+		if acquired.Value == nil && operationErr == nil {
+			operationErr = errors.New("PromptKit smoke test returned neither a report nor an error")
+		}
+		if err := promptKitContextError(invocation.Context); err != nil && !errors.Is(operationErr, err) {
+			if operationErr == nil {
+				operationErr = err
+			} else {
+				operationErr = errors.Join(operationErr, err)
+			}
+		}
+
+		results = append(results, smokeResult{Kind: kind, Report: acquired.Value, Err: operationErr})
+		diagnostics = append(diagnostics, acquired.Diagnostics...)
+		if operationErr != nil {
+			operationErrors = append(operationErrors, operationErr)
+		}
+		if err := promptKitContextError(invocation.Context); err != nil {
+			break
+		}
+	}
+
+	if len(results) == 0 {
+		return definitionFailure(errors.Join(operationErrors...), diagnostics)
+	}
+	output, encodeErr := encodeSmokeResults(results)
+	if encodeErr != nil {
+		return smokeDeliveryFailure(errors.Join(operationErrors...), encodeErr, diagnostics, false)
+	}
+	return writeSmokeReport(invocation, output, diagnostics, errors.Join(operationErrors...))
+}
+
+func smokeRequestForKind(seed promptkit.SmokeTestRequest, kind promptkit.SmokeTestKind) promptkit.SmokeTestRequest {
+	request := seed
+	request.Kind = kind
+	if seed.Execution != nil {
+		execution := *seed.Execution
+		execution.Temperature = copyFloat64(seed.Execution.Temperature)
+		execution.MaxTokens = copyInt(seed.Execution.MaxTokens)
+		execution.TopP = copyFloat64(seed.Execution.TopP)
+		execution.TimeoutSeconds = copyInt(seed.Execution.TimeoutSeconds)
+		execution.ReasoningEffort = copyString(seed.Execution.ReasoningEffort)
+		request.Execution = &execution
+	}
+	request.TransientRetries = copyInt(seed.TransientRetries)
+	return request
+}
+
+func copyFloat64(value *float64) *float64 {
+	if value == nil {
+		return nil
+	}
+	copy := *value
+	return &copy
+}
+
+func copyInt(value *int) *int {
+	if value == nil {
+		return nil
+	}
+	copy := *value
+	return &copy
+}
+
+func copyString(value *string) *string {
+	if value == nil {
+		return nil
+	}
+	copy := *value
+	return &copy
+}
+
 func writeSmokeReport(invocation commandkit.Invocation, output []byte, diagnostics []commandkit.Diagnostic, operationErr error) commandkit.Outcome {
 	written, deliveryErr := invocation.Streams.Out.Write(output)
 	if deliveryErr == nil && written != len(output) {

integrations/ckprompt/smoke_command_test.go

@@ -93,8 +93,13 @@ func TestSmokeTestCommandBindsRequestAndRendersFindings(t *testing.T) {
 	providerCalls = 0
 	output.Reset()
 	outcome = app.Execute(context.Background(), []string{"promptkit", "test", "all"}, commandkit.Streams{Out: &output, Err: io.Discard})
-	if outcome.Kind != commandkit.OutcomeUsage || providerCalls != 0 || output.Len() != 0 {
-		t.Fatalf("unexpanded aggregate token = (%#v, %q), calls=%d", outcome, output.String(), providerCalls)
+	var aggregate struct {
+		Tests []struct {
+			Kind string `yaml:"kind"`
+		} `yaml:"tests"`
+	}
+	if outcome.Kind != commandkit.OutcomeSuccess || providerCalls != 4 || yaml.Unmarshal(output.Bytes(), &aggregate) != nil || len(aggregate.Tests) != 4 {
+		t.Fatalf("aggregate smoke result = (%#v, %q), calls=%d", outcome, output.String(), providerCalls)
 	}
 }