diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 19ea49e..b323afd 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1,6 +1,6 @@ # Dependency upgrades, discovery, and smoke testing implementation plan -Status: Stages 1–10 complete; Stage 11 is next. +Status: Stages 1–11 complete; Stage 12 is next. ## Authority and baseline @@ -577,6 +577,15 @@ Run focused smoke command tests. **Complete when:** all single-kind bindings invoke the optional provider and faithfully report its result and errors. +**Completed:** 2026-10-17. Added the optional `promptkit test KIND` command with +all accepted scalar bindings, YAML preflight, prepared input/state, and native +report delivery. Explicit zero/empty overrides remain present, omitted +execution/retry overrides remain absent, and `all` is rejected before native +request construction pending aggregate coordination. Assembled tests cover +flag forwarding, findings, report-plus-error, nil-result contract failure, and +short-write delivery. `GOTOOLCHAIN=local go test ./... -run +'TestSmokeTestCommand' -count=1` passed in ckprompt. + ## Stage 12: Coordinate aggregate smoke execution **Primary files:** smoke command runner, aggregate projection/outcome helpers, diff --git a/integrations/ckprompt/command.go b/integrations/ckprompt/command.go index efa5a4e..5097cac 100644 --- a/integrations/ckprompt/command.go +++ b/integrations/ckprompt/command.go @@ -53,8 +53,8 @@ type inputAssignment struct { } // Command captures the optional PromptKit catalogs, named inspection, -// requirements discovery, and concrete request validation callbacks and adds -// only supplied operations. Discovery and validation share the +// requirements discovery, concrete request validation, and smoke-test +// callbacks and adds only supplied operations. Discovery and validation share the // `requirements` group and remain independently optional. An absent suite // claims no namespace; attached construction issues remain reportable. It adds // no configuration family; callers compose ConfigScope independently. @@ -67,7 +67,7 @@ type inputAssignment struct { // render YAML with `yaml` as the only format and default. Named-show formats are // unchanged. func Command[I, S any](suite Suite[I, S]) commandkit.Command { - if suite.InspectPrompt == nil && suite.InspectProfile == nil && suite.InspectRequirements == nil && suite.ValidateRequest == nil && suite.ListPrompts == nil && suite.ListProfiles == nil && suite.ListBackends == nil && suite.ListNamespaces == nil && suite.ListCollections == nil { + if suite.InspectPrompt == nil && suite.InspectProfile == nil && suite.InspectRequirements == nil && suite.ValidateRequest == nil && suite.SmokeTest == nil && suite.ListPrompts == nil && suite.ListProfiles == nil && suite.ListBackends == nil && suite.ListNamespaces == nil && suite.ListCollections == nil { command := commandkit.NoCommand() for _, issue := range suite.issues { command = commandkit.WithError(command, issue) @@ -116,7 +116,10 @@ func Command[I, S any](suite Suite[I, S]) commandkit.Command { if len(requirementsCommands) != 0 { rootCommands = append(rootCommands, commandkit.Group("requirements", "Inspect and validate prompt requirements.", requirementsCommands...)) } - command := commandkit.RootGroup("promptkit", "Inspect, validate, and list PromptKit definitions and requirements.", rootCommands...) + if suite.SmokeTest != nil { + rootCommands = append(rootCommands, newSmokeTestCommand(suite)) + } + command := commandkit.RootGroup("promptkit", "Inspect, list, validate requests, and run PromptKit smoke tests.", rootCommands...) for _, issue := range suite.issues { command = commandkit.WithError(command, issue) } diff --git a/integrations/ckprompt/smoke_command.go b/integrations/ckprompt/smoke_command.go new file mode 100644 index 0000000..ad0f9d3 --- /dev/null +++ b/integrations/ckprompt/smoke_command.go @@ -0,0 +1,246 @@ +package ckprompt + +import ( + "errors" + "fmt" + "io" + "strconv" + + commandkit "gitea.maximumdirect.net/eric/CommandKit" + "gitea.maximumdirect.net/eric/CommandKit/integrations/ckconfig" + promptkit "gitea.maximumdirect.net/eric/promptkit" +) + +type smokeCommandInput[I any] struct { + selection ckconfig.Selection + kind string + request promptkit.SmokeTestRequest + format string + captured commandkit.FlagValues[I] + selected I +} + +func newSmokeTestCommand[I, S any](suite Suite[I, S]) commandkit.Command { + flags := commandkit.CaptureFlags(suite.Selectors, func(input **smokeCommandInput[I]) *commandkit.FlagValues[I] { + if input == nil || *input == nil { + return nil + } + return &(*input).captured + }) + flags = append(flags, + commandkit.ValueFlag(commandkit.FlagMeta{Name: "config", Usage: "Configuration input path."}, (*string)(nil), parseDefinitionString, + func(input **smokeCommandInput[I], value commandkit.Supplied[string]) { + (*input).selection.Config = value + }), + commandkit.ValueFlag(commandkit.FlagMeta{Name: "overlay", Usage: "Configuration overlay path."}, (*string)(nil), parseDefinitionString, + func(input **smokeCommandInput[I], value commandkit.Supplied[string]) { + (*input).selection.Overlay = value + }), + commandkit.ValueFlag(commandkit.FlagMeta{Name: "format", Usage: "Smoke report format: yaml."}, stringDefault("yaml"), parseDefinitionString, + func(input **smokeCommandInput[I], value commandkit.Supplied[string]) { (*input).format = value.Value }), + commandkit.ValueFlag(commandkit.FlagMeta{Name: "backend", Usage: "PromptKit backend ID."}, (*string)(nil), parseDefinitionString, + func(input **smokeCommandInput[I], value commandkit.Supplied[string]) { + (*input).request.BackendID = value.Value + }), + commandkit.ValueFlag(commandkit.FlagMeta{Name: "prompt", Usage: "PromptKit prompt ID."}, (*string)(nil), parseDefinitionString, + func(input **smokeCommandInput[I], value commandkit.Supplied[string]) { + (*input).request.PromptID = value.Value + }), + commandkit.ValueFlag(commandkit.FlagMeta{Name: "profile", Usage: "PromptKit profile ID."}, (*string)(nil), parseDefinitionString, + func(input **smokeCommandInput[I], value commandkit.Supplied[string]) { + (*input).request.ProfileID = value.Value + }), + commandkit.ValueFlag(commandkit.FlagMeta{Name: "endpoint", Usage: "Execution endpoint override."}, (*string)(nil), parseDefinitionString, + func(input **smokeCommandInput[I], value commandkit.Supplied[string]) { + if !value.Set { + return + } + execution := smokeExecution(input) + execution.Endpoint = value.Value + }), + commandkit.ValueFlag(commandkit.FlagMeta{Name: "model", Usage: "Execution model override."}, (*string)(nil), parseDefinitionString, + func(input **smokeCommandInput[I], value commandkit.Supplied[string]) { + if !value.Set { + return + } + execution := smokeExecution(input) + execution.Model = value.Value + }), + commandkit.ValueFlag(commandkit.FlagMeta{Name: "temperature", Usage: "Execution temperature override."}, (*float64)(nil), parseSmokeFloat, + func(input **smokeCommandInput[I], value commandkit.Supplied[float64]) { + if !value.Set { + return + } + execution := smokeExecution(input) + execution.Temperature = &value.Value + }), + commandkit.ValueFlag(commandkit.FlagMeta{Name: "max-tokens", Usage: "Execution maximum tokens override."}, (*int)(nil), strconv.Atoi, + func(input **smokeCommandInput[I], value commandkit.Supplied[int]) { + if !value.Set { + return + } + execution := smokeExecution(input) + execution.MaxTokens = &value.Value + }), + commandkit.ValueFlag(commandkit.FlagMeta{Name: "top-p", Usage: "Execution top-p override."}, (*float64)(nil), parseSmokeFloat, + func(input **smokeCommandInput[I], value commandkit.Supplied[float64]) { + if !value.Set { + return + } + execution := smokeExecution(input) + execution.TopP = &value.Value + }), + commandkit.ValueFlag(commandkit.FlagMeta{Name: "timeout-seconds", Usage: "Execution timeout override."}, (*int)(nil), strconv.Atoi, + func(input **smokeCommandInput[I], value commandkit.Supplied[int]) { + if !value.Set { + return + } + execution := smokeExecution(input) + execution.TimeoutSeconds = &value.Value + }), + commandkit.ValueFlag(commandkit.FlagMeta{Name: "service-tier", Usage: "Execution service tier."}, (*string)(nil), parseDefinitionString, + func(input **smokeCommandInput[I], value commandkit.Supplied[string]) { + if !value.Set { + return + } + execution := smokeExecution(input) + execution.ServiceTier = value.Value + }), + commandkit.ValueFlag(commandkit.FlagMeta{Name: "reasoning-effort", Usage: "Execution reasoning effort override."}, (*string)(nil), parseDefinitionString, + func(input **smokeCommandInput[I], value commandkit.Supplied[string]) { + if !value.Set { + return + } + execution := smokeExecution(input) + execution.ReasoningEffort = &value.Value + }), + commandkit.ValueFlag(commandkit.FlagMeta{Name: "api-key-env", Usage: "Execution API key environment variable."}, (*string)(nil), parseDefinitionString, + func(input **smokeCommandInput[I], value commandkit.Supplied[string]) { + if !value.Set { + return + } + execution := smokeExecution(input) + execution.APIKeyEnv = value.Value + }), + commandkit.ValueFlag(commandkit.FlagMeta{Name: "transient-retries", Usage: "Inference transient retry count."}, (*int)(nil), strconv.Atoi, + func(input **smokeCommandInput[I], value commandkit.Supplied[int]) { + if value.Set { + (*input).request.TransientRetries = &value.Value + } + }), + ) + + return commandkit.NewCommand(commandkit.CommandSpec[*smokeCommandInput[I]]{ + Name: "test", + Summary: "Run one PromptKit smoke test.", + NewInput: func() *smokeCommandInput[I] { + return &smokeCommandInput[I]{format: "yaml"} + }, + Flags: flags, + Arguments: []commandkit.Argument[*smokeCommandInput[I]]{ + commandkit.ValueArgument(commandkit.ArgumentMeta{Name: "kind", Usage: "Native smoke-test kind."}, true, parseDefinitionString, + func(input **smokeCommandInput[I], value commandkit.Supplied[string]) { (*input).kind = value.Value }), + }, + CheckInput: func(input *smokeCommandInput[I]) error { + if input == nil { + return errors.New("PromptKit smoke-test input must not be nil") + } + if err := checkSmokeFormat(input.format); err != nil { + return err + } + if input.kind == "all" { + return errors.New("PromptKit aggregate smoke tests are not available") + } + input.request.Kind = promptkit.SmokeTestKind(input.kind) + selected, err := prepareSuiteInput(suite, input.selection, input.captured) + if err != nil { + return err + } + input.selected = selected + return nil + }, + Run: func(invocation commandkit.Invocation, input *smokeCommandInput[I]) commandkit.Outcome { + return runSmokeTest(invocation, suite, input) + }, + }) +} + +func smokeExecution[I any](input **smokeCommandInput[I]) *promptkit.ExecutionTargetOverride { + if (*input).request.Execution == nil { + (*input).request.Execution = &promptkit.ExecutionTargetOverride{} + } + return (*input).request.Execution +} + +func parseSmokeFloat(value string) (float64, error) { + return strconv.ParseFloat(value, 64) +} + +func checkSmokeFormat(format string) error { + if format != "yaml" { + return fmt.Errorf("unsupported smoke report format %q; expected yaml", format) + } + return nil +} + +func runSmokeTest[I, S any](invocation commandkit.Invocation, suite Suite[I, S], input *smokeCommandInput[I]) commandkit.Outcome { + if err := promptKitContextError(invocation.Context); err != nil { + return definitionFailure(err, nil) + } + var state S + if suite.NewState != nil { + state = suite.NewState() + } + if err := promptKitContextError(invocation.Context); err != nil { + return definitionFailure(err, nil) + } + + request := SmokeRequest{Selection: input.selection, Request: input.request} + acquired, operationErr := suite.SmokeTest(invocation.Context, request, input.selected, state) + if acquired.Value == nil && operationErr == nil { + operationErr = errors.New("PromptKit smoke test returned neither a report nor an error") + } + if err := promptKitContextError(invocation.Context); err != nil && !errors.Is(operationErr, err) { + if operationErr == nil { + operationErr = err + } else { + operationErr = errors.Join(operationErr, err) + } + } + if acquired.Value == nil { + return definitionFailure(operationErr, acquired.Diagnostics) + } + + output, encodeErr := encodeSmokeReport(acquired.Value) + if encodeErr != nil { + return smokeDeliveryFailure(operationErr, encodeErr, acquired.Diagnostics, false) + } + return writeSmokeReport(invocation, output, acquired.Diagnostics, operationErr) +} + +func writeSmokeReport(invocation commandkit.Invocation, output []byte, diagnostics []commandkit.Diagnostic, operationErr error) commandkit.Outcome { + written, deliveryErr := invocation.Streams.Out.Write(output) + if deliveryErr == nil && written != len(output) { + deliveryErr = io.ErrShortWrite + } + if deliveryErr != nil { + return smokeDeliveryFailure(operationErr, deliveryErr, diagnostics, true) + } + if operationErr != nil { + return definitionFailure(operationErr, diagnostics) + } + outcome := commandkit.Success() + outcome.Diagnostics = diagnostics + return outcome +} + +func smokeDeliveryFailure(operationErr, deliveryErr error, diagnostics []commandkit.Diagnostic, reportStarted bool) commandkit.Outcome { + cause := operationErr + if cause == nil { + cause = deliveryErr + } + outcome := definitionFailure(cause, diagnostics) + outcome.DeliveryError = deliveryErr + outcome.ReportIncomplete = reportStarted + return outcome +} diff --git a/integrations/ckprompt/smoke_command_test.go b/integrations/ckprompt/smoke_command_test.go new file mode 100644 index 0000000..e30b29f --- /dev/null +++ b/integrations/ckprompt/smoke_command_test.go @@ -0,0 +1,145 @@ +package ckprompt + +import ( + "bytes" + "context" + "errors" + "io" + "strings" + "testing" + + commandkit "gitea.maximumdirect.net/eric/CommandKit" + "gitea.maximumdirect.net/eric/CommandKit/integrations/ckconfig" + promptkit "gitea.maximumdirect.net/eric/promptkit" + "go.yaml.in/yaml/v4" +) + +func TestSmokeTestCommandBindsRequestAndRendersFindings(t *testing.T) { + type input struct{ campaign string } + type state struct{ initialized bool } + campaignFlag := commandkit.ValueFlag(commandkit.FlagMeta{Name: "campaign", Usage: "Campaign selector."}, (*string)(nil), parseDefinitionString, + func(value *input, campaign commandkit.Supplied[string]) { value.campaign = campaign.Value }) + + var gotSelection ckconfig.Selection + var gotRequest promptkit.SmokeTestRequest + var gotInput input + var gotState state + providerCalls, checkedCalls := 0, 0 + suite := Suite[input, state]{ + NewInput: func() input { return input{} }, + NewState: func() state { return state{initialized: true} }, + Selectors: []commandkit.Flag[input]{campaignFlag}, + CheckInput: func(selection ckconfig.Selection, value input) error { + checkedCalls++ + if checkedCalls == 1 && (!selection.Config.Set || selection.Config.Value != "config.yml" || selection.Overlay.Value != "overlay.yml" || value.campaign != "spring") { + t.Errorf("checked selection and input = (%#v, %#v)", selection, value) + } + return nil + }, + SmokeTest: func(_ context.Context, request SmokeRequest, value input, status state) (commandkit.Acquired[*promptkit.SmokeTestReport], error) { + providerCalls++ + gotSelection, gotRequest, gotInput, gotState = request.Selection, request.Request, value, status + return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind, Supported: true, Completed: true}}, nil + }, + } + app := newPromptTestApp(t, Command(suite)) + + var help bytes.Buffer + outcome := app.Execute(context.Background(), []string{"help", "promptkit"}, commandkit.Streams{Out: &help, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(help.String(), "test") { + t.Fatalf("promptkit help = (%#v, %q)", outcome, help.String()) + } + + var output bytes.Buffer + outcome = app.Execute(context.Background(), []string{ + "promptkit", "test", "authentication", "--config=config.yml", "--overlay=overlay.yml", "--campaign=spring", + "--backend=local", "--prompt=briefing", "--profile=editorial", "--endpoint=https://example.invalid", + "--model=fixture", "--temperature=0", "--max-tokens=0", "--top-p=0", "--timeout-seconds=0", + "--service-tier=", "--reasoning-effort=", "--api-key-env=TOKEN", "--transient-retries=0", + }, commandkit.Streams{Out: &output, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeSuccess || providerCalls != 1 { + t.Fatalf("smoke outcome = (%#v, %q), calls=%d", outcome, output.String(), providerCalls) + } + if gotSelection.Config.Value != "config.yml" || gotSelection.Overlay.Value != "overlay.yml" || gotInput.campaign != "spring" || !gotState.initialized { + t.Errorf("provider selection/input/state = (%#v, %#v, %#v)", gotSelection, gotInput, gotState) + } + if gotRequest.Kind != promptkit.SmokeTestAuthentication || gotRequest.BackendID != "local" || gotRequest.PromptID != "briefing" || gotRequest.ProfileID != "editorial" { + t.Errorf("smoke request identity = %#v", gotRequest) + } + if gotRequest.Execution == nil || gotRequest.Execution.Endpoint != "https://example.invalid" || gotRequest.Execution.Model != "fixture" || gotRequest.Execution.Temperature == nil || *gotRequest.Execution.Temperature != 0 || gotRequest.Execution.MaxTokens == nil || *gotRequest.Execution.MaxTokens != 0 || gotRequest.Execution.TopP == nil || *gotRequest.Execution.TopP != 0 || gotRequest.Execution.TimeoutSeconds == nil || *gotRequest.Execution.TimeoutSeconds != 0 || gotRequest.Execution.ServiceTier != "" || gotRequest.Execution.ReasoningEffort == nil || *gotRequest.Execution.ReasoningEffort != "" || gotRequest.Execution.APIKeyEnv != "TOKEN" || gotRequest.TransientRetries == nil || *gotRequest.TransientRetries != 0 { + t.Errorf("smoke execution overrides did not preserve supplied values: %#v", gotRequest) + } + var report map[string]any + if err := yaml.Unmarshal(output.Bytes(), &report); err != nil { + t.Fatalf("decode smoke report %q: %v", output.String(), err) + } + if report["kind"] != "authentication" || report["completed"] != true { + t.Errorf("smoke report = %#v", report) + } + + var omittedRequest promptkit.SmokeTestRequest + omittedApp := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ + SmokeTest: func(_ context.Context, request SmokeRequest, _ struct{}, _ struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) { + omittedRequest = request.Request + return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind}}, nil + }, + })) + output.Reset() + outcome = omittedApp.Execute(context.Background(), []string{"promptkit", "test", "text-inference"}, commandkit.Streams{Out: &output, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeSuccess || omittedRequest.Execution != nil || omittedRequest.TransientRetries != nil { + t.Fatalf("omitted execution overrides = (%#v, %#v), execution nil=%t retries nil=%t", outcome, omittedRequest, omittedRequest.Execution == nil, omittedRequest.TransientRetries == nil) + } + + providerCalls = 0 + output.Reset() + outcome = app.Execute(context.Background(), []string{"promptkit", "test", "all"}, commandkit.Streams{Out: &output, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeUsage || providerCalls != 0 || output.Len() != 0 { + t.Fatalf("unexpanded aggregate token = (%#v, %q), calls=%d", outcome, output.String(), providerCalls) + } +} + +func TestSmokeTestCommandDeliversReportWithOperationError(t *testing.T) { + sentinel := errors.New("native smoke operation failed") + app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ + SmokeTest: func(context.Context, SmokeRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) { + return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: promptkit.SmokeTestAuthentication, Completed: true}}, sentinel + }, + })) + var output bytes.Buffer + outcome := app.Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: &output, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, sentinel) || output.Len() == 0 { + t.Fatalf("report-plus-error outcome = (%#v, %q)", outcome, output.String()) + } + + app = newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ + SmokeTest: func(context.Context, SmokeRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) { + return commandkit.Acquired[*promptkit.SmokeTestReport]{}, nil + }, + })) + output.Reset() + outcome = app.Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: &output, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeOperational || outcome.Cause == nil || output.Len() != 0 { + t.Fatalf("empty acquisition outcome = (%#v, %q)", outcome, output.String()) + } +} + +type smokeShortWriter struct{} + +func (smokeShortWriter) Write(value []byte) (int, error) { + if len(value) == 0 { + return 0, nil + } + return len(value) - 1, nil +} + +func TestSmokeTestCommandRetainsShortReportWrite(t *testing.T) { + app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ + SmokeTest: func(context.Context, SmokeRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) { + return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: promptkit.SmokeTestAuthentication}}, nil + }, + })) + outcome := app.Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: smokeShortWriter{}, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, io.ErrShortWrite) || !errors.Is(outcome.DeliveryError, io.ErrShortWrite) || !outcome.ReportIncomplete { + t.Fatalf("short report delivery = %#v", outcome) + } +}