base-layer/commandkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Suggest PromptKit smoke test kinds

Browse Source
This commit is contained in:
Eric Rakestraw
2026-11-03 13:13:01 +00:00
parent 8964ad02ad
commit 5227049de6
4 changed files with 29 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
integrations/ckprompt/completion.go
View File

@@ -9,3 +9,12 @@ func completionChoices(values ...string) commandkit.CompletionHint {
}
return commandkit.CompleteChoices(choices...)
}
func smokeKindCompletion() commandkit.CompletionHint {
kinds := aggregateSmokeKinds()
values := make([]string, 0, len(kinds)+1)
for _, kind := range kinds {
values = append(values, string(kind))
}
return completionChoices(append(values, "all")...)
}

14
integrations/ckprompt/examples/composed/main_test.go
View File

@@ -353,12 +353,16 @@ func TestComposedCatalogsAndSmokeUseInjectedClientOnlyForInference(t *testing.T)
}
func TestComposedCompletionUsesHintsWithoutLoadingEngine(t *testing.T) {
var loads, calls int
var loads, calls, smokeCalls int
loader := func(ctx context.Context, selection ckconfig.Selection, input *inspectionInput, cache *runCache) (*promptkit.Engine, error) {
loads++
return loadPromptEngine(ctx, selection, input, cache)
}
app, err := newAppWithEngineLoader(&exampleLLMClient{calls: &calls}, loader, nil)
smoke := func(context.Context, ckprompt.SmokeRequest, *inspectionInput, *runCache) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
smokeCalls++
return commandkit.Acquired[*promptkit.SmokeTestReport]{}, nil
}
app, err := newAppWithEngineLoader(&exampleLLMClient{calls: &calls}, loader, smoke)
if err != nil {
t.Fatal(err)
}
@@ -366,6 +370,8 @@ func TestComposedCompletionUsesHintsWithoutLoadingEngine(t *testing.T) {
args []string
want string
}{
{[]string{"__complete", "promptkit", "test", "re"}, "reachability\n"},
{[]string{"__complete", "smoke", "al"}, "all\n"},
{[]string{"__complete", "pk", "show", "prompt", "briefing", "--fmt=y"}, "yaml\n"},
{[]string{"__complete", "pk", "show", "prompt", "briefing", "--config=fixtures/"}, ":0\n"},
} {
@@ -375,8 +381,8 @@ func TestComposedCompletionUsesHintsWithoutLoadingEngine(t *testing.T) {
t.Fatalf("query %q outcome=%#v output=%q", tc.args, outcome, output.String())
}
}
if loads != 0 || calls != 0 {
t.Fatalf("completion loaded engine %d times and invoked client %d times", loads, calls)
if loads != 0 || calls != 0 || smokeCalls != 0 {
t.Fatalf("completion loaded engine %d times and invoked client %d times, smoke calls=%d", loads, calls, smokeCalls)
}
}

10
integrations/ckprompt/smoke_command.go
View File

@@ -138,7 +138,7 @@ func newSmokeTestCommand[I, S any](suite Suite[I, S]) commandkit.Command {
},
Flags: flags,
Arguments: []commandkit.Argument[*smokeCommandInput[I]]{
commandkit.ValueArgument(commandkit.ArgumentMeta{Name: "kind", Usage: "Native smoke-test kind."}, true, parseDefinitionString,
commandkit.ValueArgument(commandkit.ArgumentMeta{Name: "kind", Usage: "Native smoke-test kind.", Completion: smokeKindCompletion()}, true, parseDefinitionString,
func(input **smokeCommandInput[I], value commandkit.Supplied[string]) { (*input).kind = value.Value }),
},
CheckInput: func(input *smokeCommandInput[I]) error {
@@ -228,13 +228,17 @@ func runSmokeTest[I, S any](invocation commandkit.Invocation, suite Suite[I, S],
return commandkit.Success()
}
func runSmokeAggregate[I, S any](invocation commandkit.Invocation, suite Suite[I, S], input *smokeCommandInput[I], state S) commandkit.Outcome {
kinds := [...]promptkit.SmokeTestKind{
func aggregateSmokeKinds() [4]promptkit.SmokeTestKind {
return [...]promptkit.SmokeTestKind{
promptkit.SmokeTestReachability,
promptkit.SmokeTestAuthentication,
promptkit.SmokeTestTextInference,
promptkit.SmokeTestStructuredOutput,
}
}
func runSmokeAggregate[I, S any](invocation commandkit.Invocation, suite Suite[I, S], input *smokeCommandInput[I], state S) commandkit.Outcome {
kinds := aggregateSmokeKinds()
results := make([]smokeResult, 0, len(kinds))
var operationErrors []error
var explanations []commandkit.Explanation