Classify PromptKit inspection failures

This commit is contained in:
2026-11-16 18:05:24 +00:00
parent 1c531e7109
commit 2bd216f0d3
7 changed files with 188 additions and 34 deletions

View File

@@ -202,7 +202,15 @@ owns the final independent ckconfig suite.
## Stage 3 — Apply the shared function to PromptKit inspection and requests ## Stage 3 — Apply the shared function to PromptKit inspection and requests
Status — pending. Status — complete. PromptKit failure handling now samples invocation
cancellation and applies the shared classifier, including the smoke handler
call sites needed for compilation. Tests cover every catalog, named inspection,
requirements, validation, preparation, native loader and builder failures,
and cancellation with preserved causes. With Go 1.27.0, the aligned workspace
passed `GOTOOLCHAIN=local GOPROXY=off go test ./integrations/ckprompt -skip
'^TestPublicConfigDiffUsesPromptKitPairAndPreparedInputs$'`; that composition
test and the composed example retain expectations for Stage 4. A temporary Go
build cache was used.
Update `definitionFailure` in `integrations/ckprompt/command.go` to receive the Update `definitionFailure` in `integrations/ckprompt/command.go` to receive the
invocation context and combine observed cancellation with the operation cause invocation context and combine observed cancellation with the operation cause

View File

@@ -202,7 +202,7 @@ func newCatalogListCommand[I, S any](suite Suite[I, S], name string) commandkit.
} }
return runCatalogList(invocation, suite, input, name, list, collectionCatalogReport) return runCatalogList(invocation, suite, input, name, list, collectionCatalogReport)
default: default:
return definitionFailure("Unable to list the selected PromptKit catalog.", commandkit.Explanation{}, fmt.Errorf("unknown PromptKit catalog %q", name)) return definitionFailure(invocation.Context, "Unable to list the selected PromptKit catalog.", commandkit.Explanation{}, fmt.Errorf("unknown PromptKit catalog %q", name))
} }
}, },
}) })
@@ -368,7 +368,7 @@ func runCatalogList[T any, I, S any](
) commandkit.Outcome { ) commandkit.Outcome {
if invocation.Context != nil { if invocation.Context != nil {
if err := invocation.Context.Err(); err != nil { if err := invocation.Context.Err(); err != nil {
return definitionFailure("PromptKit catalog listing was canceled.", commandkit.Explanation{}, err) return definitionFailure(invocation.Context, "PromptKit catalog listing was canceled.", commandkit.Explanation{}, err)
} }
} }
var state S var state S
@@ -377,18 +377,18 @@ func runCatalogList[T any, I, S any](
} }
if invocation.Context != nil { if invocation.Context != nil {
if err := invocation.Context.Err(); err != nil { if err := invocation.Context.Err(); err != nil {
return definitionFailure("PromptKit catalog listing was canceled.", commandkit.Explanation{}, err) return definitionFailure(invocation.Context, "PromptKit catalog listing was canceled.", commandkit.Explanation{}, err)
} }
} }
acquired, err := list(invocation.Context, CatalogRequest{Selection: input.selection, Filter: input.filter}, input.selected, state) acquired, err := list(invocation.Context, CatalogRequest{Selection: input.selection, Filter: input.filter}, input.selected, state)
queuePromptDiagnostics(invocation.Reporter(), acquired.Diagnostics) queuePromptDiagnostics(invocation.Reporter(), acquired.Diagnostics)
if err != nil { if err != nil {
return definitionFailure(fmt.Sprintf("Unable to list PromptKit %s.", catalog), acquired.Explanation, err) return definitionFailure(invocation.Context, fmt.Sprintf("Unable to list PromptKit %s.", catalog), acquired.Explanation, err)
} }
if invocation.Context != nil { if invocation.Context != nil {
if err := invocation.Context.Err(); err != nil { if err := invocation.Context.Err(); err != nil {
return definitionFailure("PromptKit catalog listing was canceled.", acquired.Explanation, err) return definitionFailure(invocation.Context, "PromptKit catalog listing was canceled.", acquired.Explanation, err)
} }
} }
_ = invocation.Reporter().Report(commandkit.Report{Value: project(acquired.Value)}) _ = invocation.Reporter().Report(commandkit.Report{Value: project(acquired.Value)})
@@ -397,14 +397,14 @@ func runCatalogList[T any, I, S any](
func runDefinitionInspection[I, S any](invocation commandkit.Invocation, suite Suite[I, S], input *definitionInput[I], kind string) commandkit.Outcome { func runDefinitionInspection[I, S any](invocation commandkit.Invocation, suite Suite[I, S], input *definitionInput[I], kind string) commandkit.Outcome {
if err := promptKitContextError(invocation.Context); err != nil { if err := promptKitContextError(invocation.Context); err != nil {
return definitionFailure("PromptKit inspection was canceled.", commandkit.Explanation{}, err) return definitionFailure(invocation.Context, "PromptKit inspection was canceled.", commandkit.Explanation{}, err)
} }
var state S var state S
if suite.NewState != nil { if suite.NewState != nil {
state = suite.NewState() state = suite.NewState()
} }
if err := promptKitContextError(invocation.Context); err != nil { if err := promptKitContextError(invocation.Context); err != nil {
return definitionFailure("PromptKit inspection was canceled.", commandkit.Explanation{}, err) return definitionFailure(invocation.Context, "PromptKit inspection was canceled.", commandkit.Explanation{}, err)
} }
var value any var value any
var diagnostics []commandkit.Diagnostic var diagnostics []commandkit.Diagnostic
@@ -435,13 +435,13 @@ func runDefinitionInspection[I, S any](invocation commandkit.Invocation, suite S
} }
queuePromptDiagnostics(invocation.Reporter(), diagnostics) queuePromptDiagnostics(invocation.Reporter(), diagnostics)
if err != nil { if err != nil {
return definitionFailure(fmt.Sprintf("Unable to inspect PromptKit %s %q.", kind, input.id), explanation, err) return definitionFailure(invocation.Context, fmt.Sprintf("Unable to inspect PromptKit %s %q.", kind, input.id), explanation, err)
} }
if value == nil { if value == nil {
return definitionFailure(fmt.Sprintf("PromptKit %s inspection returned no result.", kind), explanation, nil) return definitionFailure(invocation.Context, fmt.Sprintf("PromptKit %s inspection returned no result.", kind), explanation, nil)
} }
if err := promptKitContextError(invocation.Context); err != nil { if err := promptKitContextError(invocation.Context); err != nil {
return definitionFailure(fmt.Sprintf("Inspection of PromptKit %s %q was canceled.", kind, input.id), explanation, err) return definitionFailure(invocation.Context, fmt.Sprintf("Inspection of PromptKit %s %q was canceled.", kind, input.id), explanation, err)
} }
_ = invocation.Reporter().Report(commandkit.Report{Value: value}) _ = invocation.Reporter().Report(commandkit.Report{Value: value})
return commandkit.Success() return commandkit.Success()
@@ -520,7 +520,7 @@ func requirementsReportFromInspection(inspection *promptkit.RequirementInspectio
func runRequirementsInspection[I, S any](invocation commandkit.Invocation, suite Suite[I, S], input *requirementsInput[I]) commandkit.Outcome { func runRequirementsInspection[I, S any](invocation commandkit.Invocation, suite Suite[I, S], input *requirementsInput[I]) commandkit.Outcome {
if invocation.Context != nil { if invocation.Context != nil {
if err := invocation.Context.Err(); err != nil { if err := invocation.Context.Err(); err != nil {
return definitionFailure("PromptKit requirements inspection was canceled.", commandkit.Explanation{}, err) return definitionFailure(invocation.Context, "PromptKit requirements inspection was canceled.", commandkit.Explanation{}, err)
} }
} }
var state S var state S
@@ -529,7 +529,7 @@ func runRequirementsInspection[I, S any](invocation commandkit.Invocation, suite
} }
if invocation.Context != nil { if invocation.Context != nil {
if err := invocation.Context.Err(); err != nil { if err := invocation.Context.Err(); err != nil {
return definitionFailure("PromptKit requirements inspection was canceled.", commandkit.Explanation{}, err) return definitionFailure(invocation.Context, "PromptKit requirements inspection was canceled.", commandkit.Explanation{}, err)
} }
} }
@@ -537,14 +537,14 @@ func runRequirementsInspection[I, S any](invocation commandkit.Invocation, suite
acquired, err := suite.InspectRequirements(invocation.Context, request, input.selected, state) acquired, err := suite.InspectRequirements(invocation.Context, request, input.selected, state)
queuePromptDiagnostics(invocation.Reporter(), acquired.Diagnostics) queuePromptDiagnostics(invocation.Reporter(), acquired.Diagnostics)
if err != nil { if err != nil {
return definitionFailure(fmt.Sprintf("Unable to inspect requirements for PromptKit prompt %q.", input.inspection.PromptID), acquired.Explanation, err) return definitionFailure(invocation.Context, fmt.Sprintf("Unable to inspect requirements for PromptKit prompt %q.", input.inspection.PromptID), acquired.Explanation, err)
} }
if acquired.Value == nil { if acquired.Value == nil {
return definitionFailure("PromptKit requirements inspection returned no result.", acquired.Explanation, nil) return definitionFailure(invocation.Context, "PromptKit requirements inspection returned no result.", acquired.Explanation, nil)
} }
if invocation.Context != nil { if invocation.Context != nil {
if err := invocation.Context.Err(); err != nil { if err := invocation.Context.Err(); err != nil {
return definitionFailure(fmt.Sprintf("Requirements inspection for PromptKit prompt %q was canceled.", input.inspection.PromptID), acquired.Explanation, err) return definitionFailure(invocation.Context, fmt.Sprintf("Requirements inspection for PromptKit prompt %q was canceled.", input.inspection.PromptID), acquired.Explanation, err)
} }
} }
_ = invocation.Reporter().Report(commandkit.Report{Value: requirementsReportFromInspection(acquired.Value)}) _ = invocation.Reporter().Report(commandkit.Report{Value: requirementsReportFromInspection(acquired.Value)})
@@ -602,8 +602,20 @@ func queuePromptDiagnostics(reporter *commandkit.Reporter, diagnostics []command
} }
} }
func definitionFailure(message string, explanation commandkit.Explanation, cause error) commandkit.Outcome { func definitionFailure(ctx context.Context, message string, explanation commandkit.Explanation, cause error) commandkit.Outcome {
if ctx != nil {
if canceled := ctx.Err(); canceled != nil {
if cause == nil {
cause = canceled
} else if !errors.Is(cause, canceled) {
cause = errors.Join(cause, canceled)
}
}
}
outcome := commandkit.Failure(message) outcome := commandkit.Failure(message)
if cause != nil && ckconfig.ClassifyConfigurationError(cause) == commandkit.OutcomeUsage {
outcome = commandkit.Usage(message)
}
if explanation.Message != "" || len(explanation.Details) > 0 { if explanation.Message != "" || len(explanation.Details) > 0 {
outcome = outcome.WithExplanation(explanation.Message, explanation.Details...) outcome = outcome.WithExplanation(explanation.Message, explanation.Details...)
} }

View File

@@ -0,0 +1,134 @@
package ckprompt
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"strings"
"testing"
"go.maximumdirect.net/ckconfig"
"go.maximumdirect.net/commandkit"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
func TestPromptConfigurationFailureCallbacks(t *testing.T) {
_, loadErr := configkit.Load[map[string]any](configkit.Bytes("prompt-config", []byte("broken: [")))
var native *configkit.LoadError
if !errors.As(loadErr, &native) {
t.Fatalf("fixture: %v", loadErr)
}
wrapped := fmt.Errorf("load: %w", loadErr)
diag := commandkit.Explanation{Message: "configuration diagnostic"}
suite := Suite[struct{}, struct{}]{
ListPrompts: func(context.Context, CatalogRequest, struct{}, struct{}) (commandkit.Acquired[[]promptkit.PromptIdentity], error) {
return commandkit.Acquired[[]promptkit.PromptIdentity]{Explanation: diag}, wrapped
},
ListProfiles: func(context.Context, CatalogRequest, struct{}, struct{}) (commandkit.Acquired[[]string], error) {
return commandkit.Acquired[[]string]{Explanation: diag}, wrapped
},
ListBackends: func(context.Context, CatalogRequest, struct{}, struct{}) (commandkit.Acquired[[]string], error) {
return commandkit.Acquired[[]string]{Explanation: diag}, wrapped
},
ListNamespaces: func(context.Context, NamespaceRequest, struct{}, struct{}) (commandkit.Acquired[[]string], error) {
return commandkit.Acquired[[]string]{Explanation: diag}, wrapped
},
ListCollections: func(context.Context, ckconfig.Selection, struct{}, struct{}) (commandkit.Acquired[[]promptkit.CollectionInfo], error) {
return commandkit.Acquired[[]promptkit.CollectionInfo]{Explanation: diag}, wrapped
},
InspectPrompt: func(context.Context, PromptRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.PromptInspection], error) {
return commandkit.Acquired[*promptkit.PromptInspection]{Explanation: diag}, wrapped
},
InspectProfile: func(context.Context, ProfileRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.ProfileInspection], error) {
return commandkit.Acquired[*promptkit.ProfileInspection]{Explanation: diag}, wrapped
},
InspectBackend: func(context.Context, BackendRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.BackendInspection], error) {
return commandkit.Acquired[*promptkit.BackendInspection]{Explanation: diag}, wrapped
},
InspectRequirements: func(context.Context, RequirementsRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.RequirementInspection], error) {
return commandkit.Acquired[*promptkit.RequirementInspection]{Explanation: diag}, wrapped
},
ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) {
return commandkit.Acquired[struct{}]{Explanation: diag}, wrapped
},
PrepareRequest: func(context.Context, PreparationRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.PreparedRun], error) {
return commandkit.Acquired[*promptkit.PreparedRun]{Explanation: diag}, wrapped
},
}
app := newPromptTestApp(t, Command(suite))
for _, args := range [][]string{{"promptkit", "list", "prompts"}, {"promptkit", "list", "profiles"}, {"promptkit", "list", "backends"}, {"promptkit", "list", "namespaces"}, {"promptkit", "list", "collections"}, {"promptkit", "show", "prompt", "example"}, {"promptkit", "show", "profile", "example"}, {"promptkit", "show", "backend", "example"}, {"promptkit", "requirements", "show", "example"}, {"promptkit", "requirements", "validate", "example"}, {"promptkit", "show", "prepared", "example"}} {
t.Run(strings.Join(args, "_"), func(t *testing.T) {
var out, diagnostic bytes.Buffer
result := app.Execute(context.Background(), args, commandkit.Streams{Out: &out, Err: &diagnostic})
var got *configkit.LoadError
if result.Kind != commandkit.OutcomeUsage || result.Status != 2 || !errors.Is(result.Cause, loadErr) || !errors.As(result.Cause, &got) || got != native || !strings.Contains(diagnostic.String(), "configuration diagnostic") {
t.Fatalf("outcome=%#v diagnostics=%q", result, diagnostic.String())
}
})
}
}
func TestPromptNativeConfigurationFailures(t *testing.T) {
_, loadErr := configkit.Load[map[string]any](configkit.Bytes("native-config", []byte("broken: [")))
load := func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) {
return nil, fmt.Errorf("wrapped: %w", loadErr)
}
suite := FromEngine[struct{}, struct{}](load)
suite = WithRequestValidation(suite, load, nil)
suite = WithRequestPreparation(suite, load, nil)
app := newPromptTestApp(t, Command(suite))
for _, args := range [][]string{{"promptkit", "list", "prompts"}, {"promptkit", "show", "prompt", "example"}, {"promptkit", "requirements", "show", "example"}, {"promptkit", "requirements", "validate", "example"}, {"promptkit", "show", "prepared", "example"}} {
t.Run(strings.Join(args, "_"), func(t *testing.T) {
var diag bytes.Buffer
result := app.Execute(context.Background(), args, commandkit.Streams{Out: io.Discard, Err: &diag})
var native *configkit.LoadError
if result.Status != 2 || !errors.Is(result.Cause, loadErr) || !errors.As(result.Cause, &native) || !strings.Contains(diag.String(), "native-config") {
t.Fatalf("outcome=%#v diagnostics=%q", result, diag.String())
}
})
}
engine := adapterEngine(t)
good := func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) {
return engine, nil
}
validation := WithRequestValidation(Suite[struct{}, struct{}]{}, good, func(context.Context, ValidationRequest, struct{}, struct{}) (promptkit.RunRequest, error) {
return promptkit.RunRequest{}, loadErr
})
preparation := WithRequestPreparation(validation, good, func(context.Context, PreparationRequest, struct{}, struct{}) (promptkit.RunRequest, error) {
return promptkit.RunRequest{}, loadErr
})
app = newPromptTestApp(t, Command(preparation))
for _, args := range [][]string{{"promptkit", "requirements", "validate", "example"}, {"promptkit", "show", "prepared", "example"}} {
result := app.Execute(context.Background(), args, commandkit.Streams{Out: io.Discard, Err: io.Discard})
if result.Status != 2 || !errors.Is(result.Cause, loadErr) {
t.Fatalf("builder %v: %#v", args, result)
}
}
}
func TestPromptConfigurationFailureCancellation(t *testing.T) {
_, loadErr := configkit.Load[map[string]any](configkit.Bytes("invalid", []byte("broken: [")))
ctx, cancel := context.WithCancel(context.Background())
calls := 0
suite := Suite[struct{}, struct{}]{InspectPrompt: func(context.Context, PromptRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.PromptInspection], error) {
calls++
cancel()
return commandkit.Acquired[*promptkit.PromptInspection]{}, loadErr
}}
app := newPromptTestApp(t, Command(suite))
var out bytes.Buffer
result := app.Execute(ctx, []string{"promptkit", "show", "prompt", "example"}, commandkit.Streams{Out: &out, Err: io.Discard})
if result.Status != 1 || !errors.Is(result.Cause, context.Canceled) || !errors.Is(result.Cause, loadErr) || out.Len() != 0 || calls != 1 {
t.Fatalf("outcome=%#v calls=%d output=%q", result, calls, out.String())
}
ctx, cancel = context.WithCancel(context.Background())
cancel()
calls = 0
result = app.Execute(ctx, []string{"promptkit", "show", "prompt", "example"}, commandkit.Streams{Out: &out, Err: io.Discard})
if result.Status != 1 || !errors.Is(result.Cause, context.Canceled) || calls != 0 || out.Len() != 0 {
t.Fatalf("pre-canceled=%#v calls=%d", result, calls)
}
}

View File

@@ -82,26 +82,26 @@ func newPreparationCommand[I, S any](suite Suite[I, S], aliases *[]commandkit.De
func runRequestPreparation[I, S any](invocation commandkit.Invocation, suite Suite[I, S], input *concreteRequestInput[I]) commandkit.Outcome { func runRequestPreparation[I, S any](invocation commandkit.Invocation, suite Suite[I, S], input *concreteRequestInput[I]) commandkit.Outcome {
if err := promptKitContextError(invocation.Context); err != nil { if err := promptKitContextError(invocation.Context); err != nil {
return definitionFailure("PromptKit request preparation was canceled.", commandkit.Explanation{}, err) return definitionFailure(invocation.Context, "PromptKit request preparation was canceled.", commandkit.Explanation{}, err)
} }
var state S var state S
if suite.NewState != nil { if suite.NewState != nil {
state = suite.NewState() state = suite.NewState()
} }
if err := promptKitContextError(invocation.Context); err != nil { if err := promptKitContextError(invocation.Context); err != nil {
return definitionFailure("PromptKit request preparation was canceled.", commandkit.Explanation{}, err) return definitionFailure(invocation.Context, "PromptKit request preparation was canceled.", commandkit.Explanation{}, err)
} }
request := PreparationRequest{Selection: input.request.Selection, Request: input.request.Request, Namespaces: input.request.Namespaces} request := PreparationRequest{Selection: input.request.Selection, Request: input.request.Request, Namespaces: input.request.Namespaces}
acquired, err := suite.PrepareRequest(invocation.Context, request, input.selected, state) acquired, err := suite.PrepareRequest(invocation.Context, request, input.selected, state)
queuePromptDiagnostics(invocation.Reporter(), acquired.Diagnostics) queuePromptDiagnostics(invocation.Reporter(), acquired.Diagnostics)
if err != nil { if err != nil {
return definitionFailure(fmt.Sprintf("Unable to prepare the PromptKit request for prompt %q.", input.promptID), acquired.Explanation, err) return definitionFailure(invocation.Context, fmt.Sprintf("Unable to prepare the PromptKit request for prompt %q.", input.promptID), acquired.Explanation, err)
} }
if err := promptKitContextError(invocation.Context); err != nil { if err := promptKitContextError(invocation.Context); err != nil {
return definitionFailure(fmt.Sprintf("Preparation of the PromptKit request for prompt %q was canceled.", input.promptID), acquired.Explanation, err) return definitionFailure(invocation.Context, fmt.Sprintf("Preparation of the PromptKit request for prompt %q was canceled.", input.promptID), acquired.Explanation, err)
} }
if acquired.Value == nil { if acquired.Value == nil {
return definitionFailure("PromptKit request preparation returned no result.", acquired.Explanation, nil) return definitionFailure(invocation.Context, "PromptKit request preparation returned no result.", acquired.Explanation, nil)
} }
_ = invocation.Reporter().Report(commandkit.Report{Value: acquired.Value}) _ = invocation.Reporter().Report(commandkit.Report{Value: acquired.Value})
return commandkit.Success() return commandkit.Success()

View File

@@ -91,14 +91,14 @@ func newSmokeTestCommand[I, S any](suite Suite[I, S], aliases *[]commandkit.Decl
func runSmokeTest[I, S any](invocation commandkit.Invocation, suite Suite[I, S], input *smokeCommandInput[I]) commandkit.Outcome { func runSmokeTest[I, S any](invocation commandkit.Invocation, suite Suite[I, S], input *smokeCommandInput[I]) commandkit.Outcome {
if err := promptKitContextError(invocation.Context); err != nil { if err := promptKitContextError(invocation.Context); err != nil {
return definitionFailure("PromptKit smoke testing was canceled.", commandkit.Explanation{}, err) return definitionFailure(invocation.Context, "PromptKit smoke testing was canceled.", commandkit.Explanation{}, err)
} }
var state S var state S
if suite.NewState != nil { if suite.NewState != nil {
state = suite.NewState() state = suite.NewState()
} }
if err := promptKitContextError(invocation.Context); err != nil { if err := promptKitContextError(invocation.Context); err != nil {
return definitionFailure("PromptKit smoke testing was canceled.", commandkit.Explanation{}, err) return definitionFailure(invocation.Context, "PromptKit smoke testing was canceled.", commandkit.Explanation{}, err)
} }
if input.kind == "all" { if input.kind == "all" {
return runSmokeAggregate(invocation, suite, input, state) return runSmokeAggregate(invocation, suite, input, state)
@@ -118,12 +118,12 @@ func runSmokeTest[I, S any](invocation commandkit.Invocation, suite Suite[I, S],
} }
} }
if acquired.Value == nil { if acquired.Value == nil {
return definitionFailure(smokeFailureMessage(input.kind), acquired.Explanation, operationErr) return definitionFailure(invocation.Context, smokeFailureMessage(input.kind), acquired.Explanation, operationErr)
} }
_ = invocation.Reporter().Report(commandkit.Report{Value: smokeReportFromNative(acquired.Value)}) _ = invocation.Reporter().Report(commandkit.Report{Value: smokeReportFromNative(acquired.Value)})
if operationErr != nil { if operationErr != nil {
return definitionFailure(smokeFailureMessage(input.kind), acquired.Explanation, operationErr) return definitionFailure(invocation.Context, smokeFailureMessage(input.kind), acquired.Explanation, operationErr)
} }
return commandkit.Success() return commandkit.Success()
} }
@@ -146,7 +146,7 @@ func runSmokeAggregate[I, S any](invocation commandkit.Invocation, suite Suite[I
for _, kind := range kinds { for _, kind := range kinds {
if err := promptKitContextError(invocation.Context); err != nil { if err := promptKitContextError(invocation.Context); err != nil {
if len(results) == 0 { if len(results) == 0 {
return definitionFailure("PromptKit aggregate smoke testing was canceled.", commandkit.Explanation{}, err) return definitionFailure(invocation.Context, "PromptKit aggregate smoke testing was canceled.", commandkit.Explanation{}, err)
} }
operationErrors = append(operationErrors, err) operationErrors = append(operationErrors, err)
break break
@@ -182,11 +182,11 @@ func runSmokeAggregate[I, S any](invocation commandkit.Invocation, suite Suite[I
} }
if len(results) == 0 { if len(results) == 0 {
return definitionFailure("PromptKit aggregate smoke testing did not produce any results.", combineSmokeExplanations(explanations), errors.Join(operationErrors...)) return definitionFailure(invocation.Context, "PromptKit aggregate smoke testing did not produce any results.", combineSmokeExplanations(explanations), errors.Join(operationErrors...))
} }
_ = invocation.Reporter().Report(commandkit.Report{Value: smokeResultsReport(results)}) _ = invocation.Reporter().Report(commandkit.Report{Value: smokeResultsReport(results)})
if len(operationErrors) > 0 { if len(operationErrors) > 0 {
return definitionFailure("One or more PromptKit smoke tests failed.", combineSmokeExplanations(explanations), errors.Join(operationErrors...)) return definitionFailure(invocation.Context, "One or more PromptKit smoke tests failed.", combineSmokeExplanations(explanations), errors.Join(operationErrors...))
} }
return commandkit.Success() return commandkit.Success()
} }

View File

@@ -102,23 +102,23 @@ func newValidationCommand[I, S any](suite Suite[I, S], aliases *[]commandkit.Dec
func runRequestValidation[I, S any](invocation commandkit.Invocation, suite Suite[I, S], input *concreteRequestInput[I]) commandkit.Outcome { func runRequestValidation[I, S any](invocation commandkit.Invocation, suite Suite[I, S], input *concreteRequestInput[I]) commandkit.Outcome {
if err := promptKitContextError(invocation.Context); err != nil { if err := promptKitContextError(invocation.Context); err != nil {
return definitionFailure("PromptKit request validation was canceled.", commandkit.Explanation{}, err) return definitionFailure(invocation.Context, "PromptKit request validation was canceled.", commandkit.Explanation{}, err)
} }
var state S var state S
if suite.NewState != nil { if suite.NewState != nil {
state = suite.NewState() state = suite.NewState()
} }
if err := promptKitContextError(invocation.Context); err != nil { if err := promptKitContextError(invocation.Context); err != nil {
return definitionFailure("PromptKit request validation was canceled.", commandkit.Explanation{}, err) return definitionFailure(invocation.Context, "PromptKit request validation was canceled.", commandkit.Explanation{}, err)
} }
acquired, err := suite.ValidateRequest(invocation.Context, input.request, input.selected, state) acquired, err := suite.ValidateRequest(invocation.Context, input.request, input.selected, state)
queuePromptDiagnostics(invocation.Reporter(), acquired.Diagnostics) queuePromptDiagnostics(invocation.Reporter(), acquired.Diagnostics)
if err != nil { if err != nil {
return definitionFailure(fmt.Sprintf("Unable to validate the PromptKit request for prompt %q.", input.promptID), acquired.Explanation, err) return definitionFailure(invocation.Context, fmt.Sprintf("Unable to validate the PromptKit request for prompt %q.", input.promptID), acquired.Explanation, err)
} }
if err := promptKitContextError(invocation.Context); err != nil { if err := promptKitContextError(invocation.Context); err != nil {
return definitionFailure(fmt.Sprintf("Validation of the PromptKit request for prompt %q was canceled.", input.promptID), acquired.Explanation, err) return definitionFailure(invocation.Context, fmt.Sprintf("Validation of the PromptKit request for prompt %q was canceled.", input.promptID), acquired.Explanation, err)
} }
_ = invocation.Reporter().Report(commandkit.Report{Value: requestValidationReport{PromptID: input.promptID, Valid: true}}) _ = invocation.Reporter().Report(commandkit.Report{Value: requestValidationReport{PromptID: input.promptID, Valid: true}})
return commandkit.Success() return commandkit.Success()

View File

@@ -409,7 +409,7 @@ func TestValidationCommandRetainsProviderDiagnosticsAndClassifiesFailures(t *tes
}, nil))) }, nil)))
outcome = loadApp.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: io.Discard}) outcome = loadApp.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
var retained *configkit.LoadError var retained *configkit.LoadError
if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || !errors.Is(outcome.Cause, loadErr) || !errors.As(outcome.Cause, &retained) || retained != loadErr || outcome.Explanation.Message == "" && len(outcome.Explanation.Details) == 0 { if outcome.Kind != commandkit.OutcomeUsage || outcome.Status != 2 || !errors.Is(outcome.Cause, loadErr) || !errors.As(outcome.Cause, &retained) || retained != loadErr || outcome.Explanation.Message == "" && len(outcome.Explanation.Details) == 0 {
t.Fatalf("loader failure = %#v, want retained ConfigKit error and diagnostics", outcome) t.Fatalf("loader failure = %#v, want retained ConfigKit error and diagnostics", outcome)
} }
} }