Classify PromptKit inspection failures

This commit is contained in:
2026-11-16 18:05:24 +00:00
parent 1c531e7109
commit 2bd216f0d3
7 changed files with 188 additions and 34 deletions

View File

@@ -202,7 +202,7 @@ func newCatalogListCommand[I, S any](suite Suite[I, S], name string) commandkit.
}
return runCatalogList(invocation, suite, input, name, list, collectionCatalogReport)
default:
return definitionFailure("Unable to list the selected PromptKit catalog.", commandkit.Explanation{}, fmt.Errorf("unknown PromptKit catalog %q", name))
return definitionFailure(invocation.Context, "Unable to list the selected PromptKit catalog.", commandkit.Explanation{}, fmt.Errorf("unknown PromptKit catalog %q", name))
}
},
})
@@ -368,7 +368,7 @@ func runCatalogList[T any, I, S any](
) commandkit.Outcome {
if invocation.Context != nil {
if err := invocation.Context.Err(); err != nil {
return definitionFailure("PromptKit catalog listing was canceled.", commandkit.Explanation{}, err)
return definitionFailure(invocation.Context, "PromptKit catalog listing was canceled.", commandkit.Explanation{}, err)
}
}
var state S
@@ -377,18 +377,18 @@ func runCatalogList[T any, I, S any](
}
if invocation.Context != nil {
if err := invocation.Context.Err(); err != nil {
return definitionFailure("PromptKit catalog listing was canceled.", commandkit.Explanation{}, err)
return definitionFailure(invocation.Context, "PromptKit catalog listing was canceled.", commandkit.Explanation{}, err)
}
}
acquired, err := list(invocation.Context, CatalogRequest{Selection: input.selection, Filter: input.filter}, input.selected, state)
queuePromptDiagnostics(invocation.Reporter(), acquired.Diagnostics)
if err != nil {
return definitionFailure(fmt.Sprintf("Unable to list PromptKit %s.", catalog), acquired.Explanation, err)
return definitionFailure(invocation.Context, fmt.Sprintf("Unable to list PromptKit %s.", catalog), acquired.Explanation, err)
}
if invocation.Context != nil {
if err := invocation.Context.Err(); err != nil {
return definitionFailure("PromptKit catalog listing was canceled.", acquired.Explanation, err)
return definitionFailure(invocation.Context, "PromptKit catalog listing was canceled.", acquired.Explanation, err)
}
}
_ = invocation.Reporter().Report(commandkit.Report{Value: project(acquired.Value)})
@@ -397,14 +397,14 @@ func runCatalogList[T any, I, S any](
func runDefinitionInspection[I, S any](invocation commandkit.Invocation, suite Suite[I, S], input *definitionInput[I], kind string) commandkit.Outcome {
if err := promptKitContextError(invocation.Context); err != nil {
return definitionFailure("PromptKit inspection was canceled.", commandkit.Explanation{}, err)
return definitionFailure(invocation.Context, "PromptKit inspection was canceled.", commandkit.Explanation{}, err)
}
var state S
if suite.NewState != nil {
state = suite.NewState()
}
if err := promptKitContextError(invocation.Context); err != nil {
return definitionFailure("PromptKit inspection was canceled.", commandkit.Explanation{}, err)
return definitionFailure(invocation.Context, "PromptKit inspection was canceled.", commandkit.Explanation{}, err)
}
var value any
var diagnostics []commandkit.Diagnostic
@@ -435,13 +435,13 @@ func runDefinitionInspection[I, S any](invocation commandkit.Invocation, suite S
}
queuePromptDiagnostics(invocation.Reporter(), diagnostics)
if err != nil {
return definitionFailure(fmt.Sprintf("Unable to inspect PromptKit %s %q.", kind, input.id), explanation, err)
return definitionFailure(invocation.Context, fmt.Sprintf("Unable to inspect PromptKit %s %q.", kind, input.id), explanation, err)
}
if value == nil {
return definitionFailure(fmt.Sprintf("PromptKit %s inspection returned no result.", kind), explanation, nil)
return definitionFailure(invocation.Context, fmt.Sprintf("PromptKit %s inspection returned no result.", kind), explanation, nil)
}
if err := promptKitContextError(invocation.Context); err != nil {
return definitionFailure(fmt.Sprintf("Inspection of PromptKit %s %q was canceled.", kind, input.id), explanation, err)
return definitionFailure(invocation.Context, fmt.Sprintf("Inspection of PromptKit %s %q was canceled.", kind, input.id), explanation, err)
}
_ = invocation.Reporter().Report(commandkit.Report{Value: value})
return commandkit.Success()
@@ -520,7 +520,7 @@ func requirementsReportFromInspection(inspection *promptkit.RequirementInspectio
func runRequirementsInspection[I, S any](invocation commandkit.Invocation, suite Suite[I, S], input *requirementsInput[I]) commandkit.Outcome {
if invocation.Context != nil {
if err := invocation.Context.Err(); err != nil {
return definitionFailure("PromptKit requirements inspection was canceled.", commandkit.Explanation{}, err)
return definitionFailure(invocation.Context, "PromptKit requirements inspection was canceled.", commandkit.Explanation{}, err)
}
}
var state S
@@ -529,7 +529,7 @@ func runRequirementsInspection[I, S any](invocation commandkit.Invocation, suite
}
if invocation.Context != nil {
if err := invocation.Context.Err(); err != nil {
return definitionFailure("PromptKit requirements inspection was canceled.", commandkit.Explanation{}, err)
return definitionFailure(invocation.Context, "PromptKit requirements inspection was canceled.", commandkit.Explanation{}, err)
}
}
@@ -537,14 +537,14 @@ func runRequirementsInspection[I, S any](invocation commandkit.Invocation, suite
acquired, err := suite.InspectRequirements(invocation.Context, request, input.selected, state)
queuePromptDiagnostics(invocation.Reporter(), acquired.Diagnostics)
if err != nil {
return definitionFailure(fmt.Sprintf("Unable to inspect requirements for PromptKit prompt %q.", input.inspection.PromptID), acquired.Explanation, err)
return definitionFailure(invocation.Context, fmt.Sprintf("Unable to inspect requirements for PromptKit prompt %q.", input.inspection.PromptID), acquired.Explanation, err)
}
if acquired.Value == nil {
return definitionFailure("PromptKit requirements inspection returned no result.", acquired.Explanation, nil)
return definitionFailure(invocation.Context, "PromptKit requirements inspection returned no result.", acquired.Explanation, nil)
}
if invocation.Context != nil {
if err := invocation.Context.Err(); err != nil {
return definitionFailure(fmt.Sprintf("Requirements inspection for PromptKit prompt %q was canceled.", input.inspection.PromptID), acquired.Explanation, err)
return definitionFailure(invocation.Context, fmt.Sprintf("Requirements inspection for PromptKit prompt %q was canceled.", input.inspection.PromptID), acquired.Explanation, err)
}
}
_ = invocation.Reporter().Report(commandkit.Report{Value: requirementsReportFromInspection(acquired.Value)})
@@ -602,8 +602,20 @@ func queuePromptDiagnostics(reporter *commandkit.Reporter, diagnostics []command
}
}
func definitionFailure(message string, explanation commandkit.Explanation, cause error) commandkit.Outcome {
func definitionFailure(ctx context.Context, message string, explanation commandkit.Explanation, cause error) commandkit.Outcome {
if ctx != nil {
if canceled := ctx.Err(); canceled != nil {
if cause == nil {
cause = canceled
} else if !errors.Is(cause, canceled) {
cause = errors.Join(cause, canceled)
}
}
}
outcome := commandkit.Failure(message)
if cause != nil && ckconfig.ClassifyConfigurationError(cause) == commandkit.OutcomeUsage {
outcome = commandkit.Usage(message)
}
if explanation.Message != "" || len(explanation.Details) > 0 {
outcome = outcome.WithExplanation(explanation.Message, explanation.Details...)
}

View File

@@ -0,0 +1,134 @@
package ckprompt
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"strings"
"testing"
"go.maximumdirect.net/ckconfig"
"go.maximumdirect.net/commandkit"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
func TestPromptConfigurationFailureCallbacks(t *testing.T) {
_, loadErr := configkit.Load[map[string]any](configkit.Bytes("prompt-config", []byte("broken: [")))
var native *configkit.LoadError
if !errors.As(loadErr, &native) {
t.Fatalf("fixture: %v", loadErr)
}
wrapped := fmt.Errorf("load: %w", loadErr)
diag := commandkit.Explanation{Message: "configuration diagnostic"}
suite := Suite[struct{}, struct{}]{
ListPrompts: func(context.Context, CatalogRequest, struct{}, struct{}) (commandkit.Acquired[[]promptkit.PromptIdentity], error) {
return commandkit.Acquired[[]promptkit.PromptIdentity]{Explanation: diag}, wrapped
},
ListProfiles: func(context.Context, CatalogRequest, struct{}, struct{}) (commandkit.Acquired[[]string], error) {
return commandkit.Acquired[[]string]{Explanation: diag}, wrapped
},
ListBackends: func(context.Context, CatalogRequest, struct{}, struct{}) (commandkit.Acquired[[]string], error) {
return commandkit.Acquired[[]string]{Explanation: diag}, wrapped
},
ListNamespaces: func(context.Context, NamespaceRequest, struct{}, struct{}) (commandkit.Acquired[[]string], error) {
return commandkit.Acquired[[]string]{Explanation: diag}, wrapped
},
ListCollections: func(context.Context, ckconfig.Selection, struct{}, struct{}) (commandkit.Acquired[[]promptkit.CollectionInfo], error) {
return commandkit.Acquired[[]promptkit.CollectionInfo]{Explanation: diag}, wrapped
},
InspectPrompt: func(context.Context, PromptRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.PromptInspection], error) {
return commandkit.Acquired[*promptkit.PromptInspection]{Explanation: diag}, wrapped
},
InspectProfile: func(context.Context, ProfileRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.ProfileInspection], error) {
return commandkit.Acquired[*promptkit.ProfileInspection]{Explanation: diag}, wrapped
},
InspectBackend: func(context.Context, BackendRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.BackendInspection], error) {
return commandkit.Acquired[*promptkit.BackendInspection]{Explanation: diag}, wrapped
},
InspectRequirements: func(context.Context, RequirementsRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.RequirementInspection], error) {
return commandkit.Acquired[*promptkit.RequirementInspection]{Explanation: diag}, wrapped
},
ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) {
return commandkit.Acquired[struct{}]{Explanation: diag}, wrapped
},
PrepareRequest: func(context.Context, PreparationRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.PreparedRun], error) {
return commandkit.Acquired[*promptkit.PreparedRun]{Explanation: diag}, wrapped
},
}
app := newPromptTestApp(t, Command(suite))
for _, args := range [][]string{{"promptkit", "list", "prompts"}, {"promptkit", "list", "profiles"}, {"promptkit", "list", "backends"}, {"promptkit", "list", "namespaces"}, {"promptkit", "list", "collections"}, {"promptkit", "show", "prompt", "example"}, {"promptkit", "show", "profile", "example"}, {"promptkit", "show", "backend", "example"}, {"promptkit", "requirements", "show", "example"}, {"promptkit", "requirements", "validate", "example"}, {"promptkit", "show", "prepared", "example"}} {
t.Run(strings.Join(args, "_"), func(t *testing.T) {
var out, diagnostic bytes.Buffer
result := app.Execute(context.Background(), args, commandkit.Streams{Out: &out, Err: &diagnostic})
var got *configkit.LoadError
if result.Kind != commandkit.OutcomeUsage || result.Status != 2 || !errors.Is(result.Cause, loadErr) || !errors.As(result.Cause, &got) || got != native || !strings.Contains(diagnostic.String(), "configuration diagnostic") {
t.Fatalf("outcome=%#v diagnostics=%q", result, diagnostic.String())
}
})
}
}
func TestPromptNativeConfigurationFailures(t *testing.T) {
_, loadErr := configkit.Load[map[string]any](configkit.Bytes("native-config", []byte("broken: [")))
load := func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) {
return nil, fmt.Errorf("wrapped: %w", loadErr)
}
suite := FromEngine[struct{}, struct{}](load)
suite = WithRequestValidation(suite, load, nil)
suite = WithRequestPreparation(suite, load, nil)
app := newPromptTestApp(t, Command(suite))
for _, args := range [][]string{{"promptkit", "list", "prompts"}, {"promptkit", "show", "prompt", "example"}, {"promptkit", "requirements", "show", "example"}, {"promptkit", "requirements", "validate", "example"}, {"promptkit", "show", "prepared", "example"}} {
t.Run(strings.Join(args, "_"), func(t *testing.T) {
var diag bytes.Buffer
result := app.Execute(context.Background(), args, commandkit.Streams{Out: io.Discard, Err: &diag})
var native *configkit.LoadError
if result.Status != 2 || !errors.Is(result.Cause, loadErr) || !errors.As(result.Cause, &native) || !strings.Contains(diag.String(), "native-config") {
t.Fatalf("outcome=%#v diagnostics=%q", result, diag.String())
}
})
}
engine := adapterEngine(t)
good := func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) {
return engine, nil
}
validation := WithRequestValidation(Suite[struct{}, struct{}]{}, good, func(context.Context, ValidationRequest, struct{}, struct{}) (promptkit.RunRequest, error) {
return promptkit.RunRequest{}, loadErr
})
preparation := WithRequestPreparation(validation, good, func(context.Context, PreparationRequest, struct{}, struct{}) (promptkit.RunRequest, error) {
return promptkit.RunRequest{}, loadErr
})
app = newPromptTestApp(t, Command(preparation))
for _, args := range [][]string{{"promptkit", "requirements", "validate", "example"}, {"promptkit", "show", "prepared", "example"}} {
result := app.Execute(context.Background(), args, commandkit.Streams{Out: io.Discard, Err: io.Discard})
if result.Status != 2 || !errors.Is(result.Cause, loadErr) {
t.Fatalf("builder %v: %#v", args, result)
}
}
}
func TestPromptConfigurationFailureCancellation(t *testing.T) {
_, loadErr := configkit.Load[map[string]any](configkit.Bytes("invalid", []byte("broken: [")))
ctx, cancel := context.WithCancel(context.Background())
calls := 0
suite := Suite[struct{}, struct{}]{InspectPrompt: func(context.Context, PromptRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.PromptInspection], error) {
calls++
cancel()
return commandkit.Acquired[*promptkit.PromptInspection]{}, loadErr
}}
app := newPromptTestApp(t, Command(suite))
var out bytes.Buffer
result := app.Execute(ctx, []string{"promptkit", "show", "prompt", "example"}, commandkit.Streams{Out: &out, Err: io.Discard})
if result.Status != 1 || !errors.Is(result.Cause, context.Canceled) || !errors.Is(result.Cause, loadErr) || out.Len() != 0 || calls != 1 {
t.Fatalf("outcome=%#v calls=%d output=%q", result, calls, out.String())
}
ctx, cancel = context.WithCancel(context.Background())
cancel()
calls = 0
result = app.Execute(ctx, []string{"promptkit", "show", "prompt", "example"}, commandkit.Streams{Out: &out, Err: io.Discard})
if result.Status != 1 || !errors.Is(result.Cause, context.Canceled) || calls != 0 || out.Len() != 0 {
t.Fatalf("pre-canceled=%#v calls=%d", result, calls)
}
}

View File

@@ -82,26 +82,26 @@ func newPreparationCommand[I, S any](suite Suite[I, S], aliases *[]commandkit.De
func runRequestPreparation[I, S any](invocation commandkit.Invocation, suite Suite[I, S], input *concreteRequestInput[I]) commandkit.Outcome {
if err := promptKitContextError(invocation.Context); err != nil {
return definitionFailure("PromptKit request preparation was canceled.", commandkit.Explanation{}, err)
return definitionFailure(invocation.Context, "PromptKit request preparation was canceled.", commandkit.Explanation{}, err)
}
var state S
if suite.NewState != nil {
state = suite.NewState()
}
if err := promptKitContextError(invocation.Context); err != nil {
return definitionFailure("PromptKit request preparation was canceled.", commandkit.Explanation{}, err)
return definitionFailure(invocation.Context, "PromptKit request preparation was canceled.", commandkit.Explanation{}, err)
}
request := PreparationRequest{Selection: input.request.Selection, Request: input.request.Request, Namespaces: input.request.Namespaces}
acquired, err := suite.PrepareRequest(invocation.Context, request, input.selected, state)
queuePromptDiagnostics(invocation.Reporter(), acquired.Diagnostics)
if err != nil {
return definitionFailure(fmt.Sprintf("Unable to prepare the PromptKit request for prompt %q.", input.promptID), acquired.Explanation, err)
return definitionFailure(invocation.Context, fmt.Sprintf("Unable to prepare the PromptKit request for prompt %q.", input.promptID), acquired.Explanation, err)
}
if err := promptKitContextError(invocation.Context); err != nil {
return definitionFailure(fmt.Sprintf("Preparation of the PromptKit request for prompt %q was canceled.", input.promptID), acquired.Explanation, err)
return definitionFailure(invocation.Context, fmt.Sprintf("Preparation of the PromptKit request for prompt %q was canceled.", input.promptID), acquired.Explanation, err)
}
if acquired.Value == nil {
return definitionFailure("PromptKit request preparation returned no result.", acquired.Explanation, nil)
return definitionFailure(invocation.Context, "PromptKit request preparation returned no result.", acquired.Explanation, nil)
}
_ = invocation.Reporter().Report(commandkit.Report{Value: acquired.Value})
return commandkit.Success()

View File

@@ -91,14 +91,14 @@ func newSmokeTestCommand[I, S any](suite Suite[I, S], aliases *[]commandkit.Decl
func runSmokeTest[I, S any](invocation commandkit.Invocation, suite Suite[I, S], input *smokeCommandInput[I]) commandkit.Outcome {
if err := promptKitContextError(invocation.Context); err != nil {
return definitionFailure("PromptKit smoke testing was canceled.", commandkit.Explanation{}, err)
return definitionFailure(invocation.Context, "PromptKit smoke testing was canceled.", commandkit.Explanation{}, err)
}
var state S
if suite.NewState != nil {
state = suite.NewState()
}
if err := promptKitContextError(invocation.Context); err != nil {
return definitionFailure("PromptKit smoke testing was canceled.", commandkit.Explanation{}, err)
return definitionFailure(invocation.Context, "PromptKit smoke testing was canceled.", commandkit.Explanation{}, err)
}
if input.kind == "all" {
return runSmokeAggregate(invocation, suite, input, state)
@@ -118,12 +118,12 @@ func runSmokeTest[I, S any](invocation commandkit.Invocation, suite Suite[I, S],
}
}
if acquired.Value == nil {
return definitionFailure(smokeFailureMessage(input.kind), acquired.Explanation, operationErr)
return definitionFailure(invocation.Context, smokeFailureMessage(input.kind), acquired.Explanation, operationErr)
}
_ = invocation.Reporter().Report(commandkit.Report{Value: smokeReportFromNative(acquired.Value)})
if operationErr != nil {
return definitionFailure(smokeFailureMessage(input.kind), acquired.Explanation, operationErr)
return definitionFailure(invocation.Context, smokeFailureMessage(input.kind), acquired.Explanation, operationErr)
}
return commandkit.Success()
}
@@ -146,7 +146,7 @@ func runSmokeAggregate[I, S any](invocation commandkit.Invocation, suite Suite[I
for _, kind := range kinds {
if err := promptKitContextError(invocation.Context); err != nil {
if len(results) == 0 {
return definitionFailure("PromptKit aggregate smoke testing was canceled.", commandkit.Explanation{}, err)
return definitionFailure(invocation.Context, "PromptKit aggregate smoke testing was canceled.", commandkit.Explanation{}, err)
}
operationErrors = append(operationErrors, err)
break
@@ -182,11 +182,11 @@ func runSmokeAggregate[I, S any](invocation commandkit.Invocation, suite Suite[I
}
if len(results) == 0 {
return definitionFailure("PromptKit aggregate smoke testing did not produce any results.", combineSmokeExplanations(explanations), errors.Join(operationErrors...))
return definitionFailure(invocation.Context, "PromptKit aggregate smoke testing did not produce any results.", combineSmokeExplanations(explanations), errors.Join(operationErrors...))
}
_ = invocation.Reporter().Report(commandkit.Report{Value: smokeResultsReport(results)})
if len(operationErrors) > 0 {
return definitionFailure("One or more PromptKit smoke tests failed.", combineSmokeExplanations(explanations), errors.Join(operationErrors...))
return definitionFailure(invocation.Context, "One or more PromptKit smoke tests failed.", combineSmokeExplanations(explanations), errors.Join(operationErrors...))
}
return commandkit.Success()
}

View File

@@ -102,23 +102,23 @@ func newValidationCommand[I, S any](suite Suite[I, S], aliases *[]commandkit.Dec
func runRequestValidation[I, S any](invocation commandkit.Invocation, suite Suite[I, S], input *concreteRequestInput[I]) commandkit.Outcome {
if err := promptKitContextError(invocation.Context); err != nil {
return definitionFailure("PromptKit request validation was canceled.", commandkit.Explanation{}, err)
return definitionFailure(invocation.Context, "PromptKit request validation was canceled.", commandkit.Explanation{}, err)
}
var state S
if suite.NewState != nil {
state = suite.NewState()
}
if err := promptKitContextError(invocation.Context); err != nil {
return definitionFailure("PromptKit request validation was canceled.", commandkit.Explanation{}, err)
return definitionFailure(invocation.Context, "PromptKit request validation was canceled.", commandkit.Explanation{}, err)
}
acquired, err := suite.ValidateRequest(invocation.Context, input.request, input.selected, state)
queuePromptDiagnostics(invocation.Reporter(), acquired.Diagnostics)
if err != nil {
return definitionFailure(fmt.Sprintf("Unable to validate the PromptKit request for prompt %q.", input.promptID), acquired.Explanation, err)
return definitionFailure(invocation.Context, fmt.Sprintf("Unable to validate the PromptKit request for prompt %q.", input.promptID), acquired.Explanation, err)
}
if err := promptKitContextError(invocation.Context); err != nil {
return definitionFailure(fmt.Sprintf("Validation of the PromptKit request for prompt %q was canceled.", input.promptID), acquired.Explanation, err)
return definitionFailure(invocation.Context, fmt.Sprintf("Validation of the PromptKit request for prompt %q was canceled.", input.promptID), acquired.Explanation, err)
}
_ = invocation.Reporter().Report(commandkit.Report{Value: requestValidationReport{PromptID: input.promptID, Valid: true}})
return commandkit.Success()

View File

@@ -409,7 +409,7 @@ func TestValidationCommandRetainsProviderDiagnosticsAndClassifiesFailures(t *tes
}, nil)))
outcome = loadApp.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
var retained *configkit.LoadError
if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || !errors.Is(outcome.Cause, loadErr) || !errors.As(outcome.Cause, &retained) || retained != loadErr || outcome.Explanation.Message == "" && len(outcome.Explanation.Details) == 0 {
if outcome.Kind != commandkit.OutcomeUsage || outcome.Status != 2 || !errors.Is(outcome.Cause, loadErr) || !errors.As(outcome.Cause, &retained) || retained != loadErr || outcome.Explanation.Message == "" && len(outcome.Explanation.Details) == 0 {
t.Fatalf("loader failure = %#v, want retained ConfigKit error and diagnostics", outcome)
}
}