Classify PromptKit inspection failures

This commit is contained in:
2026-11-16 18:05:24 +00:00
parent 1c531e7109
commit 2bd216f0d3
7 changed files with 188 additions and 34 deletions

View File

@@ -0,0 +1,134 @@
package ckprompt
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"strings"
"testing"
"go.maximumdirect.net/ckconfig"
"go.maximumdirect.net/commandkit"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
func TestPromptConfigurationFailureCallbacks(t *testing.T) {
_, loadErr := configkit.Load[map[string]any](configkit.Bytes("prompt-config", []byte("broken: [")))
var native *configkit.LoadError
if !errors.As(loadErr, &native) {
t.Fatalf("fixture: %v", loadErr)
}
wrapped := fmt.Errorf("load: %w", loadErr)
diag := commandkit.Explanation{Message: "configuration diagnostic"}
suite := Suite[struct{}, struct{}]{
ListPrompts: func(context.Context, CatalogRequest, struct{}, struct{}) (commandkit.Acquired[[]promptkit.PromptIdentity], error) {
return commandkit.Acquired[[]promptkit.PromptIdentity]{Explanation: diag}, wrapped
},
ListProfiles: func(context.Context, CatalogRequest, struct{}, struct{}) (commandkit.Acquired[[]string], error) {
return commandkit.Acquired[[]string]{Explanation: diag}, wrapped
},
ListBackends: func(context.Context, CatalogRequest, struct{}, struct{}) (commandkit.Acquired[[]string], error) {
return commandkit.Acquired[[]string]{Explanation: diag}, wrapped
},
ListNamespaces: func(context.Context, NamespaceRequest, struct{}, struct{}) (commandkit.Acquired[[]string], error) {
return commandkit.Acquired[[]string]{Explanation: diag}, wrapped
},
ListCollections: func(context.Context, ckconfig.Selection, struct{}, struct{}) (commandkit.Acquired[[]promptkit.CollectionInfo], error) {
return commandkit.Acquired[[]promptkit.CollectionInfo]{Explanation: diag}, wrapped
},
InspectPrompt: func(context.Context, PromptRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.PromptInspection], error) {
return commandkit.Acquired[*promptkit.PromptInspection]{Explanation: diag}, wrapped
},
InspectProfile: func(context.Context, ProfileRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.ProfileInspection], error) {
return commandkit.Acquired[*promptkit.ProfileInspection]{Explanation: diag}, wrapped
},
InspectBackend: func(context.Context, BackendRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.BackendInspection], error) {
return commandkit.Acquired[*promptkit.BackendInspection]{Explanation: diag}, wrapped
},
InspectRequirements: func(context.Context, RequirementsRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.RequirementInspection], error) {
return commandkit.Acquired[*promptkit.RequirementInspection]{Explanation: diag}, wrapped
},
ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) {
return commandkit.Acquired[struct{}]{Explanation: diag}, wrapped
},
PrepareRequest: func(context.Context, PreparationRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.PreparedRun], error) {
return commandkit.Acquired[*promptkit.PreparedRun]{Explanation: diag}, wrapped
},
}
app := newPromptTestApp(t, Command(suite))
for _, args := range [][]string{{"promptkit", "list", "prompts"}, {"promptkit", "list", "profiles"}, {"promptkit", "list", "backends"}, {"promptkit", "list", "namespaces"}, {"promptkit", "list", "collections"}, {"promptkit", "show", "prompt", "example"}, {"promptkit", "show", "profile", "example"}, {"promptkit", "show", "backend", "example"}, {"promptkit", "requirements", "show", "example"}, {"promptkit", "requirements", "validate", "example"}, {"promptkit", "show", "prepared", "example"}} {
t.Run(strings.Join(args, "_"), func(t *testing.T) {
var out, diagnostic bytes.Buffer
result := app.Execute(context.Background(), args, commandkit.Streams{Out: &out, Err: &diagnostic})
var got *configkit.LoadError
if result.Kind != commandkit.OutcomeUsage || result.Status != 2 || !errors.Is(result.Cause, loadErr) || !errors.As(result.Cause, &got) || got != native || !strings.Contains(diagnostic.String(), "configuration diagnostic") {
t.Fatalf("outcome=%#v diagnostics=%q", result, diagnostic.String())
}
})
}
}
func TestPromptNativeConfigurationFailures(t *testing.T) {
_, loadErr := configkit.Load[map[string]any](configkit.Bytes("native-config", []byte("broken: [")))
load := func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) {
return nil, fmt.Errorf("wrapped: %w", loadErr)
}
suite := FromEngine[struct{}, struct{}](load)
suite = WithRequestValidation(suite, load, nil)
suite = WithRequestPreparation(suite, load, nil)
app := newPromptTestApp(t, Command(suite))
for _, args := range [][]string{{"promptkit", "list", "prompts"}, {"promptkit", "show", "prompt", "example"}, {"promptkit", "requirements", "show", "example"}, {"promptkit", "requirements", "validate", "example"}, {"promptkit", "show", "prepared", "example"}} {
t.Run(strings.Join(args, "_"), func(t *testing.T) {
var diag bytes.Buffer
result := app.Execute(context.Background(), args, commandkit.Streams{Out: io.Discard, Err: &diag})
var native *configkit.LoadError
if result.Status != 2 || !errors.Is(result.Cause, loadErr) || !errors.As(result.Cause, &native) || !strings.Contains(diag.String(), "native-config") {
t.Fatalf("outcome=%#v diagnostics=%q", result, diag.String())
}
})
}
engine := adapterEngine(t)
good := func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) {
return engine, nil
}
validation := WithRequestValidation(Suite[struct{}, struct{}]{}, good, func(context.Context, ValidationRequest, struct{}, struct{}) (promptkit.RunRequest, error) {
return promptkit.RunRequest{}, loadErr
})
preparation := WithRequestPreparation(validation, good, func(context.Context, PreparationRequest, struct{}, struct{}) (promptkit.RunRequest, error) {
return promptkit.RunRequest{}, loadErr
})
app = newPromptTestApp(t, Command(preparation))
for _, args := range [][]string{{"promptkit", "requirements", "validate", "example"}, {"promptkit", "show", "prepared", "example"}} {
result := app.Execute(context.Background(), args, commandkit.Streams{Out: io.Discard, Err: io.Discard})
if result.Status != 2 || !errors.Is(result.Cause, loadErr) {
t.Fatalf("builder %v: %#v", args, result)
}
}
}
func TestPromptConfigurationFailureCancellation(t *testing.T) {
_, loadErr := configkit.Load[map[string]any](configkit.Bytes("invalid", []byte("broken: [")))
ctx, cancel := context.WithCancel(context.Background())
calls := 0
suite := Suite[struct{}, struct{}]{InspectPrompt: func(context.Context, PromptRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.PromptInspection], error) {
calls++
cancel()
return commandkit.Acquired[*promptkit.PromptInspection]{}, loadErr
}}
app := newPromptTestApp(t, Command(suite))
var out bytes.Buffer
result := app.Execute(ctx, []string{"promptkit", "show", "prompt", "example"}, commandkit.Streams{Out: &out, Err: io.Discard})
if result.Status != 1 || !errors.Is(result.Cause, context.Canceled) || !errors.Is(result.Cause, loadErr) || out.Len() != 0 || calls != 1 {
t.Fatalf("outcome=%#v calls=%d output=%q", result, calls, out.String())
}
ctx, cancel = context.WithCancel(context.Background())
cancel()
calls = 0
result = app.Execute(ctx, []string{"promptkit", "show", "prompt", "example"}, commandkit.Streams{Out: &out, Err: io.Discard})
if result.Status != 1 || !errors.Is(result.Cause, context.Canceled) || calls != 0 || out.Len() != 0 {
t.Fatalf("pre-canceled=%#v calls=%d", result, calls)
}
}