Add PromptKit inspection commands
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
# Implement the optional integration modules and captured registration API
|
||||
|
||||
Status: implementation in progress. Stages 1 through 11 are complete; subsequent work follows the ordered roadmap.
|
||||
Status: implementation in progress. Stages 1 through 12 are complete; subsequent work follows the ordered roadmap.
|
||||
Execute the stages in order. Each stage is sized as a separate Luna coding-agent
|
||||
assignment. This document authorizes no publication or changes to sibling
|
||||
repositories.
|
||||
@@ -590,6 +590,19 @@ adapter, unused malformed binding, actual installed namespace collisions,
|
||||
definition input/state lifetimes, native report fields, unknown/blank lookup,
|
||||
ExtraParams, and failed output. Run the complete ckprompt suite when coherent.
|
||||
|
||||
**Completion evidence — complete:** `ckprompt.Command` builds a root-only,
|
||||
capability-driven `promptkit show` tree, ignores selectors for empty suites, and
|
||||
retains empty-adapter construction issues. Generated commands use captured core
|
||||
flags, required ID operands, config/overlay selection, prompt-only version, and
|
||||
validated text/YAML formatting. Tests cover installed namespace conflicts,
|
||||
selected input/state lifetimes, missing versus blank IDs, native lookup causes,
|
||||
diagnostics, cancellation and report delivery. Native prompt/profile reports
|
||||
preserve field order, zero values, empty lists, opaque hashes and nested
|
||||
`ExtraParams`; profile output reports environment-variable names and requirement
|
||||
status without resolving credentials. Focused ckprompt adapter/command tests and
|
||||
root tests pass. The complete ckprompt suite remains deferred until Stage 13
|
||||
replaces the prototype sources and mixed-package tests.
|
||||
|
||||
## Stage 13: Finish maintained consumers and module boundary evidence
|
||||
|
||||
**Outcome:** all three adoption paths use the final public API with appropriate
|
||||
|
||||
@@ -143,6 +143,13 @@ func promptKitDiagnostics(scope string, sources []configkit.SourceInfo, records
|
||||
// Snapshot provider-owned slices before deferring their report to invocation.
|
||||
sources = append([]configkit.SourceInfo(nil), sources...)
|
||||
records = append([]configkit.Diagnostic(nil), records...)
|
||||
for index := range records {
|
||||
if records[index].Primary != nil {
|
||||
primary := *records[index].Primary
|
||||
records[index].Primary = &primary
|
||||
}
|
||||
records[index].Related = append([]configkit.Location(nil), records[index].Related...)
|
||||
}
|
||||
return []commandkit.Diagnostic{{
|
||||
Severity: severity,
|
||||
Write: func(writer io.Writer) error {
|
||||
|
||||
@@ -18,12 +18,30 @@ import (
|
||||
const adapterPrompt = `prompts:
|
||||
briefing:
|
||||
"1":
|
||||
default_profile: default
|
||||
inputs:
|
||||
- name: topic
|
||||
required: true
|
||||
content_type: text/plain
|
||||
description: "Topic"
|
||||
- name: audience
|
||||
required: false
|
||||
content_type: text/plain
|
||||
description: ""
|
||||
messages:
|
||||
- role: user
|
||||
content: "Hello"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
empty:
|
||||
"1":
|
||||
messages:
|
||||
- role: user
|
||||
content: "Empty inputs"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
backends:
|
||||
local:
|
||||
endpoint: "https://example.invalid/v1"
|
||||
@@ -31,6 +49,19 @@ profiles:
|
||||
default:
|
||||
backend: local
|
||||
model: example
|
||||
temperature: 0
|
||||
max_tokens: 0
|
||||
api_key_env: PROMPTKIT_ADAPTER_TEST_KEY
|
||||
extra_params:
|
||||
token: "caller-owned-value"
|
||||
nested:
|
||||
visible: true
|
||||
null_value: null
|
||||
sequence: [first, null, 3]
|
||||
requires-key:
|
||||
backend: local
|
||||
model: example
|
||||
api_key_required: true
|
||||
`
|
||||
|
||||
func adapterEngine(t *testing.T) *promptkit.Engine {
|
||||
|
||||
357
integrations/ckprompt/command.go
Normal file
357
integrations/ckprompt/command.go
Normal file
@@ -0,0 +1,357 @@
|
||||
package ckprompt
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
commandkit "gitea.maximumdirect.net/eric/CommandKit"
|
||||
"gitea.maximumdirect.net/eric/CommandKit/integrations/ckconfig"
|
||||
promptkit "gitea.maximumdirect.net/eric/promptkit"
|
||||
"go.yaml.in/yaml/v4"
|
||||
)
|
||||
|
||||
type definitionInput[I any] struct {
|
||||
selection ckconfig.Selection
|
||||
id string
|
||||
version commandkit.Supplied[string]
|
||||
format string
|
||||
captured commandkit.FlagValues[I]
|
||||
selected I
|
||||
}
|
||||
|
||||
// Command returns the optional named PromptKit inspection tree. It adds no
|
||||
// configuration family; callers compose configuration scopes independently.
|
||||
func Command[I, S any](suite Suite[I, S]) commandkit.Command {
|
||||
if suite.InspectPrompt == nil && suite.InspectProfile == nil {
|
||||
command := commandkit.NoCommand()
|
||||
for _, issue := range suite.issues {
|
||||
command = commandkit.WithError(command, issue)
|
||||
}
|
||||
return command
|
||||
}
|
||||
|
||||
showCommands := make([]commandkit.Command, 0, 2)
|
||||
if suite.InspectPrompt != nil {
|
||||
showCommands = append(showCommands, newDefinitionCommand(suite, "prompt", true))
|
||||
}
|
||||
if suite.InspectProfile != nil {
|
||||
showCommands = append(showCommands, newDefinitionCommand(suite, "profile", false))
|
||||
}
|
||||
command := commandkit.RootGroup("promptkit", "Inspect named PromptKit definitions.",
|
||||
commandkit.Group("show", "Show a named PromptKit definition.", showCommands...),
|
||||
)
|
||||
for _, issue := range suite.issues {
|
||||
command = commandkit.WithError(command, issue)
|
||||
}
|
||||
return command
|
||||
}
|
||||
|
||||
func newDefinitionCommand[I, S any](suite Suite[I, S], name string, prompt bool) commandkit.Command {
|
||||
flags := commandkit.CaptureFlags(suite.Selectors, func(input **definitionInput[I]) *commandkit.FlagValues[I] {
|
||||
if input == nil || *input == nil {
|
||||
return nil
|
||||
}
|
||||
return &(*input).captured
|
||||
})
|
||||
flags = append(flags,
|
||||
commandkit.ValueFlag(commandkit.FlagMeta{Name: "config", Usage: "Configuration input path."}, (*string)(nil), parseDefinitionString,
|
||||
func(input **definitionInput[I], value commandkit.Supplied[string]) { (*input).selection.Config = value }),
|
||||
commandkit.ValueFlag(commandkit.FlagMeta{Name: "overlay", Usage: "Configuration overlay path."}, (*string)(nil), parseDefinitionString,
|
||||
func(input **definitionInput[I], value commandkit.Supplied[string]) {
|
||||
(*input).selection.Overlay = value
|
||||
}),
|
||||
commandkit.ValueFlag(commandkit.FlagMeta{Name: "format", Usage: "Report format: text or yaml."}, stringDefault("text"), parseDefinitionString,
|
||||
func(input **definitionInput[I], value commandkit.Supplied[string]) { (*input).format = value.Value }),
|
||||
)
|
||||
if prompt {
|
||||
flags = append(flags, commandkit.ValueFlag(commandkit.FlagMeta{Name: "version", Usage: "Prompt version to inspect."}, (*string)(nil), parseDefinitionString,
|
||||
func(input **definitionInput[I], value commandkit.Supplied[string]) { (*input).version = value }))
|
||||
}
|
||||
|
||||
spec := commandkit.CommandSpec[*definitionInput[I]]{
|
||||
Name: name,
|
||||
Summary: "Inspect a named PromptKit " + name + ".",
|
||||
NewInput: func() *definitionInput[I] {
|
||||
return &definitionInput[I]{}
|
||||
},
|
||||
Flags: flags,
|
||||
Arguments: []commandkit.Argument[*definitionInput[I]]{
|
||||
commandkit.ValueArgument(commandkit.ArgumentMeta{Name: "id", Usage: "Prompt or profile ID."}, true, parseDefinitionString,
|
||||
func(input **definitionInput[I], value commandkit.Supplied[string]) { (*input).id = value.Value }),
|
||||
},
|
||||
CheckInput: func(input *definitionInput[I]) error {
|
||||
if err := checkDefinitionFormat(input.format); err != nil {
|
||||
return err
|
||||
}
|
||||
selected, err := prepareDefinitionInput(suite, input)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
input.selected = selected
|
||||
return nil
|
||||
},
|
||||
Run: func(invocation commandkit.Invocation, input *definitionInput[I]) commandkit.Outcome {
|
||||
return runDefinitionInspection(invocation, suite, input, prompt)
|
||||
},
|
||||
}
|
||||
return commandkit.NewCommand(spec)
|
||||
}
|
||||
|
||||
func prepareDefinitionInput[I, S any](suite Suite[I, S], input *definitionInput[I]) (I, error) {
|
||||
var selected I
|
||||
if suite.NewInput != nil {
|
||||
selected = suite.NewInput()
|
||||
}
|
||||
if err := input.captured.ApplyTo(&selected, suite.Selectors...); err != nil {
|
||||
return selected, err
|
||||
}
|
||||
if suite.CheckInput != nil {
|
||||
if err := suite.CheckInput(input.selection, selected); err != nil {
|
||||
return selected, err
|
||||
}
|
||||
}
|
||||
return selected, nil
|
||||
}
|
||||
|
||||
func runDefinitionInspection[I, S any](invocation commandkit.Invocation, suite Suite[I, S], input *definitionInput[I], prompt bool) commandkit.Outcome {
|
||||
if invocation.Context != nil {
|
||||
if err := invocation.Context.Err(); err != nil {
|
||||
return definitionFailure(err, nil)
|
||||
}
|
||||
}
|
||||
var state S
|
||||
if suite.NewState != nil {
|
||||
state = suite.NewState()
|
||||
}
|
||||
if invocation.Context != nil {
|
||||
if err := invocation.Context.Err(); err != nil {
|
||||
return definitionFailure(err, nil)
|
||||
}
|
||||
}
|
||||
|
||||
if prompt {
|
||||
request := PromptRequest{Selection: input.selection, ID: input.id, Version: input.version}
|
||||
acquired, err := suite.InspectPrompt(invocation.Context, request, input.selected, state)
|
||||
if err != nil {
|
||||
return definitionFailure(err, acquired.Diagnostics)
|
||||
}
|
||||
if acquired.Value == nil {
|
||||
return definitionFailure(errors.New("PromptKit prompt inspection returned nil"), acquired.Diagnostics)
|
||||
}
|
||||
if invocation.Context != nil {
|
||||
if err := invocation.Context.Err(); err != nil {
|
||||
return definitionFailure(err, acquired.Diagnostics)
|
||||
}
|
||||
}
|
||||
return renderPromptReport(invocation, promptReportFromInspection(acquired.Value), input.format, acquired.Diagnostics)
|
||||
}
|
||||
|
||||
request := ProfileRequest{Selection: input.selection, ID: input.id}
|
||||
acquired, err := suite.InspectProfile(invocation.Context, request, input.selected, state)
|
||||
if err != nil {
|
||||
return definitionFailure(err, acquired.Diagnostics)
|
||||
}
|
||||
if acquired.Value == nil {
|
||||
return definitionFailure(errors.New("PromptKit profile inspection returned nil"), acquired.Diagnostics)
|
||||
}
|
||||
if invocation.Context != nil {
|
||||
if err := invocation.Context.Err(); err != nil {
|
||||
return definitionFailure(err, acquired.Diagnostics)
|
||||
}
|
||||
}
|
||||
return renderProfileReport(invocation, profileReportFromInspection(acquired.Value), input.format, acquired.Diagnostics)
|
||||
}
|
||||
|
||||
type promptInputReport struct {
|
||||
Name string `yaml:"name"`
|
||||
Required bool `yaml:"required"`
|
||||
ContentType string `yaml:"content_type"`
|
||||
Description string `yaml:"description"`
|
||||
}
|
||||
|
||||
type promptOutputContractReport struct {
|
||||
Format string `yaml:"format"`
|
||||
ValidationMode string `yaml:"validation_mode"`
|
||||
SchemaPath string `yaml:"schema_path"`
|
||||
RepairAttempts int `yaml:"repair_attempts"`
|
||||
}
|
||||
|
||||
type promptInspectionReport struct {
|
||||
PromptID string `yaml:"prompt_id"`
|
||||
PromptVersion string `yaml:"prompt_version"`
|
||||
PromptHash string `yaml:"prompt_hash"`
|
||||
DefaultProfileID string `yaml:"default_profile_id"`
|
||||
Inputs []promptInputReport `yaml:"inputs"`
|
||||
OutputContract promptOutputContractReport `yaml:"output_contract"`
|
||||
}
|
||||
|
||||
type modelParamsReport struct {
|
||||
BackendID string `yaml:"backend_id"`
|
||||
Endpoint string `yaml:"endpoint"`
|
||||
Model string `yaml:"model"`
|
||||
Temperature float64 `yaml:"temperature"`
|
||||
MaxTokens int `yaml:"max_tokens"`
|
||||
TopP float64 `yaml:"top_p"`
|
||||
TimeoutSeconds int `yaml:"timeout_seconds"`
|
||||
ServiceTier string `yaml:"service_tier"`
|
||||
ReasoningEffort string `yaml:"reasoning_effort"`
|
||||
APIKeyEnv string `yaml:"api_key_env"`
|
||||
ExtraParams map[string]any `yaml:"extra_params"`
|
||||
}
|
||||
|
||||
type profileInspectionReport struct {
|
||||
ProfileID string `yaml:"profile_id"`
|
||||
EffectiveModelParams modelParamsReport `yaml:"effective_model_params"`
|
||||
APIKeyRequired bool `yaml:"api_key_required"`
|
||||
}
|
||||
|
||||
func promptReportFromInspection(inspection *promptkit.PromptInspection) promptInspectionReport {
|
||||
report := promptInspectionReport{
|
||||
PromptID: inspection.PromptID,
|
||||
PromptVersion: inspection.PromptVersion,
|
||||
PromptHash: inspection.PromptHash,
|
||||
DefaultProfileID: inspection.DefaultProfileID,
|
||||
Inputs: make([]promptInputReport, 0, len(inspection.Inputs)),
|
||||
OutputContract: promptOutputContractReport{
|
||||
Format: string(inspection.OutputContract.Format),
|
||||
ValidationMode: string(inspection.OutputContract.ValidationMode),
|
||||
SchemaPath: inspection.OutputContract.SchemaPath,
|
||||
RepairAttempts: inspection.OutputContract.RepairAttempts,
|
||||
},
|
||||
}
|
||||
for _, input := range inspection.Inputs {
|
||||
report.Inputs = append(report.Inputs, promptInputReport{Name: input.Name, Required: input.Required, ContentType: input.ContentType, Description: input.Description})
|
||||
}
|
||||
return report
|
||||
}
|
||||
|
||||
func profileReportFromInspection(inspection *promptkit.ProfileInspection) profileInspectionReport {
|
||||
params := inspection.EffectiveModelParams
|
||||
extra := params.ExtraParams
|
||||
if extra == nil {
|
||||
extra = map[string]any{}
|
||||
}
|
||||
return profileInspectionReport{
|
||||
ProfileID: inspection.ProfileID,
|
||||
EffectiveModelParams: modelParamsReport{
|
||||
BackendID: params.BackendID, Endpoint: params.Endpoint, Model: params.Model,
|
||||
Temperature: params.Temperature, MaxTokens: params.MaxTokens, TopP: params.TopP,
|
||||
TimeoutSeconds: params.TimeoutSeconds, ServiceTier: params.ServiceTier,
|
||||
ReasoningEffort: params.ReasoningEffort, APIKeyEnv: params.APIKeyEnv, ExtraParams: extra,
|
||||
},
|
||||
APIKeyRequired: inspection.APIKeyRequired,
|
||||
}
|
||||
}
|
||||
|
||||
func renderPromptReport(invocation commandkit.Invocation, report promptInspectionReport, format string, diagnostics []commandkit.Diagnostic) commandkit.Outcome {
|
||||
var output bytes.Buffer
|
||||
if format == "yaml" {
|
||||
if err := encodePromptYAML(&output, report); err != nil {
|
||||
return definitionFailure(err, diagnostics)
|
||||
}
|
||||
} else {
|
||||
output.WriteString(formatPromptText(report))
|
||||
}
|
||||
return writePromptReport(invocation, output.Bytes(), diagnostics)
|
||||
}
|
||||
|
||||
func renderProfileReport(invocation commandkit.Invocation, report profileInspectionReport, format string, diagnostics []commandkit.Diagnostic) commandkit.Outcome {
|
||||
var output bytes.Buffer
|
||||
if format == "yaml" {
|
||||
if err := encodePromptYAML(&output, report); err != nil {
|
||||
return definitionFailure(err, diagnostics)
|
||||
}
|
||||
} else {
|
||||
text, err := formatProfileText(report)
|
||||
if err != nil {
|
||||
return definitionFailure(err, diagnostics)
|
||||
}
|
||||
output.WriteString(text)
|
||||
}
|
||||
return writePromptReport(invocation, output.Bytes(), diagnostics)
|
||||
}
|
||||
|
||||
func formatPromptText(report promptInspectionReport) string {
|
||||
var output strings.Builder
|
||||
fmt.Fprintf(&output, "Prompt ID: %s\n", strconv.Quote(report.PromptID))
|
||||
fmt.Fprintf(&output, "Prompt version: %s\n", strconv.Quote(report.PromptVersion))
|
||||
fmt.Fprintf(&output, "Prompt hash: %s\n", strconv.Quote(report.PromptHash))
|
||||
fmt.Fprintf(&output, "Default profile ID: %s\n", strconv.Quote(report.DefaultProfileID))
|
||||
output.WriteString("Inputs:\n")
|
||||
for _, input := range report.Inputs {
|
||||
fmt.Fprintf(&output, " - name: %s\n required: %t\n content_type: %s\n description: %s\n", strconv.Quote(input.Name), input.Required, strconv.Quote(input.ContentType), strconv.Quote(input.Description))
|
||||
}
|
||||
if len(report.Inputs) == 0 {
|
||||
output.WriteString(" []\n")
|
||||
}
|
||||
fmt.Fprintf(&output, "Output contract:\n format: %s\n validation_mode: %s\n schema_path: %s\n repair_attempts: %d\n", strconv.Quote(report.OutputContract.Format), strconv.Quote(report.OutputContract.ValidationMode), strconv.Quote(report.OutputContract.SchemaPath), report.OutputContract.RepairAttempts)
|
||||
return output.String()
|
||||
}
|
||||
|
||||
func formatProfileText(report profileInspectionReport) (string, error) {
|
||||
params := report.EffectiveModelParams
|
||||
var output strings.Builder
|
||||
fmt.Fprintf(&output, "Profile ID: %s\n", strconv.Quote(report.ProfileID))
|
||||
output.WriteString("Effective model params:\n")
|
||||
fmt.Fprintf(&output, " backend_id: %s\n endpoint: %s\n model: %s\n temperature: %s\n max_tokens: %d\n top_p: %s\n timeout_seconds: %d\n service_tier: %s\n reasoning_effort: %s\n api_key_env: %s\n", strconv.Quote(params.BackendID), strconv.Quote(params.Endpoint), strconv.Quote(params.Model), strconv.FormatFloat(params.Temperature, 'g', -1, 64), params.MaxTokens, strconv.FormatFloat(params.TopP, 'g', -1, 64), params.TimeoutSeconds, strconv.Quote(params.ServiceTier), strconv.Quote(params.ReasoningEffort), strconv.Quote(params.APIKeyEnv))
|
||||
output.WriteString(" extra_params:\n")
|
||||
var nested strings.Builder
|
||||
if err := encodePromptYAML(&nested, params.ExtraParams); err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, line := range strings.Split(strings.TrimSuffix(nested.String(), "\n"), "\n") {
|
||||
output.WriteString(" ")
|
||||
output.WriteString(line)
|
||||
output.WriteByte('\n')
|
||||
}
|
||||
fmt.Fprintf(&output, "API key required: %t\n", report.APIKeyRequired)
|
||||
return output.String(), nil
|
||||
}
|
||||
|
||||
func encodePromptYAML(writer io.Writer, value any) error {
|
||||
encoder := yaml.NewEncoder(writer)
|
||||
encoder.SetIndent(2)
|
||||
return encoder.Encode(value)
|
||||
}
|
||||
|
||||
func writePromptReport(invocation commandkit.Invocation, output []byte, diagnostics []commandkit.Diagnostic) commandkit.Outcome {
|
||||
if invocation.Context != nil {
|
||||
if err := invocation.Context.Err(); err != nil {
|
||||
return definitionFailure(err, diagnostics)
|
||||
}
|
||||
}
|
||||
written, err := invocation.Streams.Out.Write(output)
|
||||
if err == nil && written != len(output) {
|
||||
err = io.ErrShortWrite
|
||||
}
|
||||
if err != nil {
|
||||
outcome := definitionFailure(err, diagnostics)
|
||||
outcome.ReportIncomplete = true
|
||||
outcome.DeliveryError = err
|
||||
return outcome
|
||||
}
|
||||
outcome := commandkit.Success()
|
||||
outcome.Diagnostics = diagnostics
|
||||
return outcome
|
||||
}
|
||||
|
||||
func definitionFailure(cause error, diagnostics []commandkit.Diagnostic) commandkit.Outcome {
|
||||
outcome := commandkit.Failure(cause)
|
||||
outcome.Diagnostics = diagnostics
|
||||
return outcome
|
||||
}
|
||||
|
||||
func checkDefinitionFormat(format string) error {
|
||||
if format != "text" && format != "yaml" {
|
||||
return fmt.Errorf("unsupported format %q; expected text or yaml", format)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseDefinitionString(value string) (string, error) { return value, nil }
|
||||
|
||||
func stringDefault(value string) *string { return &value }
|
||||
311
integrations/ckprompt/command_test.go
Normal file
311
integrations/ckprompt/command_test.go
Normal file
@@ -0,0 +1,311 @@
|
||||
package ckprompt
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
commandkit "gitea.maximumdirect.net/eric/CommandKit"
|
||||
"gitea.maximumdirect.net/eric/CommandKit/integrations/ckconfig"
|
||||
promptkit "gitea.maximumdirect.net/eric/promptkit"
|
||||
"go.yaml.in/yaml/v4"
|
||||
)
|
||||
|
||||
func newPromptTestApp(t *testing.T, commands ...commandkit.Command) *commandkit.App {
|
||||
t.Helper()
|
||||
app, err := commandkit.New(commandkit.Identity{Name: "example"}, commands...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return app
|
||||
}
|
||||
|
||||
func TestCommandBuildsIndependentOptionalTrees(t *testing.T) {
|
||||
promptOnly := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{InspectPrompt: func(context.Context, PromptRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.PromptInspection], error) {
|
||||
return commandkit.Acquired[*promptkit.PromptInspection]{}, errors.New("unused")
|
||||
}}))
|
||||
var output bytes.Buffer
|
||||
outcome := promptOnly.Execute(context.Background(), []string{"help", "promptkit", "show"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
||||
if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(output.String(), "\n prompt ") || strings.Contains(output.String(), "\n profile ") || strings.Contains(output.String(), "config") {
|
||||
t.Fatalf("prompt-only help = (%#v, %q)", outcome, output.String())
|
||||
}
|
||||
|
||||
profileOnly := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{InspectProfile: func(context.Context, ProfileRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.ProfileInspection], error) {
|
||||
return commandkit.Acquired[*promptkit.ProfileInspection]{}, errors.New("unused")
|
||||
}}))
|
||||
output.Reset()
|
||||
outcome = profileOnly.Execute(context.Background(), []string{"help", "promptkit", "show"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
||||
if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(output.String(), "\n profile ") || strings.Contains(output.String(), "\n prompt ") {
|
||||
t.Fatalf("profile-only help = (%#v, %q)", outcome, output.String())
|
||||
}
|
||||
|
||||
both := Command(Suite[struct{}, struct{}]{
|
||||
InspectPrompt: func(context.Context, PromptRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.PromptInspection], error) {
|
||||
return commandkit.Acquired[*promptkit.PromptInspection]{}, nil
|
||||
},
|
||||
InspectProfile: func(context.Context, ProfileRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.ProfileInspection], error) {
|
||||
return commandkit.Acquired[*promptkit.ProfileInspection]{}, nil
|
||||
},
|
||||
})
|
||||
if err := newPromptTestApp(t).Register(both); err != nil {
|
||||
t.Fatalf("both-capability tree failed to register: %v", err)
|
||||
}
|
||||
|
||||
app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{Selectors: []commandkit.Flag[struct{}]{{}}}))
|
||||
output.Reset()
|
||||
outcome = app.Execute(context.Background(), []string{"help"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
||||
if outcome.Kind != commandkit.OutcomeSuccess || strings.Contains(output.String(), "promptkit") {
|
||||
t.Fatalf("empty suite installed a namespace: (%#v, %q)", outcome, output.String())
|
||||
}
|
||||
|
||||
invalid := FromEngine[int, int](nil)
|
||||
invalid.InspectPrompt, invalid.InspectProfile = nil, nil
|
||||
if err := newPromptTestApp(t).Register(Command(invalid)); err == nil || !strings.Contains(err.Error(), "PromptKit engine loader is nil") {
|
||||
t.Fatalf("empty invalid adapter lost its construction issue: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCommandUsesInstalledNamespaceCollisionChecks(t *testing.T) {
|
||||
command := Command(Suite[struct{}, struct{}]{InspectPrompt: func(context.Context, PromptRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.PromptInspection], error) {
|
||||
return commandkit.Acquired[*promptkit.PromptInspection]{}, nil
|
||||
}})
|
||||
other := commandkit.Group("promptkit", "Another command tree.", commandkit.NewCommand(commandkit.CommandSpec[struct{}]{Name: "child", Summary: "A child."}))
|
||||
first, err := commandkit.New(commandkit.Identity{Name: "example"}, other)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := first.Register(command); err == nil {
|
||||
t.Fatal("PromptKit tree installed over an existing namespace")
|
||||
}
|
||||
second, err := commandkit.New(commandkit.Identity{Name: "example"}, command)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := second.Register(other); err == nil {
|
||||
t.Fatal("later command tree installed over the PromptKit namespace")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCommandValidatesOnlyGeneratedCapturedBindings(t *testing.T) {
|
||||
unused := Command(Suite[struct{}, struct{}]{Selectors: []commandkit.Flag[struct{}]{{}}})
|
||||
if err := newPromptTestApp(t).Register(unused); err != nil {
|
||||
t.Fatalf("unused malformed selector affected empty suite: %v", err)
|
||||
}
|
||||
generated := Command(Suite[struct{}, struct{}]{Selectors: []commandkit.Flag[struct{}]{{}}, InspectProfile: func(context.Context, ProfileRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.ProfileInspection], error) {
|
||||
return commandkit.Acquired[*promptkit.ProfileInspection]{}, nil
|
||||
}})
|
||||
if err := newPromptTestApp(t).Register(generated); err == nil {
|
||||
t.Fatal("generated command accepted an invalid captured selector")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefinitionInputAndStateLifetimesFollowChecks(t *testing.T) {
|
||||
type inspectInput struct{ Campaign commandkit.Supplied[string] }
|
||||
type runState struct{}
|
||||
conversions := 0
|
||||
flag := commandkit.ValueFlag[*inspectInput, string](commandkit.FlagMeta{Name: "campaign", Required: true}, (*string)(nil), func(value string) (string, error) {
|
||||
conversions++
|
||||
return value, nil
|
||||
}, func(input **inspectInput, value commandkit.Supplied[string]) { (*input).Campaign = value })
|
||||
newInputs, newStates, checks, callbacks := 0, 0, 0, 0
|
||||
rejectInput := false
|
||||
var checked *inspectInput
|
||||
var wantContext context.Context
|
||||
suite := Suite[*inspectInput, *runState]{
|
||||
NewInput: func() *inspectInput { newInputs++; return &inspectInput{} },
|
||||
NewState: func() *runState {
|
||||
newStates++
|
||||
if checks != 1 {
|
||||
t.Errorf("state initialized after %d input checks", checks)
|
||||
}
|
||||
return &runState{}
|
||||
},
|
||||
Selectors: []commandkit.Flag[*inspectInput]{flag},
|
||||
CheckInput: func(selection ckconfig.Selection, input *inspectInput) error {
|
||||
checks++
|
||||
checked = input
|
||||
if rejectInput {
|
||||
return errors.New("rejected definition input")
|
||||
}
|
||||
if !selection.Config.Set || selection.Config.Value != "base.yml" || !selection.Overlay.Set || selection.Overlay.Value != "local.yml" || !input.Campaign.Set || input.Campaign.Value != "spring" {
|
||||
return errors.New("definition input was incomplete")
|
||||
}
|
||||
return nil
|
||||
},
|
||||
InspectPrompt: func(ctx context.Context, request PromptRequest, input *inspectInput, state *runState) (commandkit.Acquired[*promptkit.PromptInspection], error) {
|
||||
callbacks++
|
||||
if ctx != wantContext || input != checked || request.ID != "briefing" || !request.Version.Set || request.Version.Value != "2" || state == nil {
|
||||
t.Errorf("prompt callback arguments = ctx=%v request=%#v input=%p state=%p", ctx, request, input, state)
|
||||
}
|
||||
return commandkit.Acquired[*promptkit.PromptInspection]{Value: &promptkit.PromptInspection{PromptID: request.ID, PromptVersion: request.Version.Value}}, nil
|
||||
},
|
||||
}
|
||||
app := newPromptTestApp(t, Command(suite))
|
||||
wantContext = context.WithValue(context.Background(), struct{}{}, "context")
|
||||
var output bytes.Buffer
|
||||
outcome := app.Execute(wantContext, []string{"promptkit", "show", "prompt", "briefing", "--campaign", "spring", "--config", "base.yml", "--overlay", "local.yml", "--version", "2"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
||||
if outcome.Kind != commandkit.OutcomeSuccess || newInputs != 1 || newStates != 1 || checks != 1 || callbacks != 1 || conversions != 1 {
|
||||
t.Fatalf("execution outcome=%#v counts conversion/input/state/check/callback=%d/%d/%d/%d/%d report=%q", outcome, conversions, newInputs, newStates, checks, callbacks, output.String())
|
||||
}
|
||||
rejectInput = true
|
||||
output.Reset()
|
||||
outcome = app.Execute(wantContext, []string{"promptkit", "show", "prompt", "briefing", "--campaign", "spring", "--config", "base.yml", "--overlay", "local.yml", "--version", "2"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
||||
if outcome.Kind != commandkit.OutcomeUsage || newInputs != 2 || newStates != 1 || checks != 2 || callbacks != 1 || conversions != 2 {
|
||||
t.Fatalf("rejected input reached state/callback: outcome=%#v counts=%d/%d/%d/%d/%d", outcome, conversions, newInputs, newStates, checks, callbacks)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefinitionReportsRetainNativeFieldsAndSafeCredentialReferences(t *testing.T) {
|
||||
t.Setenv("PROMPTKIT_ADAPTER_TEST_KEY", "credential-value-must-not-appear")
|
||||
engine := adapterEngine(t)
|
||||
loadCalls := 0
|
||||
suite := FromEngine(func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) {
|
||||
loadCalls++
|
||||
return engine, nil
|
||||
})
|
||||
app := newPromptTestApp(t, Command(suite))
|
||||
var output bytes.Buffer
|
||||
outcome := app.Execute(context.Background(), []string{"promptkit", "show", "prompt", "briefing", "--version", "1", "--format", "yaml"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
||||
if outcome.Kind != commandkit.OutcomeSuccess || loadCalls != 1 {
|
||||
t.Fatalf("prompt report outcome=%#v load calls=%d report=%s", outcome, loadCalls, output.String())
|
||||
}
|
||||
var prompt map[string]any
|
||||
if err := yaml.Unmarshal(output.Bytes(), &prompt); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inputs := prompt["inputs"].([]any)
|
||||
if len(inputs) != 2 || inputs[0].(map[string]any)["name"] != "topic" || inputs[1].(map[string]any)["name"] != "audience" || prompt["prompt_hash"] == "" {
|
||||
t.Fatalf("prompt fields/order = %#v", prompt)
|
||||
}
|
||||
contract := prompt["output_contract"].(map[string]any)
|
||||
if contract["repair_attempts"] != 0 {
|
||||
t.Fatalf("zero-valued contract field was lost: %#v", contract)
|
||||
}
|
||||
|
||||
output.Reset()
|
||||
outcome = app.Execute(context.Background(), []string{"promptkit", "show", "prompt", "briefing"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
||||
if outcome.Kind != commandkit.OutcomeSuccess || loadCalls != 2 || !strings.Contains(output.String(), "Prompt version: \"1\"") || !strings.Contains(output.String(), `- name: "topic"`) {
|
||||
t.Fatalf("default text prompt report = (%#v, calls=%d, %s)", outcome, loadCalls, output.String())
|
||||
}
|
||||
|
||||
output.Reset()
|
||||
outcome = app.Execute(context.Background(), []string{"promptkit", "show", "profile", "default", "--format", "yaml"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
||||
if outcome.Kind != commandkit.OutcomeSuccess || loadCalls != 3 {
|
||||
t.Fatalf("profile report outcome=%#v calls=%d report=%s", outcome, loadCalls, output.String())
|
||||
}
|
||||
var profile map[string]any
|
||||
if err := yaml.Unmarshal(output.Bytes(), &profile); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
params := profile["effective_model_params"].(map[string]any)
|
||||
if params["api_key_env"] != "PROMPTKIT_ADAPTER_TEST_KEY" || params["temperature"] != 0 || params["max_tokens"] != 0 || strings.Contains(output.String(), "credential-value-must-not-appear") {
|
||||
t.Fatalf("credential or zero model fields = %#v / %s", params, output.String())
|
||||
}
|
||||
extra := params["extra_params"].(map[string]any)
|
||||
nested := extra["nested"].(map[string]any)
|
||||
sequence := extra["sequence"].([]any)
|
||||
if extra["token"] != "caller-owned-value" || nested["null_value"] != nil || nested["visible"] != true || len(sequence) != 3 || sequence[1] != nil || sequence[2] != 3 {
|
||||
t.Fatalf("nested ExtraParams changed: %#v", extra)
|
||||
}
|
||||
|
||||
output.Reset()
|
||||
outcome = app.Execute(context.Background(), []string{"promptkit", "show", "profile", "default"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
||||
if outcome.Kind != commandkit.OutcomeSuccess || loadCalls != 4 || !strings.Contains(output.String(), "API key required: false") || !strings.Contains(output.String(), `api_key_env: "PROMPTKIT_ADAPTER_TEST_KEY"`) {
|
||||
t.Fatalf("default text profile report = (%#v, calls=%d, %s)", outcome, loadCalls, output.String())
|
||||
}
|
||||
|
||||
output.Reset()
|
||||
outcome = app.Execute(context.Background(), []string{"promptkit", "show", "profile", "requires-key", "--format", "yaml"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
||||
if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(output.String(), "api_key_required: true") {
|
||||
t.Fatalf("credential requirement report = (%#v, %s)", outcome, output.String())
|
||||
}
|
||||
|
||||
output.Reset()
|
||||
outcome = app.Execute(context.Background(), []string{"promptkit", "show", "prompt", "empty", "--format", "yaml"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
||||
if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(output.String(), "inputs: []") {
|
||||
t.Fatalf("empty prompt input list = (%#v, %s)", outcome, output.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeLookupErrorsAndBlankIDsReachCallbacks(t *testing.T) {
|
||||
nativeErr := errors.New("native lookup failure")
|
||||
calls := 0
|
||||
diagnostic := commandkit.MessageDiagnostic(commandkit.SeverityWarning, "lookup context")
|
||||
app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{InspectPrompt: func(_ context.Context, request PromptRequest, _ struct{}, _ struct{}) (commandkit.Acquired[*promptkit.PromptInspection], error) {
|
||||
calls++
|
||||
if request.ID != "" {
|
||||
t.Errorf("blank ID was rewritten to %q", request.ID)
|
||||
}
|
||||
return commandkit.Acquired[*promptkit.PromptInspection]{Diagnostics: []commandkit.Diagnostic{diagnostic}}, nativeErr
|
||||
}}))
|
||||
var output, diagnosticOutput bytes.Buffer
|
||||
outcome := app.Execute(context.Background(), []string{"promptkit", "show", "prompt"}, commandkit.Streams{Out: &output, Err: &diagnosticOutput})
|
||||
if outcome.Kind != commandkit.OutcomeUsage || calls != 0 || output.Len() != 0 {
|
||||
t.Fatalf("missing ID was not a usage failure: (%#v, calls=%d, out=%q)", outcome, calls, output.String())
|
||||
}
|
||||
output.Reset()
|
||||
diagnosticOutput.Reset()
|
||||
outcome = app.Execute(context.Background(), []string{"promptkit", "show", "prompt", ""}, commandkit.Streams{Out: &output, Err: &diagnosticOutput})
|
||||
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, nativeErr) || calls != 1 || output.Len() != 0 || !strings.Contains(diagnosticOutput.String(), "lookup context") {
|
||||
t.Fatalf("native lookup failure = (%#v, calls=%d, out=%q, err=%q)", outcome, calls, output.String(), diagnosticOutput.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestCancelledInspectionDoesNotRender(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
called := 0
|
||||
app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{InspectProfile: func(context.Context, ProfileRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.ProfileInspection], error) {
|
||||
called++
|
||||
cancel()
|
||||
return commandkit.Acquired[*promptkit.ProfileInspection]{Value: &promptkit.ProfileInspection{ProfileID: "default"}}, nil
|
||||
}}))
|
||||
var output bytes.Buffer
|
||||
outcome := app.Execute(ctx, []string{"promptkit", "show", "profile", "default"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
||||
if !errors.Is(outcome.Cause, context.Canceled) || outcome.Kind != commandkit.OutcomeOperational || called != 1 || output.Len() != 0 {
|
||||
t.Fatalf("cancelled inspection = (%#v, called=%d, output=%q)", outcome, called, output.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefinitionFormatPreflightAndReportDeliveryFailure(t *testing.T) {
|
||||
calls := 0
|
||||
app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{InspectProfile: func(context.Context, ProfileRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.ProfileInspection], error) {
|
||||
calls++
|
||||
return commandkit.Acquired[*promptkit.ProfileInspection]{Value: &promptkit.ProfileInspection{ProfileID: "default"}}, nil
|
||||
}}))
|
||||
var output bytes.Buffer
|
||||
outcome := app.Execute(context.Background(), []string{"promptkit", "show", "profile", "default", "--format", "json"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
||||
if outcome.Kind != commandkit.OutcomeUsage || calls != 0 || output.Len() != 0 {
|
||||
t.Fatalf("bad format was not rejected before callback: outcome=%#v calls=%d output=%q", outcome, calls, output.String())
|
||||
}
|
||||
|
||||
writeErr := errors.New("report writer failed")
|
||||
outcome = app.Execute(context.Background(), []string{"promptkit", "show", "profile", "default"}, commandkit.Streams{Out: partialReportWriter{err: writeErr}, Err: io.Discard})
|
||||
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, writeErr) || outcome.DeliveryError != writeErr || !outcome.ReportIncomplete || calls != 1 {
|
||||
t.Fatalf("report delivery outcome = %#v calls=%d", outcome, calls)
|
||||
}
|
||||
}
|
||||
|
||||
type partialReportWriter struct{ err error }
|
||||
|
||||
func (writer partialReportWriter) Write(content []byte) (int, error) {
|
||||
if len(content) == 0 {
|
||||
return 0, writer.err
|
||||
}
|
||||
return 1, writer.err
|
||||
}
|
||||
|
||||
var _ io.Writer = partialReportWriter{}
|
||||
|
||||
func TestProfileTextUsesCredentialNamesAndExplicitEmptyExtraParams(t *testing.T) {
|
||||
report := profileReportFromInspection(&promptkit.ProfileInspection{ProfileID: "default", EffectiveModelParams: promptkit.ExecutionTarget{APIKeyEnv: "APP_KEY", ExtraParams: nil}})
|
||||
text, err := formatProfileText(report)
|
||||
if err != nil || !strings.Contains(text, "api_key_env: \"APP_KEY\"") || !strings.Contains(text, "extra_params:\n {}\n") || !strings.Contains(text, "API key required: false") {
|
||||
t.Fatalf("profile text = (%q, %v)", text, err)
|
||||
}
|
||||
if strings.Contains(text, "APP_KEY_VALUE") {
|
||||
t.Fatalf("report contains a credential value: %q", text)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user