Files
commandkit/integrations/ckprompt/requirements_command_test.go

431 lines
20 KiB
Go

package ckprompt
import (
"bytes"
"context"
"errors"
"io"
"reflect"
"strings"
"testing"
commandkit "gitea.maximumdirect.net/eric/CommandKit"
"gitea.maximumdirect.net/eric/CommandKit/integrations/ckconfig"
promptkit "gitea.maximumdirect.net/eric/promptkit"
"go.yaml.in/yaml/v4"
)
const requirementsCommandPrompt = `prompts:
briefing:
version: "2"
default_profile: editorial
session_id: '{{input "session-only"}} {{.session_variable}}'
messages:
- role: user
content: '{{if .mode}}{{input "notes"}}{{end}}'
output:
format: text
validation_mode: none
required-variable:
version: "1"
default_profile: editorial
messages:
- role: user
content: '{{.required}}'
output:
format: text
validation_mode: none
ambiguous-first:
default_profile: editorial
messages:
- role: user
content: "First definition"
output:
format: text
validation_mode: none
ambiguous-second:
default_profile: editorial
messages:
- role: user
content: "Second definition"
output:
format: text
validation_mode: none
backends:
local:
endpoint: "https://example.invalid/v1"
profiles:
editorial:
backend: local
model: example
api_key_required: true
`
func requirementsCommandEngine(t *testing.T) *promptkit.Engine {
t.Helper()
return adapterEngineFrom(t, "requirements command fixture", requirementsCommandPrompt)
}
func executeRequirementsAgainstNative(t *testing.T, engine *promptkit.Engine, request promptkit.RequirementInspectionRequest, args []string) (*promptkit.RequirementInspection, commandkit.Outcome, map[string]any, string) {
t.Helper()
native, err := engine.InspectRequirements(context.Background(), request)
if err != nil {
t.Fatalf("direct native inspection: %v", err)
}
expectedBytes, err := encodePromptYAMLBytes(requirementsReportFromInspection(native))
if err != nil {
t.Fatalf("encode direct native report: %v", err)
}
var expected map[string]any
if err := yaml.Unmarshal(expectedBytes, &expected); err != nil {
t.Fatalf("decode direct native report %q: %v", expectedBytes, err)
}
app := newPromptTestApp(t, Command(FromEngine[int, int](func(context.Context, ckconfig.Selection, int, int) (*promptkit.Engine, error) {
return engine, nil
})))
var output, diagnostic bytes.Buffer
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: &diagnostic})
if outcome.Kind != commandkit.OutcomeSuccess || outcome.Status != 0 {
t.Fatalf("requirements command outcome = %#v, report=%q diagnostics=%q", outcome, output.String(), diagnostic.String())
}
var actual map[string]any
if err := yaml.Unmarshal(output.Bytes(), &actual); err != nil {
t.Fatalf("decode command report %q: %v", output.Bytes(), err)
}
if !reflect.DeepEqual(actual, expected) {
t.Fatalf("command report differs from native inspection:\n got: %#v\nwant: %#v", actual, expected)
}
return native, outcome, actual, diagnostic.String()
}
func TestRequirementsCommandForwardsNativeRequestAndProjectsInspection(t *testing.T) {
engine := requirementsCommandEngine(t)
for _, test := range []struct {
name string
request promptkit.RequirementInspectionRequest
args []string
}{
{
name: "conditional discovery",
request: promptkit.RequirementInspectionRequest{PromptID: "briefing", ProfileID: "editorial"},
args: []string{"promptkit", "requirements", "show", "briefing", "--profile", "editorial"},
},
{
name: "supplied branch variable",
request: promptkit.RequirementInspectionRequest{PromptID: "briefing", ProfileID: "editorial", Vars: map[string]string{"mode": "full"}},
args: []string{"promptkit", "requirements", "show", "briefing", "--profile", "editorial", "--var", "mode=full", "--format", "yaml"},
},
{
name: "prompt with a required variable",
request: promptkit.RequirementInspectionRequest{PromptID: "required-variable", ProfileID: "editorial"},
args: []string{"promptkit", "requirements", "show", "required-variable", "--profile", "editorial"},
},
{
name: "default profile selection",
request: promptkit.RequirementInspectionRequest{PromptID: "briefing"},
args: []string{"promptkit", "requirements", "show", "briefing"},
},
{
name: "first distinct prompt ID",
request: promptkit.RequirementInspectionRequest{PromptID: "ambiguous-first", ProfileID: "editorial"},
args: []string{"promptkit", "requirements", "show", "ambiguous-first", "--profile", "editorial"},
},
{
name: "second distinct prompt ID",
request: promptkit.RequirementInspectionRequest{PromptID: "ambiguous-second", ProfileID: "editorial"},
args: []string{"promptkit", "requirements", "show", "ambiguous-second", "--profile", "editorial"},
},
{
name: "session template context",
request: promptkit.RequirementInspectionRequest{PromptID: "briefing", ProfileID: "editorial"},
args: []string{"promptkit", "requirements", "show", "briefing", "--profile", "editorial"},
},
{
name: "session ID context",
request: promptkit.RequirementInspectionRequest{PromptID: "briefing", ProfileID: "editorial", SessionID: "session-42"},
args: []string{"promptkit", "requirements", "show", "briefing", "--profile", "editorial", "--session-id", "session-42"},
},
} {
t.Run(test.name, func(t *testing.T) {
_, outcome, _, _ := executeRequirementsAgainstNative(t, engine, test.request, test.args)
if outcome.Status != 0 {
t.Fatalf("inspection returned status %d", outcome.Status)
}
})
}
}
func TestRequirementsCommandPreservesNativeSelectionErrors(t *testing.T) {
engine := requirementsCommandEngine(t)
for _, test := range []struct {
name string
args []string
cause error
}{
{
name: "missing prompt",
args: []string{"promptkit", "requirements", "show", "missing", "--profile", "editorial"},
cause: promptkit.ErrPromptNotFound,
},
{
name: "missing profile",
args: []string{"promptkit", "requirements", "show", "briefing", "--profile", "missing"},
cause: promptkit.ErrProfileNotFound,
},
{
name: "removed prompt version flag",
args: []string{"promptkit", "requirements", "show", "briefing", "--version", "2", "--profile", "editorial"},
cause: nil,
},
} {
t.Run(test.name, func(t *testing.T) {
app := newPromptTestApp(t, Command(FromEngine[int, int](func(context.Context, ckconfig.Selection, int, int) (*promptkit.Engine, error) {
return engine, nil
})))
var output bytes.Buffer
outcome := app.Execute(context.Background(), test.args, commandkit.Streams{Out: &output, Err: io.Discard})
if test.cause == nil {
if outcome.Kind != commandkit.OutcomeUsage || output.Len() != 0 {
t.Fatalf("removed flag outcome = %#v output=%q", outcome, output.String())
}
return
}
if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || !errors.Is(outcome.Cause, test.cause) || output.Len() != 0 {
t.Fatalf("native selection outcome = %#v output=%q", outcome, output.String())
}
})
}
}
func TestRequirementsCommandKeepsLifecycleAndCapturedDeclaration(t *testing.T) {
type input struct{ campaign commandkit.Supplied[string] }
type state struct{ calls int }
newCampaignFlag := func(name string) commandkit.Flag[input] {
return commandkit.ValueFlag(commandkit.FlagMeta{Name: name, Required: true}, (*string)(nil), parseDefinitionString,
func(target *input, value commandkit.Supplied[string]) { target.campaign = value })
}
selectors := []commandkit.Flag[input]{newCampaignFlag("campaign")}
callbackData := &struct{ value string }{value: "initial"}
newInputCalls, checkInputCalls, newStateCalls, providerCalls, replacementCalls := 0, 0, 0, 0, 0
var received []input
suite := Suite[input, state]{
NewInput: func() input {
newInputCalls++
return input{}
},
NewState: func() state {
newStateCalls++
return state{calls: newStateCalls}
},
Selectors: selectors,
CheckInput: func(_ ckconfig.Selection, value input) error {
checkInputCalls++
if !value.campaign.Set {
t.Errorf("selected campaign lost presence: %#v", value)
}
return nil
},
InspectRequirements: func(_ context.Context, _ RequirementsRequest, value input, gotState state) (commandkit.Acquired[*promptkit.RequirementInspection], error) {
providerCalls++
received = append(received, value)
if gotState.calls != providerCalls || callbackData.value != "updated" {
t.Errorf("provider state or callback-owned data = %#v/%q", gotState, callbackData.value)
}
return commandkit.Acquired[*promptkit.RequirementInspection]{Value: &promptkit.RequirementInspection{PromptID: "briefing"}}, nil
},
}
declaration := Command(suite)
suite.Selectors[0] = newCampaignFlag("replacement")
suite.InspectRequirements = func(context.Context, RequirementsRequest, input, state) (commandkit.Acquired[*promptkit.RequirementInspection], error) {
replacementCalls++
return commandkit.Acquired[*promptkit.RequirementInspection]{Value: &promptkit.RequirementInspection{}}, nil
}
selectors[0] = newCampaignFlag("also-replacement")
callbackData.value = "updated"
app := newPromptTestApp(t, declaration)
for _, campaign := range []string{"spring", "autumn"} {
args := []string{"promptkit", "requirements", "show", "briefing", "--campaign", campaign}
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: io.Discard, Err: io.Discard})
if outcome.Kind != commandkit.OutcomeSuccess {
t.Fatalf("captured declaration invocation = %#v", outcome)
}
}
if newInputCalls != 2 || checkInputCalls != 2 || newStateCalls != 2 || providerCalls != 2 || replacementCalls != 0 || len(received) != 2 {
t.Fatalf("captured declaration calls = input/check/state/provider/replacement %d/%d/%d/%d/%d", newInputCalls, checkInputCalls, newStateCalls, providerCalls, replacementCalls)
}
if received[0].campaign.Value != "spring" || received[1].campaign.Value != "autumn" {
t.Fatalf("captured selector values = %#v", received)
}
zeroValueApp := newPromptTestApp(t, Command(Suite[*input, *state]{InspectRequirements: func(_ context.Context, _ RequirementsRequest, gotInput *input, gotState *state) (commandkit.Acquired[*promptkit.RequirementInspection], error) {
if gotInput != nil || gotState != nil {
t.Errorf("omitted factories allocated pointer values: %#v/%#v", gotInput, gotState)
}
return commandkit.Acquired[*promptkit.RequirementInspection]{Value: &promptkit.RequirementInspection{}}, nil
}}))
zeroOutcome := zeroValueApp.Execute(context.Background(), []string{"promptkit", "requirements", "show", "briefing"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
if zeroOutcome.Kind != commandkit.OutcomeSuccess {
t.Fatalf("omitted-factory zero values = %#v", zeroOutcome)
}
}
func TestRequirementsCommandChecksInputAndCancellationBeforeAcquisition(t *testing.T) {
inputErr := errors.New("requirements input rejected")
newInputCalls, checkInputCalls, newStateCalls, providerCalls := 0, 0, 0, 0
suite := Suite[struct{}, struct{}]{
NewInput: func() struct{} {
newInputCalls++
return struct{}{}
},
CheckInput: func(ckconfig.Selection, struct{}) error {
checkInputCalls++
return inputErr
},
NewState: func() struct{} {
newStateCalls++
return struct{}{}
},
InspectRequirements: func(context.Context, RequirementsRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.RequirementInspection], error) {
providerCalls++
return commandkit.Acquired[*promptkit.RequirementInspection]{Value: &promptkit.RequirementInspection{}}, nil
},
}
app := newPromptTestApp(t, Command(suite))
args := []string{"promptkit", "requirements", "show", "briefing"}
inputOutcome := app.Execute(context.Background(), args, commandkit.Streams{Out: io.Discard, Err: io.Discard})
if inputOutcome.Kind != commandkit.OutcomeUsage || !errors.Is(inputOutcome.Cause, inputErr) || newInputCalls != 1 || checkInputCalls != 1 || newStateCalls != 0 || providerCalls != 0 {
t.Fatalf("rejected input reached state/provider: outcome=%#v counts=%d/%d/%d/%d", inputOutcome, newInputCalls, checkInputCalls, newStateCalls, providerCalls)
}
suite.CheckInput = nil
app = newPromptTestApp(t, Command(suite))
ctx, cancel := context.WithCancel(context.Background())
cancel()
cancelOutcome := app.Execute(ctx, args, commandkit.Streams{Out: io.Discard, Err: io.Discard})
if cancelOutcome.Kind != commandkit.OutcomeOperational || !errors.Is(cancelOutcome.Cause, context.Canceled) || newStateCalls != 0 || providerCalls != 0 {
t.Fatalf("pre-acquisition cancellation reached state/provider: outcome=%#v counts=%d/%d", cancelOutcome, newStateCalls, providerCalls)
}
}
func TestRequirementsCommandPreservesDiagnosticsCancellationAndDelivery(t *testing.T) {
diagnostic := commandkit.MessageDiagnostic(commandkit.SeverityWarning, "provider warning")
issue := promptkit.RequirementIssue{Code: "native_issue", VariableName: "mode", Description: "native issue details"}
t.Run("partial inspection succeeds and keeps issues in YAML", func(t *testing.T) {
app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{InspectRequirements: func(context.Context, RequirementsRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.RequirementInspection], error) {
return commandkit.Acquired[*promptkit.RequirementInspection]{Value: &promptkit.RequirementInspection{Issues: []promptkit.RequirementIssue{issue}}, Diagnostics: []commandkit.Diagnostic{diagnostic}}, nil
}}))
var report, diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "show", "briefing", "--verbose"}, commandkit.Streams{Out: &report, Err: &diagnostics})
var decoded map[string]any
if err := yaml.Unmarshal(report.Bytes(), &decoded); err != nil {
t.Fatalf("decode partial report %q: %v", report.String(), err)
}
if outcome.Kind != commandkit.OutcomeSuccess || outcome.Status != 0 || outcome.Presentation.Diagnostics.State != commandkit.DeliveryComplete || !strings.Contains(diagnostics.String(), "provider warning") || strings.Contains(diagnostics.String(), "native issue details") {
t.Fatalf("partial inspection outcome=%#v report=%q diagnostics=%q", outcome, report.String(), diagnostics.String())
}
issues, ok := decoded["issues"].([]any)
if !ok || len(issues) != 1 || !strings.Contains(report.String(), "native issue details") {
t.Fatalf("native issue not retained in report: %q", report.String())
}
})
t.Run("provider failure retains diagnostic and cause", func(t *testing.T) {
providerErr := errors.New("requirements provider failed")
app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{InspectRequirements: func(context.Context, RequirementsRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.RequirementInspection], error) {
return commandkit.Acquired[*promptkit.RequirementInspection]{Diagnostics: []commandkit.Diagnostic{diagnostic}}, providerErr
}}))
var report, diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "show", "briefing", "--verbose"}, commandkit.Streams{Out: &report, Err: &diagnostics})
if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || !errors.Is(outcome.Cause, providerErr) || outcome.Presentation.Diagnostics.State != commandkit.DeliveryComplete || !strings.Contains(diagnostics.String(), "provider warning") || report.Len() != 0 {
t.Fatalf("provider failure outcome=%#v report=%q diagnostics=%q", outcome, report.String(), diagnostics.String())
}
})
t.Run("cancellation after provider retains diagnostics and omits report", func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{InspectRequirements: func(context.Context, RequirementsRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.RequirementInspection], error) {
cancel()
return commandkit.Acquired[*promptkit.RequirementInspection]{Value: &promptkit.RequirementInspection{Complete: true}, Diagnostics: []commandkit.Diagnostic{diagnostic}}, nil
}}))
var report, diagnostics bytes.Buffer
outcome := app.Execute(ctx, []string{"promptkit", "requirements", "show", "briefing", "--verbose"}, commandkit.Streams{Out: &report, Err: &diagnostics})
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, context.Canceled) || !strings.Contains(diagnostics.String(), "provider warning") || report.Len() != 0 {
t.Fatalf("post-provider cancellation = %#v report=%q", outcome, report.String())
}
})
t.Run("nil provider result is operational failure", func(t *testing.T) {
app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{InspectRequirements: func(context.Context, RequirementsRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.RequirementInspection], error) {
return commandkit.Acquired[*promptkit.RequirementInspection]{}, nil
}}))
outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "show", "briefing"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || outcome.Cause != nil || !strings.Contains(outcome.Explanation.Message, "returned no result") {
t.Fatalf("nil provider result = %#v", outcome)
}
})
t.Run("report writer failure and short write preserve delivery state", func(t *testing.T) {
writeErr := errors.New("requirements report writer failed")
for _, test := range []struct {
name string
want error
}{
{name: "writer error", want: writeErr},
{name: "short write", want: io.ErrShortWrite},
} {
t.Run(test.name, func(t *testing.T) {
writer := &requirementsPartialWriter{err: test.want}
if test.name == "short write" {
writer.err = nil
writer.short = true
}
app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{InspectRequirements: func(context.Context, RequirementsRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.RequirementInspection], error) {
return commandkit.Acquired[*promptkit.RequirementInspection]{Value: &promptkit.RequirementInspection{PromptID: "briefing"}, Diagnostics: []commandkit.Diagnostic{diagnostic}}, nil
}}))
outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "show", "briefing"}, commandkit.Streams{Out: writer, Err: io.Discard})
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Presentation.Output.WriteError, test.want) || outcome.Presentation.Output.State != commandkit.DeliveryIncomplete || writer.output.Len() == 0 {
t.Fatalf("report delivery = %#v, partial=%q", outcome, writer.output.String())
}
})
}
})
t.Run("diagnostic writer failure is retained", func(t *testing.T) {
diagnosticErr := errors.New("requirements diagnostic writer failed")
badDiagnostic := commandkit.Diagnostic{Severity: commandkit.SeverityWarning, Write: func(io.Writer) error { return diagnosticErr }}
app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{InspectRequirements: func(context.Context, RequirementsRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.RequirementInspection], error) {
return commandkit.Acquired[*promptkit.RequirementInspection]{Value: &promptkit.RequirementInspection{PromptID: "briefing"}, Diagnostics: []commandkit.Diagnostic{badDiagnostic}}, nil
}}))
var report bytes.Buffer
outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "show", "briefing", "--verbose"}, commandkit.Streams{Out: &report, Err: io.Discard})
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Presentation.Diagnostics.RenderError, diagnosticErr) || outcome.Presentation.Diagnostics.State != commandkit.DeliveryIncomplete || report.Len() == 0 {
t.Fatalf("diagnostic delivery outcome = %#v report=%q", outcome, report.String())
}
})
}
type requirementsPartialWriter struct {
output bytes.Buffer
err error
short bool
}
func (writer *requirementsPartialWriter) Write(value []byte) (int, error) {
if len(value) == 0 {
return 0, writer.err
}
count := 1
if count > len(value) {
count = len(value)
}
_, _ = writer.output.Write(value[:count])
if writer.short {
return count, nil
}
return count, writer.err
}