Verify the complete native CLI cutover and record roadmap completion

This commit is contained in:
2026-11-20 05:17:05 +00:00
parent bd7136ceeb
commit f5573f7c8f
7 changed files with 281 additions and 11 deletions

View File

@@ -6,6 +6,7 @@ import (
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
@@ -129,6 +130,41 @@ type failedDeliveryWriter struct{ cause error }
func (writer failedDeliveryWriter) Write([]byte) (int, error) { return 0, writer.cause }
type partialDeliveryWriter struct {
bytes.Buffer
cause error
}
func (writer *partialDeliveryWriter) Write(value []byte) (int, error) {
count := min(2, len(value))
_, _ = writer.Buffer.Write(value[:count])
return count, writer.cause
}
func TestPartialOutputAndDiagnosticFailureUseNativeFinalization(t *testing.T) {
t.Chdir(t.TempDir())
failure := errors.New("presentation failure")
app, err := commandkit.New(commandkit.Identity{Name: "delivery"}, commandkit.NewCommand(commandkit.CommandSpec[struct{}]{Name: "run", Run: func(inv commandkit.Invocation, _ struct{}) commandkit.Outcome {
return deliverRunResult(inv, runInput{ReportPath: commandkit.Supplied[string]{Set: true, Value: "report"}}, &promptkit.RunResult{Artifact: promptkit.Artifact{Body: []byte("artifact")}})
}}))
if err != nil {
t.Fatal(err)
}
output := &partialDeliveryWriter{cause: failure}
var diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), []string{"run"}, commandkit.Streams{Out: output, Err: &diagnostics})
if outcome.Status != 1 || output.String() != "ar" || !errors.Is(outcome.Presentation.Output.WriteError, failure) {
t.Fatalf("partial output was lost: %+v %q", outcome, output.String())
}
if _, err := os.Stat("report"); err != nil {
t.Fatal("report was not attempted", err)
}
outcome = Execute(context.Background(), commandkit.Identity{Name: "scriptorium"}, []string{"run"}, io.Discard, failedDeliveryWriter{failure})
if outcome.Status != 1 || outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Presentation.Diagnostics.WriteError, failure) {
t.Fatalf("usage diagnostic failure not finalized: %+v", outcome)
}
}
func TestNativeDeliveryAttemptsReportAfterStdoutFailure(t *testing.T) {
t.Chdir(t.TempDir())
result := &promptkit.RunResult{RunID: "known-result", Artifact: promptkit.Artifact{Body: []byte("body")}}

View File

@@ -3,11 +3,15 @@ package cli
import (
"encoding/json"
"fmt"
"go.maximumdirect.net/promptkit"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"go.maximumdirect.net/promptkit"
)
func TestCollectionCheckoutDiscoveryAcrossInvocations(t *testing.T) {
@@ -91,3 +95,102 @@ func TestCollectionCheckoutDiscoveryAcrossInvocations(t *testing.T) {
})
}
}
func TestMaintainedCollectionWorkflow(t *testing.T) {
root, err := filepath.Abs(filepath.Join("..", "..", ".."))
if err != nil {
t.Fatal(err)
}
t.Chdir(root)
requests := make(chan []promptkit.RenderedMessage, 2)
provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
var body struct {
Messages []promptkit.RenderedMessage `json:"messages"`
}
if err := json.NewDecoder(request.Body).Decode(&body); err != nil {
t.Error(err)
}
select {
case requests <- body.Messages:
default:
t.Error("unexpected provider dispatch")
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, `{"choices":[{"message":{"content":"collection result"}}]}`)
}))
defer provider.Close()
const config = "examples/config.discovery.yml"
outcome, inventory, diagnostics := executeNative(t, "promptkit", "list", "prompts", "--all-namespaces", "--config="+config, "--format=json")
if outcome.Status != 0 || !strings.Contains(inventory, "support/answer") || !strings.Contains(inventory, "support/summarize") || !strings.Contains(inventory, "review/answer") {
t.Fatalf("collection discovery: %+v %q %q", outcome, inventory, diagnostics)
}
inputPath := filepath.Join(t.TempDir(), "future-transcript.md")
for _, namespace := range []string{"support", "review"} {
profile := "fast"
if namespace == "review" {
profile = "base"
}
outcome, output, diagnostics := executeNative(t, "promptkit", "requirements", "show", "answer", "--config="+config, "--prompt-namespace="+namespace, "--profile="+profile, "--profile-namespace="+namespace, "--format=json")
var report consumerRequirements
if err := json.Unmarshal([]byte(output), &report); err != nil {
t.Fatal(err)
}
if outcome.Status != 0 || !report.Complete || report.PromptID != namespace+"/answer" || report.ProfileID != namespace+"/"+profile {
t.Fatalf("requirements: %+v %+v %q", outcome, report, diagnostics)
}
if namespace == "support" && (len(report.Inputs) != 1 || report.Inputs[0].Name != "transcript" || report.Inputs[0].Kind != promptkit.RequirementRequired) {
t.Fatalf("required binding: %+v", report)
}
}
if _, err := os.Stat(inputPath); !os.IsNotExist(err) {
t.Fatalf("discovery created input artifact: %v", err)
}
writeNativeFile(t, inputPath, "generated collection transcript")
for _, namespace := range []string{"support", "review"} {
profile := "fast"
if namespace == "review" {
profile = "base"
}
flags := []string{"answer", "--config=" + config, "--prompt-namespace=" + namespace, "--profile=" + profile, "--profile-namespace=" + namespace, "--endpoint=" + provider.URL + "/v1", "--model=controlled", "--timeout-seconds=0", "--transient-retries=0", "--session-id=collection-session", "--format=json"}
if namespace == "support" {
flags = append(flags, "--input-file=transcript="+inputPath)
}
outcome, _, diagnostics := executeNative(t, append([]string{"promptkit", "requirements", "validate"}, flags...)...)
if outcome.Status != 0 {
t.Fatalf("validation: %+v %q", outcome, diagnostics)
}
outcome, output, diagnostics := executeNative(t, append([]string{"promptkit", "show", "prepared"}, flags...)...)
var prepared promptkit.PreparedRun
if err := json.Unmarshal([]byte(output), &prepared); err != nil {
t.Fatal(err)
}
if outcome.Status != 0 {
t.Fatalf("preview: %+v %q", outcome, diagnostics)
}
outcome, output, diagnostics = executeNative(t, append([]string{"run"}, flags...)...)
var result promptkit.RunResult
if err := json.Unmarshal([]byte(output), &result); err != nil {
t.Fatal(err)
}
if outcome.Status != 0 || result.PromptID != prepared.PromptID || result.SelectedProfileID != prepared.SelectedProfileID || result.RenderedPromptHash != prepared.RenderedPromptHash || !reflect.DeepEqual(result.InputHashes, prepared.InputHashes) || string(result.Artifact.Body) != "collection result" {
t.Fatalf("workflow selection/context diverged: prepared=%+v result=%+v outcome=%+v %q", prepared, result, outcome, diagnostics)
}
select {
case messages := <-requests:
if !reflect.DeepEqual(messages, prepared.Messages) {
t.Fatalf("provider saw different messages: %+v %+v", messages, prepared.Messages)
}
default:
t.Fatal("generation did not reach controlled provider")
}
}
for _, args := range [][]string{
{"config", "diff", "--scope=app", "--config=" + config, "--against-config=" + config, `--set=/promptkit/schema_dir="left"`, `--against-set=/promptkit/schema_dir="right"`, "--check"},
{"config", "diff", "--scope=promptkit", "--config=" + config, "--against-config=examples/config.yml", "--check"},
} {
outcome, output, diagnostics := executeNative(t, args...)
if outcome.Status != 3 || output == "" {
t.Fatalf("independent configuration sides: %+v %q %q", outcome, output, diagnostics)
}
}
}

View File

@@ -168,10 +168,13 @@ func TestRunDistinguishesCompletedInvalidContentAndOperationalFailure(t *testing
func TestRunUsageAndConfigurationFailures(t *testing.T) {
t.Chdir(t.TempDir())
for _, args := range [][]string{
{"render"}, {"inspect", "prompt"},
{"run"}, {"run", "one", "two"}, {"run", "one", "--prompt=two"}, {"run", "one", "--llm-prompt=two"},
{"run", "one", "--backend=local"}, {"run", "one", "--out=result"}, {"run", "one", "--execution-report=result"},
{"run", "one", "--llm-base-url=http://example.invalid"}, {"run", "one", "--timeout=1s"},
{"run", "one", "--prompt-id=two"}, {"run", "one", "--profile-id=local"}, {"run", "one", "--run-timeout=bad"},
{"run", "one", "--prompt-version=1"}, {"run", "one", "--prompt-dir=prompts"}, {"run", "one", "--profile-dir=profiles"},
{"run", "one", "--config-overlay=local"}, {"run", "one", "--output=file"},
} {
outcome, _, _ := executeNative(t, args...)
if outcome.Kind != commandkit.OutcomeUsage || outcome.Status != 2 {