Verify the complete native CLI cutover and record roadmap completion

This commit is contained in:
2026-11-20 05:17:05 +00:00
parent bd7136ceeb
commit f5573f7c8f
7 changed files with 281 additions and 11 deletions

View File

@@ -3,8 +3,20 @@ package main
import ( import (
"bytes" "bytes"
"context" "context"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"runtime"
"slices"
"strings" "strings"
"syscall"
"testing" "testing"
"time"
) )
func TestProcessIdentityAndNativeStatus(t *testing.T) { func TestProcessIdentityAndNativeStatus(t *testing.T) {
@@ -23,3 +35,79 @@ func TestProcessIdentityAndNativeStatus(t *testing.T) {
t.Fatalf("canceled status=%d diagnostics=%q", status, diagnostics.String()) t.Fatalf("canceled status=%d diagnostics=%q", status, diagnostics.String())
} }
} }
func TestProcessSignalCancellation(t *testing.T) {
if os.Getenv("SCRIPTORIUM_SIGNAL_CHILD") == "1" {
separator := slices.Index(os.Args, "--")
if separator < 0 {
os.Exit(2)
}
os.Exit(runWithSignals(os.Args[separator+1:], os.Stdout, os.Stderr))
}
if runtime.GOOS == "windows" {
t.Skip("process interrupt/SIGTERM integration is Unix-specific")
}
for _, signal := range []os.Signal{os.Interrupt, syscall.SIGTERM} {
t.Run(signal.String(), func(t *testing.T) {
started, stopped, release := make(chan struct{}), make(chan struct{}), make(chan struct{})
provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = io.Copy(io.Discard, r.Body)
close(started)
select {
case <-r.Context().Done():
close(stopped)
case <-release:
}
}))
defer provider.Close()
defer close(release)
root := t.TempDir()
framework := filepath.Join(root, "framework.yml")
content := fmt.Sprintf("profiles:\n - id: local\n endpoint: %q\n model: controlled\nprompts:\n - id: selected\n default_profile: local\n messages:\n - role: user\n content: Signal test\n output:\n format: text\n validation_mode: none\n", provider.URL+"/v1")
if err := os.WriteFile(framework, []byte(content), 0600); err != nil {
t.Fatal(err)
}
config := filepath.Join(root, "app.yml")
if err := os.WriteFile(config, []byte(fmt.Sprintf("promptkit:\n configuration: %q\ntransient_retries: 0\n", framework)), 0600); err != nil {
t.Fatal(err)
}
executable, err := os.Executable()
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
command := exec.CommandContext(ctx, executable, "-test.run=^TestProcessSignalCancellation$", "--", "run", "selected", "--config="+config, "--timeout-seconds=0")
command.Env = append(os.Environ(), "SCRIPTORIUM_SIGNAL_CHILD=1")
var output, diagnostics bytes.Buffer
command.Stdout, command.Stderr = &output, &diagnostics
if err := command.Start(); err != nil {
t.Fatal(err)
}
defer func() {
cancel()
if command.ProcessState == nil {
_ = command.Wait()
}
}()
select {
case <-started:
case <-ctx.Done():
t.Fatal("child did not reach controlled provider")
}
if err := command.Process.Signal(signal); err != nil {
t.Fatal(err)
}
err = command.Wait()
var exit *exec.ExitError
if !errors.As(err, &exit) || exit.ExitCode() != 1 || output.Len() != 0 || diagnostics.Len() == 0 {
t.Fatalf("signal outcome: %v stdout=%q stderr=%q", err, output.String(), diagnostics.String())
}
select {
case <-stopped:
case <-time.After(5 * time.Second):
t.Fatal("signal did not cancel provider request")
}
})
}
}

View File

@@ -2,8 +2,9 @@
## Status And Intent ## Status And Intent
Initial roadmap, prepared 2026-11-19. The product direction is agreed; Implementation and assembled acceptance verification are complete; see the
implementation is underway. This document owns the target and product intent, [implementation plan](implementation.md) for evidence and selected releases.
Initial roadmap prepared 2026-11-19. This document owns the target and product intent,
incorporating the decisions in the [migration notepad](commandkit-notepad.md). incorporating the decisions in the [migration notepad](commandkit-notepad.md).
The notepad remains the discussion The notepad remains the discussion
record. The [Narratio proposal](narratio-feature-request.md) supplies consumer record. The [Narratio proposal](narratio-feature-request.md) supplies consumer
@@ -26,13 +27,13 @@ report shapes, and exit conventions are not compatibility constraints. Release
migration guidance replaces a compatibility layer. Implementation may proceed migration guidance replaces a compatibility layer. Implementation may proceed
in reviewable increments; this does not require shipping both interfaces. in reviewable increments; this does not require shipping both interfaces.
## Current Baseline And Upstream References ## Planning Baseline And Upstream References
Reviewed Scriptorium baseline: `5f32a4f2adb2d382a6cfce3a787ada4576be7705`. Reviewed Scriptorium baseline: `5f32a4f2adb2d382a6cfce3a787ada4576be7705`.
The module uses Go 1.27.0, PromptKit v0.23.3, and ConfigKit v1.2.1. CommandKit At that baseline the module used Go 1.27.0, PromptKit v0.23.3, and ConfigKit
is not yet a dependency. v1.2.1, without CommandKit. The following table records the starting point.
| Current implementation | Migration work | | Baseline implementation | Migration work |
| --- | --- | | --- | --- |
| `internal/adapter/cli` dispatches and parses commands, resolves configuration, builds engines and requests, writes diagnostics, and returns integer statuses. | Replace dispatch and shared CLI mechanics with CommandKit; retain application loading, request mapping, and effects through its extension points. | | `internal/adapter/cli` dispatches and parses commands, resolves configuration, builds engines and requests, writes diagnostics, and returns integer statuses. | Replace dispatch and shared CLI mechanics with CommandKit; retain application loading, request mapping, and effects through its extension points. |
| `run` uses flag-based prompt selection and writes generated content; `--execution-report` writes an observer-derived Scriptorium report. | Adopt positional prompt selection, native request flags, and native result reporting. | | `run` uses flag-based prompt selection and writes generated content; `--execution-report` writes an observer-derived Scriptorium report. | Adopt positional prompt selection, native request flags, and native result reporting. |
@@ -44,8 +45,8 @@ is not yet a dependency.
The [current CLI](../cli.md), [configuration contract](../config.md), The [current CLI](../cli.md), [configuration contract](../config.md),
[adapter internals](../internal/adapters.md), and [adapter internals](../internal/adapters.md), and
[subprocess contract](../integrations/subprocess.md) describe the starting [subprocess contract](../integrations/subprocess.md) now describe the implemented
behavior, not a compatibility checklist for this migration. native interface, not the historical starting point or a compatibility layer.
The reviewed upstream combination is CommandKit v0.7.0, ckconfig v0.9.0, The reviewed upstream combination is CommandKit v0.7.0, ckconfig v0.9.0,
and ckprompt v0.10.0, documented at revision and ckprompt v0.10.0, documented at revision

View File

@@ -723,6 +723,44 @@ current documentation links resolve. Migration examples match the built CLI.
## Stage 9: Verify The Complete Native Cutover ## Stage 9: Verify The Complete Native Cutover
**Status: Complete.** Verified the assembled native CLI on Go 1.27.4 with
`go test ./... -count=1`, `go test -race ./... -count=1`, `go vet ./...`,
and `go build ./cmd/scriptorium`. Formatting, diff whitespace, module tidiness,
shell syntax, maintained examples, and local Markdown links/anchors pass.
The maintained discovery configuration now has an end-to-end controlled-provider
test covering support/review inventories, requirements before artifacts exist,
concrete validation, preview/run context agreement, and independent native app
and framework comparisons. Existing checkout tests verify both optional entry
spellings and discovery on later invocations. Additional checks cover rejected
legacy surfaces, partial output, diagnostic delivery failure, and real child
process interrupt/SIGTERM cancellation reaching a provider request.
Built Linux amd64 and arm64 with Go 1.27.4, `CGO_ENABLED=0`, `-trimpath`,
and `-s -w` plus version/revision linker metadata. The amd64 binary reports
the injected `verification` identity and full source revision
`bd7136ceebbdc7498e9a768ce8abbe0cbb00318f`; arm64 was cross-built, not executed.
The binary also exposes help, native scope discovery, and Bash/Zsh/Fish
completion without loading configuration or contacting a provider.
Selected releases remain CommandKit v0.7.0
(`b57bfe176cff05a892e6cb07ebb5f350239e55d4`), ckconfig v0.9.0
(`9962368f0c1ea1f6ad010638e94621091105d745`), ckprompt v0.10.0
(`c9a56e1b15f2169881a91fca4e1f4a1c475e057a`), ConfigKit v1.2.1
(`51ef3144158db1a09fabdbd1c7ae59908c84923b`), and PromptKit v0.23.3
(`804a4d32b23aa2e7973400bb0a322a7016574c9f`). Module and linked-runtime
inspection confirms no replacements, workspace, vendoring, or linked maintainer
tools; attribution follows the linked dependencies, including conditional
Windows mousetrap. Public native adapters, request bindings, reporters,
`RenderReport`, and typed outcomes remain the selected integration APIs.
The final implementation audit against this plan and ADRs 0004–0006 found no
remaining required work. HTTP/report implementation and accepted ADRs remain
unchanged; no compatibility surface, post-resolution config rewriting, extra
readiness gate, workflow state, snapshot promise, or transactional delivery was
added. Embedded framework configuration, dynamic-resource completion, the
optional format accessor, and HTTP redesign remain outside this migration.
No release was published and no commits were pushed.
1. Finish any deferred stage checks against the complete application. Use real 1. Finish any deferred stage checks against the complete application. Use real
ConfigKit, PromptKit, and CommandKit with deterministic provider substitutes ConfigKit, PromptKit, and CommandKit with deterministic provider substitutes
at the external boundary. Tests remain offline and independent of real keys. at the external boundary. Tests remain offline and independent of real keys.

View File

@@ -1,7 +1,8 @@
# Prompt input requirements for Narratio # Prompt input requirements for Narratio
Status: the native interface and focused consumer scenarios are implemented. Status: the native interface and consumer acceptance scenarios are implemented
The [implementation plan](implementation.md) tracks final assembled verification. and verified, including the assembled collection workflow. The
[implementation plan](implementation.md) records the verification evidence.
This proposal records Narratio's consumer needs and their mapping to the native This proposal records Narratio's consumer needs and their mapping to the native
interface; the [CLI contract](../cli.md) owns current behavior. interface; the [CLI contract](../cli.md) owns current behavior.
The [CommandKit notepad](commandkit-notepad.md) records the migration decisions. The [CommandKit notepad](commandkit-notepad.md) records the migration decisions.

View File

@@ -6,6 +6,7 @@ import (
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
"io"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"os" "os"
@@ -129,6 +130,41 @@ type failedDeliveryWriter struct{ cause error }
func (writer failedDeliveryWriter) Write([]byte) (int, error) { return 0, writer.cause } func (writer failedDeliveryWriter) Write([]byte) (int, error) { return 0, writer.cause }
type partialDeliveryWriter struct {
bytes.Buffer
cause error
}
func (writer *partialDeliveryWriter) Write(value []byte) (int, error) {
count := min(2, len(value))
_, _ = writer.Buffer.Write(value[:count])
return count, writer.cause
}
func TestPartialOutputAndDiagnosticFailureUseNativeFinalization(t *testing.T) {
t.Chdir(t.TempDir())
failure := errors.New("presentation failure")
app, err := commandkit.New(commandkit.Identity{Name: "delivery"}, commandkit.NewCommand(commandkit.CommandSpec[struct{}]{Name: "run", Run: func(inv commandkit.Invocation, _ struct{}) commandkit.Outcome {
return deliverRunResult(inv, runInput{ReportPath: commandkit.Supplied[string]{Set: true, Value: "report"}}, &promptkit.RunResult{Artifact: promptkit.Artifact{Body: []byte("artifact")}})
}}))
if err != nil {
t.Fatal(err)
}
output := &partialDeliveryWriter{cause: failure}
var diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), []string{"run"}, commandkit.Streams{Out: output, Err: &diagnostics})
if outcome.Status != 1 || output.String() != "ar" || !errors.Is(outcome.Presentation.Output.WriteError, failure) {
t.Fatalf("partial output was lost: %+v %q", outcome, output.String())
}
if _, err := os.Stat("report"); err != nil {
t.Fatal("report was not attempted", err)
}
outcome = Execute(context.Background(), commandkit.Identity{Name: "scriptorium"}, []string{"run"}, io.Discard, failedDeliveryWriter{failure})
if outcome.Status != 1 || outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Presentation.Diagnostics.WriteError, failure) {
t.Fatalf("usage diagnostic failure not finalized: %+v", outcome)
}
}
func TestNativeDeliveryAttemptsReportAfterStdoutFailure(t *testing.T) { func TestNativeDeliveryAttemptsReportAfterStdoutFailure(t *testing.T) {
t.Chdir(t.TempDir()) t.Chdir(t.TempDir())
result := &promptkit.RunResult{RunID: "known-result", Artifact: promptkit.Artifact{Body: []byte("body")}} result := &promptkit.RunResult{RunID: "known-result", Artifact: promptkit.Artifact{Body: []byte("body")}}

View File

@@ -3,11 +3,15 @@ package cli
import ( import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"go.maximumdirect.net/promptkit" "net/http"
"net/http/httptest"
"os" "os"
"path/filepath" "path/filepath"
"reflect"
"strings" "strings"
"testing" "testing"
"go.maximumdirect.net/promptkit"
) )
func TestCollectionCheckoutDiscoveryAcrossInvocations(t *testing.T) { func TestCollectionCheckoutDiscoveryAcrossInvocations(t *testing.T) {
@@ -91,3 +95,102 @@ func TestCollectionCheckoutDiscoveryAcrossInvocations(t *testing.T) {
}) })
} }
} }
func TestMaintainedCollectionWorkflow(t *testing.T) {
root, err := filepath.Abs(filepath.Join("..", "..", ".."))
if err != nil {
t.Fatal(err)
}
t.Chdir(root)
requests := make(chan []promptkit.RenderedMessage, 2)
provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
var body struct {
Messages []promptkit.RenderedMessage `json:"messages"`
}
if err := json.NewDecoder(request.Body).Decode(&body); err != nil {
t.Error(err)
}
select {
case requests <- body.Messages:
default:
t.Error("unexpected provider dispatch")
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, `{"choices":[{"message":{"content":"collection result"}}]}`)
}))
defer provider.Close()
const config = "examples/config.discovery.yml"
outcome, inventory, diagnostics := executeNative(t, "promptkit", "list", "prompts", "--all-namespaces", "--config="+config, "--format=json")
if outcome.Status != 0 || !strings.Contains(inventory, "support/answer") || !strings.Contains(inventory, "support/summarize") || !strings.Contains(inventory, "review/answer") {
t.Fatalf("collection discovery: %+v %q %q", outcome, inventory, diagnostics)
}
inputPath := filepath.Join(t.TempDir(), "future-transcript.md")
for _, namespace := range []string{"support", "review"} {
profile := "fast"
if namespace == "review" {
profile = "base"
}
outcome, output, diagnostics := executeNative(t, "promptkit", "requirements", "show", "answer", "--config="+config, "--prompt-namespace="+namespace, "--profile="+profile, "--profile-namespace="+namespace, "--format=json")
var report consumerRequirements
if err := json.Unmarshal([]byte(output), &report); err != nil {
t.Fatal(err)
}
if outcome.Status != 0 || !report.Complete || report.PromptID != namespace+"/answer" || report.ProfileID != namespace+"/"+profile {
t.Fatalf("requirements: %+v %+v %q", outcome, report, diagnostics)
}
if namespace == "support" && (len(report.Inputs) != 1 || report.Inputs[0].Name != "transcript" || report.Inputs[0].Kind != promptkit.RequirementRequired) {
t.Fatalf("required binding: %+v", report)
}
}
if _, err := os.Stat(inputPath); !os.IsNotExist(err) {
t.Fatalf("discovery created input artifact: %v", err)
}
writeNativeFile(t, inputPath, "generated collection transcript")
for _, namespace := range []string{"support", "review"} {
profile := "fast"
if namespace == "review" {
profile = "base"
}
flags := []string{"answer", "--config=" + config, "--prompt-namespace=" + namespace, "--profile=" + profile, "--profile-namespace=" + namespace, "--endpoint=" + provider.URL + "/v1", "--model=controlled", "--timeout-seconds=0", "--transient-retries=0", "--session-id=collection-session", "--format=json"}
if namespace == "support" {
flags = append(flags, "--input-file=transcript="+inputPath)
}
outcome, _, diagnostics := executeNative(t, append([]string{"promptkit", "requirements", "validate"}, flags...)...)
if outcome.Status != 0 {
t.Fatalf("validation: %+v %q", outcome, diagnostics)
}
outcome, output, diagnostics := executeNative(t, append([]string{"promptkit", "show", "prepared"}, flags...)...)
var prepared promptkit.PreparedRun
if err := json.Unmarshal([]byte(output), &prepared); err != nil {
t.Fatal(err)
}
if outcome.Status != 0 {
t.Fatalf("preview: %+v %q", outcome, diagnostics)
}
outcome, output, diagnostics = executeNative(t, append([]string{"run"}, flags...)...)
var result promptkit.RunResult
if err := json.Unmarshal([]byte(output), &result); err != nil {
t.Fatal(err)
}
if outcome.Status != 0 || result.PromptID != prepared.PromptID || result.SelectedProfileID != prepared.SelectedProfileID || result.RenderedPromptHash != prepared.RenderedPromptHash || !reflect.DeepEqual(result.InputHashes, prepared.InputHashes) || string(result.Artifact.Body) != "collection result" {
t.Fatalf("workflow selection/context diverged: prepared=%+v result=%+v outcome=%+v %q", prepared, result, outcome, diagnostics)
}
select {
case messages := <-requests:
if !reflect.DeepEqual(messages, prepared.Messages) {
t.Fatalf("provider saw different messages: %+v %+v", messages, prepared.Messages)
}
default:
t.Fatal("generation did not reach controlled provider")
}
}
for _, args := range [][]string{
{"config", "diff", "--scope=app", "--config=" + config, "--against-config=" + config, `--set=/promptkit/schema_dir="left"`, `--against-set=/promptkit/schema_dir="right"`, "--check"},
{"config", "diff", "--scope=promptkit", "--config=" + config, "--against-config=examples/config.yml", "--check"},
} {
outcome, output, diagnostics := executeNative(t, args...)
if outcome.Status != 3 || output == "" {
t.Fatalf("independent configuration sides: %+v %q %q", outcome, output, diagnostics)
}
}
}

View File

@@ -168,10 +168,13 @@ func TestRunDistinguishesCompletedInvalidContentAndOperationalFailure(t *testing
func TestRunUsageAndConfigurationFailures(t *testing.T) { func TestRunUsageAndConfigurationFailures(t *testing.T) {
t.Chdir(t.TempDir()) t.Chdir(t.TempDir())
for _, args := range [][]string{ for _, args := range [][]string{
{"render"}, {"inspect", "prompt"},
{"run"}, {"run", "one", "two"}, {"run", "one", "--prompt=two"}, {"run", "one", "--llm-prompt=two"}, {"run"}, {"run", "one", "two"}, {"run", "one", "--prompt=two"}, {"run", "one", "--llm-prompt=two"},
{"run", "one", "--backend=local"}, {"run", "one", "--out=result"}, {"run", "one", "--execution-report=result"}, {"run", "one", "--backend=local"}, {"run", "one", "--out=result"}, {"run", "one", "--execution-report=result"},
{"run", "one", "--llm-base-url=http://example.invalid"}, {"run", "one", "--timeout=1s"}, {"run", "one", "--llm-base-url=http://example.invalid"}, {"run", "one", "--timeout=1s"},
{"run", "one", "--prompt-id=two"}, {"run", "one", "--profile-id=local"}, {"run", "one", "--run-timeout=bad"}, {"run", "one", "--prompt-id=two"}, {"run", "one", "--profile-id=local"}, {"run", "one", "--run-timeout=bad"},
{"run", "one", "--prompt-version=1"}, {"run", "one", "--prompt-dir=prompts"}, {"run", "one", "--profile-dir=profiles"},
{"run", "one", "--config-overlay=local"}, {"run", "one", "--output=file"},
} { } {
outcome, _, _ := executeNative(t, args...) outcome, _, _ := executeNative(t, args...)
if outcome.Kind != commandkit.OutcomeUsage || outcome.Status != 2 { if outcome.Kind != commandkit.OutcomeUsage || outcome.Status != 2 {