Verify the complete native CLI cutover and record roadmap completion
This commit is contained in:
@@ -3,8 +3,20 @@ package main
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
|
"syscall"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestProcessIdentityAndNativeStatus(t *testing.T) {
|
func TestProcessIdentityAndNativeStatus(t *testing.T) {
|
||||||
@@ -23,3 +35,79 @@ func TestProcessIdentityAndNativeStatus(t *testing.T) {
|
|||||||
t.Fatalf("canceled status=%d diagnostics=%q", status, diagnostics.String())
|
t.Fatalf("canceled status=%d diagnostics=%q", status, diagnostics.String())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestProcessSignalCancellation(t *testing.T) {
|
||||||
|
if os.Getenv("SCRIPTORIUM_SIGNAL_CHILD") == "1" {
|
||||||
|
separator := slices.Index(os.Args, "--")
|
||||||
|
if separator < 0 {
|
||||||
|
os.Exit(2)
|
||||||
|
}
|
||||||
|
os.Exit(runWithSignals(os.Args[separator+1:], os.Stdout, os.Stderr))
|
||||||
|
}
|
||||||
|
if runtime.GOOS == "windows" {
|
||||||
|
t.Skip("process interrupt/SIGTERM integration is Unix-specific")
|
||||||
|
}
|
||||||
|
for _, signal := range []os.Signal{os.Interrupt, syscall.SIGTERM} {
|
||||||
|
t.Run(signal.String(), func(t *testing.T) {
|
||||||
|
started, stopped, release := make(chan struct{}), make(chan struct{}), make(chan struct{})
|
||||||
|
provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
_, _ = io.Copy(io.Discard, r.Body)
|
||||||
|
close(started)
|
||||||
|
select {
|
||||||
|
case <-r.Context().Done():
|
||||||
|
close(stopped)
|
||||||
|
case <-release:
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
defer provider.Close()
|
||||||
|
defer close(release)
|
||||||
|
root := t.TempDir()
|
||||||
|
framework := filepath.Join(root, "framework.yml")
|
||||||
|
content := fmt.Sprintf("profiles:\n - id: local\n endpoint: %q\n model: controlled\nprompts:\n - id: selected\n default_profile: local\n messages:\n - role: user\n content: Signal test\n output:\n format: text\n validation_mode: none\n", provider.URL+"/v1")
|
||||||
|
if err := os.WriteFile(framework, []byte(content), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
config := filepath.Join(root, "app.yml")
|
||||||
|
if err := os.WriteFile(config, []byte(fmt.Sprintf("promptkit:\n configuration: %q\ntransient_retries: 0\n", framework)), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
executable, err := os.Executable()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
command := exec.CommandContext(ctx, executable, "-test.run=^TestProcessSignalCancellation$", "--", "run", "selected", "--config="+config, "--timeout-seconds=0")
|
||||||
|
command.Env = append(os.Environ(), "SCRIPTORIUM_SIGNAL_CHILD=1")
|
||||||
|
var output, diagnostics bytes.Buffer
|
||||||
|
command.Stdout, command.Stderr = &output, &diagnostics
|
||||||
|
if err := command.Start(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
cancel()
|
||||||
|
if command.ProcessState == nil {
|
||||||
|
_ = command.Wait()
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-started:
|
||||||
|
case <-ctx.Done():
|
||||||
|
t.Fatal("child did not reach controlled provider")
|
||||||
|
}
|
||||||
|
if err := command.Process.Signal(signal); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
err = command.Wait()
|
||||||
|
var exit *exec.ExitError
|
||||||
|
if !errors.As(err, &exit) || exit.ExitCode() != 1 || output.Len() != 0 || diagnostics.Len() == 0 {
|
||||||
|
t.Fatalf("signal outcome: %v stdout=%q stderr=%q", err, output.String(), diagnostics.String())
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-stopped:
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("signal did not cancel provider request")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -2,8 +2,9 @@
|
|||||||
|
|
||||||
## Status And Intent
|
## Status And Intent
|
||||||
|
|
||||||
Initial roadmap, prepared 2026-11-19. The product direction is agreed;
|
Implementation and assembled acceptance verification are complete; see the
|
||||||
implementation is underway. This document owns the target and product intent,
|
[implementation plan](implementation.md) for evidence and selected releases.
|
||||||
|
Initial roadmap prepared 2026-11-19. This document owns the target and product intent,
|
||||||
incorporating the decisions in the [migration notepad](commandkit-notepad.md).
|
incorporating the decisions in the [migration notepad](commandkit-notepad.md).
|
||||||
The notepad remains the discussion
|
The notepad remains the discussion
|
||||||
record. The [Narratio proposal](narratio-feature-request.md) supplies consumer
|
record. The [Narratio proposal](narratio-feature-request.md) supplies consumer
|
||||||
@@ -26,13 +27,13 @@ report shapes, and exit conventions are not compatibility constraints. Release
|
|||||||
migration guidance replaces a compatibility layer. Implementation may proceed
|
migration guidance replaces a compatibility layer. Implementation may proceed
|
||||||
in reviewable increments; this does not require shipping both interfaces.
|
in reviewable increments; this does not require shipping both interfaces.
|
||||||
|
|
||||||
## Current Baseline And Upstream References
|
## Planning Baseline And Upstream References
|
||||||
|
|
||||||
Reviewed Scriptorium baseline: `5f32a4f2adb2d382a6cfce3a787ada4576be7705`.
|
Reviewed Scriptorium baseline: `5f32a4f2adb2d382a6cfce3a787ada4576be7705`.
|
||||||
The module uses Go 1.27.0, PromptKit v0.23.3, and ConfigKit v1.2.1. CommandKit
|
At that baseline the module used Go 1.27.0, PromptKit v0.23.3, and ConfigKit
|
||||||
is not yet a dependency.
|
v1.2.1, without CommandKit. The following table records the starting point.
|
||||||
|
|
||||||
| Current implementation | Migration work |
|
| Baseline implementation | Migration work |
|
||||||
| --- | --- |
|
| --- | --- |
|
||||||
| `internal/adapter/cli` dispatches and parses commands, resolves configuration, builds engines and requests, writes diagnostics, and returns integer statuses. | Replace dispatch and shared CLI mechanics with CommandKit; retain application loading, request mapping, and effects through its extension points. |
|
| `internal/adapter/cli` dispatches and parses commands, resolves configuration, builds engines and requests, writes diagnostics, and returns integer statuses. | Replace dispatch and shared CLI mechanics with CommandKit; retain application loading, request mapping, and effects through its extension points. |
|
||||||
| `run` uses flag-based prompt selection and writes generated content; `--execution-report` writes an observer-derived Scriptorium report. | Adopt positional prompt selection, native request flags, and native result reporting. |
|
| `run` uses flag-based prompt selection and writes generated content; `--execution-report` writes an observer-derived Scriptorium report. | Adopt positional prompt selection, native request flags, and native result reporting. |
|
||||||
@@ -44,8 +45,8 @@ is not yet a dependency.
|
|||||||
|
|
||||||
The [current CLI](../cli.md), [configuration contract](../config.md),
|
The [current CLI](../cli.md), [configuration contract](../config.md),
|
||||||
[adapter internals](../internal/adapters.md), and
|
[adapter internals](../internal/adapters.md), and
|
||||||
[subprocess contract](../integrations/subprocess.md) describe the starting
|
[subprocess contract](../integrations/subprocess.md) now describe the implemented
|
||||||
behavior, not a compatibility checklist for this migration.
|
native interface, not the historical starting point or a compatibility layer.
|
||||||
|
|
||||||
The reviewed upstream combination is CommandKit v0.7.0, ckconfig v0.9.0,
|
The reviewed upstream combination is CommandKit v0.7.0, ckconfig v0.9.0,
|
||||||
and ckprompt v0.10.0, documented at revision
|
and ckprompt v0.10.0, documented at revision
|
||||||
|
|||||||
@@ -723,6 +723,44 @@ current documentation links resolve. Migration examples match the built CLI.
|
|||||||
|
|
||||||
## Stage 9: Verify The Complete Native Cutover
|
## Stage 9: Verify The Complete Native Cutover
|
||||||
|
|
||||||
|
**Status: Complete.** Verified the assembled native CLI on Go 1.27.4 with
|
||||||
|
`go test ./... -count=1`, `go test -race ./... -count=1`, `go vet ./...`,
|
||||||
|
and `go build ./cmd/scriptorium`. Formatting, diff whitespace, module tidiness,
|
||||||
|
shell syntax, maintained examples, and local Markdown links/anchors pass.
|
||||||
|
The maintained discovery configuration now has an end-to-end controlled-provider
|
||||||
|
test covering support/review inventories, requirements before artifacts exist,
|
||||||
|
concrete validation, preview/run context agreement, and independent native app
|
||||||
|
and framework comparisons. Existing checkout tests verify both optional entry
|
||||||
|
spellings and discovery on later invocations. Additional checks cover rejected
|
||||||
|
legacy surfaces, partial output, diagnostic delivery failure, and real child
|
||||||
|
process interrupt/SIGTERM cancellation reaching a provider request.
|
||||||
|
|
||||||
|
Built Linux amd64 and arm64 with Go 1.27.4, `CGO_ENABLED=0`, `-trimpath`,
|
||||||
|
and `-s -w` plus version/revision linker metadata. The amd64 binary reports
|
||||||
|
the injected `verification` identity and full source revision
|
||||||
|
`bd7136ceebbdc7498e9a768ce8abbe0cbb00318f`; arm64 was cross-built, not executed.
|
||||||
|
The binary also exposes help, native scope discovery, and Bash/Zsh/Fish
|
||||||
|
completion without loading configuration or contacting a provider.
|
||||||
|
|
||||||
|
Selected releases remain CommandKit v0.7.0
|
||||||
|
(`b57bfe176cff05a892e6cb07ebb5f350239e55d4`), ckconfig v0.9.0
|
||||||
|
(`9962368f0c1ea1f6ad010638e94621091105d745`), ckprompt v0.10.0
|
||||||
|
(`c9a56e1b15f2169881a91fca4e1f4a1c475e057a`), ConfigKit v1.2.1
|
||||||
|
(`51ef3144158db1a09fabdbd1c7ae59908c84923b`), and PromptKit v0.23.3
|
||||||
|
(`804a4d32b23aa2e7973400bb0a322a7016574c9f`). Module and linked-runtime
|
||||||
|
inspection confirms no replacements, workspace, vendoring, or linked maintainer
|
||||||
|
tools; attribution follows the linked dependencies, including conditional
|
||||||
|
Windows mousetrap. Public native adapters, request bindings, reporters,
|
||||||
|
`RenderReport`, and typed outcomes remain the selected integration APIs.
|
||||||
|
|
||||||
|
The final implementation audit against this plan and ADRs 0004–0006 found no
|
||||||
|
remaining required work. HTTP/report implementation and accepted ADRs remain
|
||||||
|
unchanged; no compatibility surface, post-resolution config rewriting, extra
|
||||||
|
readiness gate, workflow state, snapshot promise, or transactional delivery was
|
||||||
|
added. Embedded framework configuration, dynamic-resource completion, the
|
||||||
|
optional format accessor, and HTTP redesign remain outside this migration.
|
||||||
|
No release was published and no commits were pushed.
|
||||||
|
|
||||||
1. Finish any deferred stage checks against the complete application. Use real
|
1. Finish any deferred stage checks against the complete application. Use real
|
||||||
ConfigKit, PromptKit, and CommandKit with deterministic provider substitutes
|
ConfigKit, PromptKit, and CommandKit with deterministic provider substitutes
|
||||||
at the external boundary. Tests remain offline and independent of real keys.
|
at the external boundary. Tests remain offline and independent of real keys.
|
||||||
|
|||||||
@@ -1,7 +1,8 @@
|
|||||||
# Prompt input requirements for Narratio
|
# Prompt input requirements for Narratio
|
||||||
|
|
||||||
Status: the native interface and focused consumer scenarios are implemented.
|
Status: the native interface and consumer acceptance scenarios are implemented
|
||||||
The [implementation plan](implementation.md) tracks final assembled verification.
|
and verified, including the assembled collection workflow. The
|
||||||
|
[implementation plan](implementation.md) records the verification evidence.
|
||||||
This proposal records Narratio's consumer needs and their mapping to the native
|
This proposal records Narratio's consumer needs and their mapping to the native
|
||||||
interface; the [CLI contract](../cli.md) owns current behavior.
|
interface; the [CLI contract](../cli.md) owns current behavior.
|
||||||
The [CommandKit notepad](commandkit-notepad.md) records the migration decisions.
|
The [CommandKit notepad](commandkit-notepad.md) records the migration decisions.
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"os"
|
"os"
|
||||||
@@ -129,6 +130,41 @@ type failedDeliveryWriter struct{ cause error }
|
|||||||
|
|
||||||
func (writer failedDeliveryWriter) Write([]byte) (int, error) { return 0, writer.cause }
|
func (writer failedDeliveryWriter) Write([]byte) (int, error) { return 0, writer.cause }
|
||||||
|
|
||||||
|
type partialDeliveryWriter struct {
|
||||||
|
bytes.Buffer
|
||||||
|
cause error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (writer *partialDeliveryWriter) Write(value []byte) (int, error) {
|
||||||
|
count := min(2, len(value))
|
||||||
|
_, _ = writer.Buffer.Write(value[:count])
|
||||||
|
return count, writer.cause
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPartialOutputAndDiagnosticFailureUseNativeFinalization(t *testing.T) {
|
||||||
|
t.Chdir(t.TempDir())
|
||||||
|
failure := errors.New("presentation failure")
|
||||||
|
app, err := commandkit.New(commandkit.Identity{Name: "delivery"}, commandkit.NewCommand(commandkit.CommandSpec[struct{}]{Name: "run", Run: func(inv commandkit.Invocation, _ struct{}) commandkit.Outcome {
|
||||||
|
return deliverRunResult(inv, runInput{ReportPath: commandkit.Supplied[string]{Set: true, Value: "report"}}, &promptkit.RunResult{Artifact: promptkit.Artifact{Body: []byte("artifact")}})
|
||||||
|
}}))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
output := &partialDeliveryWriter{cause: failure}
|
||||||
|
var diagnostics bytes.Buffer
|
||||||
|
outcome := app.Execute(context.Background(), []string{"run"}, commandkit.Streams{Out: output, Err: &diagnostics})
|
||||||
|
if outcome.Status != 1 || output.String() != "ar" || !errors.Is(outcome.Presentation.Output.WriteError, failure) {
|
||||||
|
t.Fatalf("partial output was lost: %+v %q", outcome, output.String())
|
||||||
|
}
|
||||||
|
if _, err := os.Stat("report"); err != nil {
|
||||||
|
t.Fatal("report was not attempted", err)
|
||||||
|
}
|
||||||
|
outcome = Execute(context.Background(), commandkit.Identity{Name: "scriptorium"}, []string{"run"}, io.Discard, failedDeliveryWriter{failure})
|
||||||
|
if outcome.Status != 1 || outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Presentation.Diagnostics.WriteError, failure) {
|
||||||
|
t.Fatalf("usage diagnostic failure not finalized: %+v", outcome)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestNativeDeliveryAttemptsReportAfterStdoutFailure(t *testing.T) {
|
func TestNativeDeliveryAttemptsReportAfterStdoutFailure(t *testing.T) {
|
||||||
t.Chdir(t.TempDir())
|
t.Chdir(t.TempDir())
|
||||||
result := &promptkit.RunResult{RunID: "known-result", Artifact: promptkit.Artifact{Body: []byte("body")}}
|
result := &promptkit.RunResult{RunID: "known-result", Artifact: promptkit.Artifact{Body: []byte("body")}}
|
||||||
|
|||||||
@@ -3,11 +3,15 @@ package cli
|
|||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"go.maximumdirect.net/promptkit"
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestCollectionCheckoutDiscoveryAcrossInvocations(t *testing.T) {
|
func TestCollectionCheckoutDiscoveryAcrossInvocations(t *testing.T) {
|
||||||
@@ -91,3 +95,102 @@ func TestCollectionCheckoutDiscoveryAcrossInvocations(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMaintainedCollectionWorkflow(t *testing.T) {
|
||||||
|
root, err := filepath.Abs(filepath.Join("..", "..", ".."))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Chdir(root)
|
||||||
|
requests := make(chan []promptkit.RenderedMessage, 2)
|
||||||
|
provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
|
||||||
|
var body struct {
|
||||||
|
Messages []promptkit.RenderedMessage `json:"messages"`
|
||||||
|
}
|
||||||
|
if err := json.NewDecoder(request.Body).Decode(&body); err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case requests <- body.Messages:
|
||||||
|
default:
|
||||||
|
t.Error("unexpected provider dispatch")
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
fmt.Fprint(w, `{"choices":[{"message":{"content":"collection result"}}]}`)
|
||||||
|
}))
|
||||||
|
defer provider.Close()
|
||||||
|
const config = "examples/config.discovery.yml"
|
||||||
|
outcome, inventory, diagnostics := executeNative(t, "promptkit", "list", "prompts", "--all-namespaces", "--config="+config, "--format=json")
|
||||||
|
if outcome.Status != 0 || !strings.Contains(inventory, "support/answer") || !strings.Contains(inventory, "support/summarize") || !strings.Contains(inventory, "review/answer") {
|
||||||
|
t.Fatalf("collection discovery: %+v %q %q", outcome, inventory, diagnostics)
|
||||||
|
}
|
||||||
|
inputPath := filepath.Join(t.TempDir(), "future-transcript.md")
|
||||||
|
for _, namespace := range []string{"support", "review"} {
|
||||||
|
profile := "fast"
|
||||||
|
if namespace == "review" {
|
||||||
|
profile = "base"
|
||||||
|
}
|
||||||
|
outcome, output, diagnostics := executeNative(t, "promptkit", "requirements", "show", "answer", "--config="+config, "--prompt-namespace="+namespace, "--profile="+profile, "--profile-namespace="+namespace, "--format=json")
|
||||||
|
var report consumerRequirements
|
||||||
|
if err := json.Unmarshal([]byte(output), &report); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if outcome.Status != 0 || !report.Complete || report.PromptID != namespace+"/answer" || report.ProfileID != namespace+"/"+profile {
|
||||||
|
t.Fatalf("requirements: %+v %+v %q", outcome, report, diagnostics)
|
||||||
|
}
|
||||||
|
if namespace == "support" && (len(report.Inputs) != 1 || report.Inputs[0].Name != "transcript" || report.Inputs[0].Kind != promptkit.RequirementRequired) {
|
||||||
|
t.Fatalf("required binding: %+v", report)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(inputPath); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("discovery created input artifact: %v", err)
|
||||||
|
}
|
||||||
|
writeNativeFile(t, inputPath, "generated collection transcript")
|
||||||
|
for _, namespace := range []string{"support", "review"} {
|
||||||
|
profile := "fast"
|
||||||
|
if namespace == "review" {
|
||||||
|
profile = "base"
|
||||||
|
}
|
||||||
|
flags := []string{"answer", "--config=" + config, "--prompt-namespace=" + namespace, "--profile=" + profile, "--profile-namespace=" + namespace, "--endpoint=" + provider.URL + "/v1", "--model=controlled", "--timeout-seconds=0", "--transient-retries=0", "--session-id=collection-session", "--format=json"}
|
||||||
|
if namespace == "support" {
|
||||||
|
flags = append(flags, "--input-file=transcript="+inputPath)
|
||||||
|
}
|
||||||
|
outcome, _, diagnostics := executeNative(t, append([]string{"promptkit", "requirements", "validate"}, flags...)...)
|
||||||
|
if outcome.Status != 0 {
|
||||||
|
t.Fatalf("validation: %+v %q", outcome, diagnostics)
|
||||||
|
}
|
||||||
|
outcome, output, diagnostics := executeNative(t, append([]string{"promptkit", "show", "prepared"}, flags...)...)
|
||||||
|
var prepared promptkit.PreparedRun
|
||||||
|
if err := json.Unmarshal([]byte(output), &prepared); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if outcome.Status != 0 {
|
||||||
|
t.Fatalf("preview: %+v %q", outcome, diagnostics)
|
||||||
|
}
|
||||||
|
outcome, output, diagnostics = executeNative(t, append([]string{"run"}, flags...)...)
|
||||||
|
var result promptkit.RunResult
|
||||||
|
if err := json.Unmarshal([]byte(output), &result); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if outcome.Status != 0 || result.PromptID != prepared.PromptID || result.SelectedProfileID != prepared.SelectedProfileID || result.RenderedPromptHash != prepared.RenderedPromptHash || !reflect.DeepEqual(result.InputHashes, prepared.InputHashes) || string(result.Artifact.Body) != "collection result" {
|
||||||
|
t.Fatalf("workflow selection/context diverged: prepared=%+v result=%+v outcome=%+v %q", prepared, result, outcome, diagnostics)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case messages := <-requests:
|
||||||
|
if !reflect.DeepEqual(messages, prepared.Messages) {
|
||||||
|
t.Fatalf("provider saw different messages: %+v %+v", messages, prepared.Messages)
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
t.Fatal("generation did not reach controlled provider")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, args := range [][]string{
|
||||||
|
{"config", "diff", "--scope=app", "--config=" + config, "--against-config=" + config, `--set=/promptkit/schema_dir="left"`, `--against-set=/promptkit/schema_dir="right"`, "--check"},
|
||||||
|
{"config", "diff", "--scope=promptkit", "--config=" + config, "--against-config=examples/config.yml", "--check"},
|
||||||
|
} {
|
||||||
|
outcome, output, diagnostics := executeNative(t, args...)
|
||||||
|
if outcome.Status != 3 || output == "" {
|
||||||
|
t.Fatalf("independent configuration sides: %+v %q %q", outcome, output, diagnostics)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -168,10 +168,13 @@ func TestRunDistinguishesCompletedInvalidContentAndOperationalFailure(t *testing
|
|||||||
func TestRunUsageAndConfigurationFailures(t *testing.T) {
|
func TestRunUsageAndConfigurationFailures(t *testing.T) {
|
||||||
t.Chdir(t.TempDir())
|
t.Chdir(t.TempDir())
|
||||||
for _, args := range [][]string{
|
for _, args := range [][]string{
|
||||||
|
{"render"}, {"inspect", "prompt"},
|
||||||
{"run"}, {"run", "one", "two"}, {"run", "one", "--prompt=two"}, {"run", "one", "--llm-prompt=two"},
|
{"run"}, {"run", "one", "two"}, {"run", "one", "--prompt=two"}, {"run", "one", "--llm-prompt=two"},
|
||||||
{"run", "one", "--backend=local"}, {"run", "one", "--out=result"}, {"run", "one", "--execution-report=result"},
|
{"run", "one", "--backend=local"}, {"run", "one", "--out=result"}, {"run", "one", "--execution-report=result"},
|
||||||
{"run", "one", "--llm-base-url=http://example.invalid"}, {"run", "one", "--timeout=1s"},
|
{"run", "one", "--llm-base-url=http://example.invalid"}, {"run", "one", "--timeout=1s"},
|
||||||
{"run", "one", "--prompt-id=two"}, {"run", "one", "--profile-id=local"}, {"run", "one", "--run-timeout=bad"},
|
{"run", "one", "--prompt-id=two"}, {"run", "one", "--profile-id=local"}, {"run", "one", "--run-timeout=bad"},
|
||||||
|
{"run", "one", "--prompt-version=1"}, {"run", "one", "--prompt-dir=prompts"}, {"run", "one", "--profile-dir=profiles"},
|
||||||
|
{"run", "one", "--config-overlay=local"}, {"run", "one", "--output=file"},
|
||||||
} {
|
} {
|
||||||
outcome, _, _ := executeNative(t, args...)
|
outcome, _, _ := executeNative(t, args...)
|
||||||
if outcome.Kind != commandkit.OutcomeUsage || outcome.Status != 2 {
|
if outcome.Kind != commandkit.OutcomeUsage || outcome.Status != 2 {
|
||||||
|
|||||||
Reference in New Issue
Block a user