197 lines
7.9 KiB
Go
197 lines
7.9 KiB
Go
package cli
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
|
|
"go.maximumdirect.net/promptkit"
|
|
)
|
|
|
|
func TestCollectionCheckoutDiscoveryAcrossInvocations(t *testing.T) {
|
|
fixture, err := filepath.Abs(filepath.Join("..", "..", "..", "examples", "discovery"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, tc := range []struct {
|
|
name string
|
|
first string
|
|
second string
|
|
}{
|
|
{name: "yml then yaml", first: "support-repo", second: "review-repo"},
|
|
{name: "yaml then yml", first: "review-repo", second: "support-repo"},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
root := t.TempDir()
|
|
framework, err := os.ReadFile(filepath.Join(fixture, "framework.yml"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
frameworkPath := filepath.Join(root, "framework.yml")
|
|
if err := os.WriteFile(frameworkPath, framework, 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
collections := filepath.Join(root, "collections")
|
|
if err := os.MkdirAll(collections, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
install := func(name string) {
|
|
t.Helper()
|
|
if err := os.CopyFS(filepath.Join(collections, name), os.DirFS(filepath.Join(fixture, "collections", name))); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
config := filepath.Join(root, "app.yml")
|
|
writeNativeFile(t, config, fmt.Sprintf("promptkit:\n configuration: %q\n", frameworkPath))
|
|
input := filepath.Join(root, "transcript.md")
|
|
if err := os.WriteFile(input, []byte("synthetic discovery transcript"), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
args := func(name string) []string {
|
|
if name == "support-repo" {
|
|
return []string{"support/answer", "--config", config, "--profile", "support/fast", "--input-file", "transcript=" + input}
|
|
}
|
|
return []string{"review/answer", "--config", config, "--profile", "review/base"}
|
|
}
|
|
check := func(name string) {
|
|
t.Helper()
|
|
command := append([]string{"promptkit", "show", "prepared"}, args(name)...)
|
|
outcome, out, stderr := executeNative(t, append(command, "--format=json")...)
|
|
if outcome.Status != 0 {
|
|
t.Fatalf("prepare %s: outcome=%+v stderr=%s", name, outcome, stderr)
|
|
}
|
|
var prepared promptkit.PreparedRun
|
|
if err := json.Unmarshal([]byte(out), &prepared); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if name == "support-repo" {
|
|
for _, want := range []string{"support/answer", "support/fast", "support-example", "Answer the support request", "synthetic discovery transcript"} {
|
|
if !strings.Contains(out, want) {
|
|
t.Fatalf("missing %q: %s", want, out)
|
|
}
|
|
}
|
|
if prepared.EffectiveModelParams.MaxTokens != 256 {
|
|
t.Fatalf("inherited profile setting missing: %+v", prepared)
|
|
}
|
|
} else if !strings.Contains(out, "review/answer") || !strings.Contains(out, "review-example") || !strings.Contains(out, "Review this material") {
|
|
t.Fatalf("unexpected review render: %s", out)
|
|
}
|
|
}
|
|
install(tc.first)
|
|
check(tc.first)
|
|
outcome, _, _ := executeNative(t, append([]string{"promptkit", "show", "prepared"}, args(tc.second)...)...)
|
|
if outcome.Status != 1 {
|
|
t.Fatal("uninstalled collection unexpectedly selected")
|
|
}
|
|
install(tc.second)
|
|
check(tc.first)
|
|
check(tc.second)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestMaintainedCollectionWorkflow(t *testing.T) {
|
|
root, err := filepath.Abs(filepath.Join("..", "..", ".."))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Chdir(root)
|
|
requests := make(chan []promptkit.RenderedMessage, 2)
|
|
provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
|
|
var body struct {
|
|
Messages []promptkit.RenderedMessage `json:"messages"`
|
|
}
|
|
if err := json.NewDecoder(request.Body).Decode(&body); err != nil {
|
|
t.Error(err)
|
|
}
|
|
select {
|
|
case requests <- body.Messages:
|
|
default:
|
|
t.Error("unexpected provider dispatch")
|
|
}
|
|
w.Header().Set("Content-Type", "application/json")
|
|
fmt.Fprint(w, `{"choices":[{"message":{"content":"collection result"}}]}`)
|
|
}))
|
|
defer provider.Close()
|
|
const config = "examples/config.discovery.yml"
|
|
outcome, inventory, diagnostics := executeNative(t, "promptkit", "list", "prompts", "--all-namespaces", "--config="+config, "--format=json")
|
|
if outcome.Status != 0 || !strings.Contains(inventory, "support/answer") || !strings.Contains(inventory, "support/summarize") || !strings.Contains(inventory, "review/answer") {
|
|
t.Fatalf("collection discovery: %+v %q %q", outcome, inventory, diagnostics)
|
|
}
|
|
inputPath := filepath.Join(t.TempDir(), "future-transcript.md")
|
|
for _, namespace := range []string{"support", "review"} {
|
|
profile := "fast"
|
|
if namespace == "review" {
|
|
profile = "base"
|
|
}
|
|
outcome, output, diagnostics := executeNative(t, "promptkit", "requirements", "show", "answer", "--config="+config, "--prompt-namespace="+namespace, "--profile="+profile, "--profile-namespace="+namespace, "--format=json")
|
|
var report consumerRequirements
|
|
if err := json.Unmarshal([]byte(output), &report); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if outcome.Status != 0 || !report.Complete || report.PromptID != namespace+"/answer" || report.ProfileID != namespace+"/"+profile {
|
|
t.Fatalf("requirements: %+v %+v %q", outcome, report, diagnostics)
|
|
}
|
|
if namespace == "support" && (len(report.Inputs) != 1 || report.Inputs[0].Name != "transcript" || report.Inputs[0].Kind != promptkit.RequirementRequired) {
|
|
t.Fatalf("required binding: %+v", report)
|
|
}
|
|
}
|
|
if _, err := os.Stat(inputPath); !os.IsNotExist(err) {
|
|
t.Fatalf("discovery created input artifact: %v", err)
|
|
}
|
|
writeNativeFile(t, inputPath, "generated collection transcript")
|
|
for _, namespace := range []string{"support", "review"} {
|
|
profile := "fast"
|
|
if namespace == "review" {
|
|
profile = "base"
|
|
}
|
|
flags := []string{"answer", "--config=" + config, "--prompt-namespace=" + namespace, "--profile=" + profile, "--profile-namespace=" + namespace, "--endpoint=" + provider.URL + "/v1", "--model=controlled", "--timeout-seconds=0", "--transient-retries=0", "--session-id=collection-session", "--format=json"}
|
|
if namespace == "support" {
|
|
flags = append(flags, "--input-file=transcript="+inputPath)
|
|
}
|
|
outcome, _, diagnostics := executeNative(t, append([]string{"promptkit", "requirements", "validate"}, flags...)...)
|
|
if outcome.Status != 0 {
|
|
t.Fatalf("validation: %+v %q", outcome, diagnostics)
|
|
}
|
|
outcome, output, diagnostics := executeNative(t, append([]string{"promptkit", "show", "prepared"}, flags...)...)
|
|
var prepared promptkit.PreparedRun
|
|
if err := json.Unmarshal([]byte(output), &prepared); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if outcome.Status != 0 {
|
|
t.Fatalf("preview: %+v %q", outcome, diagnostics)
|
|
}
|
|
outcome, output, diagnostics = executeNative(t, append([]string{"run"}, flags...)...)
|
|
var result promptkit.RunResult
|
|
if err := json.Unmarshal([]byte(output), &result); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if outcome.Status != 0 || result.PromptID != prepared.PromptID || result.SelectedProfileID != prepared.SelectedProfileID || result.RenderedPromptHash != prepared.RenderedPromptHash || !reflect.DeepEqual(result.InputHashes, prepared.InputHashes) || string(result.Artifact.Body) != "collection result" {
|
|
t.Fatalf("workflow selection/context diverged: prepared=%+v result=%+v outcome=%+v %q", prepared, result, outcome, diagnostics)
|
|
}
|
|
select {
|
|
case messages := <-requests:
|
|
if !reflect.DeepEqual(messages, prepared.Messages) {
|
|
t.Fatalf("provider saw different messages: %+v %+v", messages, prepared.Messages)
|
|
}
|
|
default:
|
|
t.Fatal("generation did not reach controlled provider")
|
|
}
|
|
}
|
|
for _, args := range [][]string{
|
|
{"config", "diff", "--scope=app", "--config=" + config, "--against-config=" + config, `--set=/promptkit/schema_dir="left"`, `--against-set=/promptkit/schema_dir="right"`, "--check"},
|
|
{"config", "diff", "--scope=promptkit", "--config=" + config, "--against-config=examples/config.yml", "--check"},
|
|
} {
|
|
outcome, output, diagnostics := executeNative(t, args...)
|
|
if outcome.Status != 3 || output == "" {
|
|
t.Fatalf("independent configuration sides: %+v %q %q", outcome, output, diagnostics)
|
|
}
|
|
}
|
|
}
|