package cli import ( "encoding/json" "fmt" "net/http" "net/http/httptest" "os" "path/filepath" "reflect" "strings" "testing" "go.maximumdirect.net/promptkit" ) func TestCollectionCheckoutDiscoveryAcrossInvocations(t *testing.T) { fixture, err := filepath.Abs(filepath.Join("..", "..", "..", "examples", "discovery")) if err != nil { t.Fatal(err) } for _, tc := range []struct { name string first string second string }{ {name: "yml then yaml", first: "support-repo", second: "review-repo"}, {name: "yaml then yml", first: "review-repo", second: "support-repo"}, } { t.Run(tc.name, func(t *testing.T) { root := t.TempDir() framework, err := os.ReadFile(filepath.Join(fixture, "framework.yml")) if err != nil { t.Fatal(err) } frameworkPath := filepath.Join(root, "framework.yml") if err := os.WriteFile(frameworkPath, framework, 0o644); err != nil { t.Fatal(err) } collections := filepath.Join(root, "collections") if err := os.MkdirAll(collections, 0o755); err != nil { t.Fatal(err) } install := func(name string) { t.Helper() if err := os.CopyFS(filepath.Join(collections, name), os.DirFS(filepath.Join(fixture, "collections", name))); err != nil { t.Fatal(err) } } config := filepath.Join(root, "app.yml") writeNativeFile(t, config, fmt.Sprintf("promptkit:\n configuration: %q\n", frameworkPath)) input := filepath.Join(root, "transcript.md") if err := os.WriteFile(input, []byte("synthetic discovery transcript"), 0o644); err != nil { t.Fatal(err) } args := func(name string) []string { if name == "support-repo" { return []string{"support/answer", "--config", config, "--profile", "support/fast", "--input-file", "transcript=" + input} } return []string{"review/answer", "--config", config, "--profile", "review/base"} } check := func(name string) { t.Helper() command := append([]string{"promptkit", "show", "prepared"}, args(name)...) outcome, out, stderr := executeNative(t, append(command, "--format=json")...) if outcome.Status != 0 { t.Fatalf("prepare %s: outcome=%+v stderr=%s", name, outcome, stderr) } var prepared promptkit.PreparedRun if err := json.Unmarshal([]byte(out), &prepared); err != nil { t.Fatal(err) } if name == "support-repo" { for _, want := range []string{"support/answer", "support/fast", "support-example", "Answer the support request", "synthetic discovery transcript"} { if !strings.Contains(out, want) { t.Fatalf("missing %q: %s", want, out) } } if prepared.EffectiveModelParams.MaxTokens != 256 { t.Fatalf("inherited profile setting missing: %+v", prepared) } } else if !strings.Contains(out, "review/answer") || !strings.Contains(out, "review-example") || !strings.Contains(out, "Review this material") { t.Fatalf("unexpected review render: %s", out) } } install(tc.first) check(tc.first) outcome, _, _ := executeNative(t, append([]string{"promptkit", "show", "prepared"}, args(tc.second)...)...) if outcome.Status != 1 { t.Fatal("uninstalled collection unexpectedly selected") } install(tc.second) check(tc.first) check(tc.second) }) } } func TestMaintainedCollectionWorkflow(t *testing.T) { root, err := filepath.Abs(filepath.Join("..", "..", "..")) if err != nil { t.Fatal(err) } t.Chdir(root) requests := make(chan []promptkit.RenderedMessage, 2) provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { var body struct { Messages []promptkit.RenderedMessage `json:"messages"` } if err := json.NewDecoder(request.Body).Decode(&body); err != nil { t.Error(err) } select { case requests <- body.Messages: default: t.Error("unexpected provider dispatch") } w.Header().Set("Content-Type", "application/json") fmt.Fprint(w, `{"choices":[{"message":{"content":"collection result"}}]}`) })) defer provider.Close() const config = "examples/config.discovery.yml" outcome, inventory, diagnostics := executeNative(t, "promptkit", "list", "prompts", "--all-namespaces", "--config="+config, "--format=json") if outcome.Status != 0 || !strings.Contains(inventory, "support/answer") || !strings.Contains(inventory, "support/summarize") || !strings.Contains(inventory, "review/answer") { t.Fatalf("collection discovery: %+v %q %q", outcome, inventory, diagnostics) } inputPath := filepath.Join(t.TempDir(), "future-transcript.md") for _, namespace := range []string{"support", "review"} { profile := "fast" if namespace == "review" { profile = "base" } outcome, output, diagnostics := executeNative(t, "promptkit", "requirements", "show", "answer", "--config="+config, "--prompt-namespace="+namespace, "--profile="+profile, "--profile-namespace="+namespace, "--format=json") var report consumerRequirements if err := json.Unmarshal([]byte(output), &report); err != nil { t.Fatal(err) } if outcome.Status != 0 || !report.Complete || report.PromptID != namespace+"/answer" || report.ProfileID != namespace+"/"+profile { t.Fatalf("requirements: %+v %+v %q", outcome, report, diagnostics) } if namespace == "support" && (len(report.Inputs) != 1 || report.Inputs[0].Name != "transcript" || report.Inputs[0].Kind != promptkit.RequirementRequired) { t.Fatalf("required binding: %+v", report) } } if _, err := os.Stat(inputPath); !os.IsNotExist(err) { t.Fatalf("discovery created input artifact: %v", err) } writeNativeFile(t, inputPath, "generated collection transcript") for _, namespace := range []string{"support", "review"} { profile := "fast" if namespace == "review" { profile = "base" } flags := []string{"answer", "--config=" + config, "--prompt-namespace=" + namespace, "--profile=" + profile, "--profile-namespace=" + namespace, "--endpoint=" + provider.URL + "/v1", "--model=controlled", "--timeout-seconds=0", "--transient-retries=0", "--session-id=collection-session", "--format=json"} if namespace == "support" { flags = append(flags, "--input-file=transcript="+inputPath) } outcome, _, diagnostics := executeNative(t, append([]string{"promptkit", "requirements", "validate"}, flags...)...) if outcome.Status != 0 { t.Fatalf("validation: %+v %q", outcome, diagnostics) } outcome, output, diagnostics := executeNative(t, append([]string{"promptkit", "show", "prepared"}, flags...)...) var prepared promptkit.PreparedRun if err := json.Unmarshal([]byte(output), &prepared); err != nil { t.Fatal(err) } if outcome.Status != 0 { t.Fatalf("preview: %+v %q", outcome, diagnostics) } outcome, output, diagnostics = executeNative(t, append([]string{"run"}, flags...)...) var result promptkit.RunResult if err := json.Unmarshal([]byte(output), &result); err != nil { t.Fatal(err) } if outcome.Status != 0 || result.PromptID != prepared.PromptID || result.SelectedProfileID != prepared.SelectedProfileID || result.RenderedPromptHash != prepared.RenderedPromptHash || !reflect.DeepEqual(result.InputHashes, prepared.InputHashes) || string(result.Artifact.Body) != "collection result" { t.Fatalf("workflow selection/context diverged: prepared=%+v result=%+v outcome=%+v %q", prepared, result, outcome, diagnostics) } select { case messages := <-requests: if !reflect.DeepEqual(messages, prepared.Messages) { t.Fatalf("provider saw different messages: %+v %+v", messages, prepared.Messages) } default: t.Fatal("generation did not reach controlled provider") } } for _, args := range [][]string{ {"config", "diff", "--scope=app", "--config=" + config, "--against-config=" + config, `--set=/promptkit/schema_dir="left"`, `--against-set=/promptkit/schema_dir="right"`, "--check"}, {"config", "diff", "--scope=promptkit", "--config=" + config, "--against-config=examples/config.yml", "--check"}, } { outcome, output, diagnostics := executeNative(t, args...) if outcome.Status != 3 || output == "" { t.Fatalf("independent configuration sides: %+v %q %q", outcome, output, diagnostics) } } }