Retire the legacy CLI and document native operator workflows
This commit is contained in:
@@ -1,39 +0,0 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config"
|
||||
"go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
func loadApplicationSettings(configPath, overlay string, explicit bool, overrides []configkit.Override) (appconfig.AppSettings, []configkit.Diagnostic, error) {
|
||||
result, err := appconfig.Load(appconfig.LoadOptions{
|
||||
Path: configPath, Overlay: overlay, Explicit: explicit, Overrides: overrides,
|
||||
})
|
||||
if err != nil {
|
||||
return appconfig.AppSettings{}, nil, err
|
||||
}
|
||||
return appconfig.Settings(result.Config), result.Diagnostics(), nil
|
||||
}
|
||||
|
||||
func writeApplicationError(output io.Writer, label string, err error) {
|
||||
var loadError *configkit.LoadError
|
||||
if errors.As(err, &loadError) {
|
||||
_, _ = fmt.Fprintf(output, "%s error:\n", label)
|
||||
if writeErr := configkit.WriteDiagnostics(output, loadError.Diagnostics()); writeErr != nil {
|
||||
_, _ = fmt.Fprintf(output, "%s diagnostic write error: %v\n", label, writeErr)
|
||||
}
|
||||
return
|
||||
}
|
||||
_, _ = fmt.Fprintf(output, "%s error: %v\n", label, err)
|
||||
}
|
||||
|
||||
func writeApplicationWarnings(output io.Writer, diagnostics []configkit.Diagnostic) error {
|
||||
if len(diagnostics) == 0 {
|
||||
return nil
|
||||
}
|
||||
return configkit.WriteDiagnostics(output, diagnostics)
|
||||
}
|
||||
@@ -1,101 +0,0 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config"
|
||||
"go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
type configurationSource struct {
|
||||
ID string `json:"id"`
|
||||
Label string `json:"label"`
|
||||
Layer configkit.Layer `json:"layer"`
|
||||
Path string `json:"path,omitempty"`
|
||||
}
|
||||
|
||||
func configCommand(args []string, stdout, stderr io.Writer) int {
|
||||
if len(args) == 0 {
|
||||
fmt.Fprintln(stderr, "config parse error: configuration mode is required")
|
||||
return ExitRuntimeError
|
||||
}
|
||||
mode := args[0]
|
||||
if mode != "validate" && mode != "show" && mode != "sources" && mode != "explain" {
|
||||
fmt.Fprintln(stderr, "config parse error: unknown configuration mode")
|
||||
return ExitRuntimeError
|
||||
}
|
||||
|
||||
fs := flag.NewFlagSet("config "+mode, flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
var path, overlay, pointer string
|
||||
registerConfigSelectionFlags(fs, &path, &overlay)
|
||||
registerConfigurationOverrides(fs)
|
||||
if mode == "explain" {
|
||||
fs.StringVar(&pointer, "path", "", "configuration path as a JSON Pointer")
|
||||
}
|
||||
if err := fs.Parse(args[1:]); err != nil {
|
||||
writeApplicationError(stderr, "config parse", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
if fs.NArg() != 0 {
|
||||
fmt.Fprintln(stderr, "config parse error: positional arguments are not accepted")
|
||||
return ExitRuntimeError
|
||||
}
|
||||
var explanationPath configkit.Path
|
||||
if mode == "explain" {
|
||||
if !flagWasSet(fs, "path") {
|
||||
fmt.Fprintln(stderr, "config parse error: --path is required")
|
||||
return ExitRuntimeError
|
||||
}
|
||||
var err error
|
||||
explanationPath, err = configkit.ParsePath(pointer)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "config parse error: --path must be a JSON Pointer")
|
||||
return ExitRuntimeError
|
||||
}
|
||||
}
|
||||
|
||||
result, err := appconfig.Load(appconfig.LoadOptions{
|
||||
Path: path, Explicit: flagWasSet(fs, "config"), Overlay: overlay,
|
||||
Overrides: applicationOverrides(fs),
|
||||
})
|
||||
if err != nil {
|
||||
writeApplicationError(stderr, "config", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
if err := writeApplicationWarnings(stderr, result.Diagnostics()); err != nil {
|
||||
return ExitRuntimeError
|
||||
}
|
||||
switch mode {
|
||||
case "validate":
|
||||
return ExitOK
|
||||
case "show":
|
||||
err = result.WriteEffective(stdout)
|
||||
case "sources":
|
||||
sources := make([]configurationSource, 0, len(result.Sources()))
|
||||
for _, source := range result.Sources() {
|
||||
sources = append(sources, configurationSource{
|
||||
ID: source.ID, Label: source.Label, Layer: source.Layer, Path: source.Path,
|
||||
})
|
||||
}
|
||||
err = json.NewEncoder(stdout).Encode(sources)
|
||||
case "explain":
|
||||
err = result.WriteExplanation(stdout, explanationPath)
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "config output error: %v\n", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
return ExitOK
|
||||
}
|
||||
|
||||
func registerConfigurationOverrides(fs *flag.FlagSet) {
|
||||
fs.String("addr", "", "HTTP listen address override")
|
||||
fs.String("artifact-root", "", "HTTP artifact root override")
|
||||
fs.Int64("max-request-bytes", 0, "HTTP request byte limit override")
|
||||
fs.Int64("max-artifact-bytes", 0, "HTTP artifact byte limit override")
|
||||
fs.Int64("max-response-bytes", 0, "HTTP response byte limit override")
|
||||
}
|
||||
@@ -1,226 +0,0 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config"
|
||||
)
|
||||
|
||||
func TestConfigCommandsInspectApplicationWithoutEngine(t *testing.T) {
|
||||
path := writeAppConfigFile(t, "{}\n")
|
||||
for _, mode := range []string{"validate", "show", "sources"} {
|
||||
code, stdout, stderr := runCLICommand(t, configCommand, []string{mode, "--config", path})
|
||||
if code != ExitOK || stderr != "" {
|
||||
t.Fatalf("%s failed: code=%d stdout=%q stderr=%q", mode, code, stdout, stderr)
|
||||
}
|
||||
switch mode {
|
||||
case "validate":
|
||||
if stdout != "" {
|
||||
t.Fatalf("validation wrote stdout: %q", stdout)
|
||||
}
|
||||
case "show":
|
||||
if !strings.Contains(stdout, "render_format: text") || !strings.Contains(stdout, "max_request_bytes: 16777216") {
|
||||
t.Fatalf("effective defaults missing: %q", stdout)
|
||||
}
|
||||
case "sources":
|
||||
var sources []configurationSource
|
||||
if err := json.Unmarshal([]byte(stdout), &sources); err != nil || len(sources) != 1 || sources[0].Layer != "main" {
|
||||
t.Fatalf("unexpected source records: %q (%v)", stdout, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectionExampleLoadsAndRenders(t *testing.T) {
|
||||
t.Chdir(filepath.Join("..", "..", ".."))
|
||||
want := []appconfig.SourceSelection{
|
||||
{File: "examples/framework/backends.yml"},
|
||||
{File: "examples/profiles/local-fast.yaml"},
|
||||
{File: "examples/prompts/generic.markdown_summary.yaml"},
|
||||
}
|
||||
for _, overlay := range []string{"", "collection"} {
|
||||
result, err := appconfig.Load(appconfig.LoadOptions{Path: "examples/config.collections.yml", Explicit: true, Overlay: overlay})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
expected := append([]appconfig.SourceSelection{}, want...)
|
||||
if overlay != "" {
|
||||
expected = append(expected, appconfig.SourceSelection{File: "examples/profiles/local-gpu.yml"})
|
||||
}
|
||||
if !reflect.DeepEqual(result.Config.PromptKitSources, expected) {
|
||||
t.Fatalf("overlay %q selectors = %+v, want %+v", overlay, result.Config.PromptKitSources, expected)
|
||||
}
|
||||
args := []string{"--config", "examples/config.collections.yml", "--prompt", "generic.markdown_summary", "--input", "transcript=examples/fixtures/transcript.md", "--input", "glossary=examples/fixtures/glossary.yml"}
|
||||
if overlay != "" {
|
||||
args = append(args, "--config-overlay", overlay)
|
||||
}
|
||||
code, stdout, stderr := runCLICommand(t, renderCommand, args)
|
||||
if code != ExitOK || !strings.Contains(stdout, "generic.markdown_summary") || !strings.Contains(stdout, "local-fast") {
|
||||
t.Fatalf("collection render failed: code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigCommandsExposeComposedSourcesAndResolvedValues(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(dir, "overlays"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
base := filepath.Join(dir, "config.yml")
|
||||
for path, body := range map[string]string{
|
||||
base: "config:\n imports:\n - path: fragment.yml\n required: true\npromptkit_sources:\n - directory: ' ./before/../prompts '\nschema_dir: ./schemas\n",
|
||||
filepath.Join(dir, "fragment.yml"): "server:\n addr: :8081\n",
|
||||
filepath.Join(dir, "overlays", "local.yml"): "promptkit_sources:\n - directory: ./overlay-prompts\nserver:\n addr: :9090\n",
|
||||
} {
|
||||
if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
args := []string{"--config", base, "--config-overlay", "local", "--schema-dir=", "--addr", ":7070"}
|
||||
code, stdout, stderr := runCLICommand(t, configCommand, append([]string{"show"}, args...))
|
||||
if code != ExitOK || stderr != "" || !strings.Contains(stdout, "directory: overlay-prompts") || !strings.Contains(stdout, "schema_dir: ''") || !strings.Contains(stdout, "addr: :7070") || strings.Contains(stdout, "before/../prompts") {
|
||||
t.Fatalf("resolved effective output changed: code=%d out=%q err=%q", code, stdout, stderr)
|
||||
}
|
||||
code, stdout, stderr = runCLICommand(t, configCommand, append([]string{"sources"}, args...))
|
||||
if code != ExitOK || stderr != "" || !strings.HasSuffix(stdout, "\n") {
|
||||
t.Fatalf("source command failed: code=%d out=%q err=%q", code, stdout, stderr)
|
||||
}
|
||||
var sources []configurationSource
|
||||
if err := json.Unmarshal([]byte(stdout), &sources); err != nil || len(sources) != 5 {
|
||||
t.Fatalf("unexpected composed source records: %+v (%v)", sources, err)
|
||||
}
|
||||
for i, layer := range []string{"main", "main", "overlay", "cli", "cli"} {
|
||||
if string(sources[i].Layer) != layer {
|
||||
t.Fatalf("source %d layer = %q, want %q", i, sources[i].Layer, layer)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplicationInspectionDoesNotAcquireFrameworkSources(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
frameworkPath := filepath.Join(dir, "invalid-framework.yml")
|
||||
if err := os.WriteFile(frameworkPath, []byte("not: [valid\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, frameworkSource := range []struct {
|
||||
name string
|
||||
path string
|
||||
}{
|
||||
{name: "invalid definition", path: frameworkPath},
|
||||
{name: "missing path", path: filepath.Join(dir, "missing-framework.yml")},
|
||||
} {
|
||||
t.Run(frameworkSource.name, func(t *testing.T) {
|
||||
path := writeAppConfigFile(t, "promptkit_sources:\n - file: "+frameworkSource.path+"\n")
|
||||
for _, args := range [][]string{
|
||||
{"validate"},
|
||||
{"show"},
|
||||
{"sources"},
|
||||
{"explain", "--path", "/promptkit_sources/0/file"},
|
||||
} {
|
||||
code, stdout, stderr := runCLICommand(t, configCommand, append(args, "--config", path))
|
||||
if code != ExitOK || stderr != "" {
|
||||
t.Fatalf("application inspection acquired configured framework source: args=%q code=%d out=%q err=%q", args, code, stdout, stderr)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigCommandsRejectRetiredFrameworkFields(t *testing.T) {
|
||||
for _, field := range []string{"prompt_dir", "profile_dir", "backends"} {
|
||||
t.Run(field, func(t *testing.T) {
|
||||
path := writeAppConfigFile(t, field+": {}\n")
|
||||
code, stdout, stderr := runCLICommand(t, configCommand, []string{"validate", "--config", path})
|
||||
if code != ExitRuntimeError || stdout != "" || stderr == "" {
|
||||
t.Fatalf("retired application field %q was accepted: code=%d out=%q err=%q", field, code, stdout, stderr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type errorOutputWriter struct{}
|
||||
|
||||
func (errorOutputWriter) Write([]byte) (int, error) { return 0, errors.New("output unavailable") }
|
||||
|
||||
func TestConfigCommandFailuresDoNotProduceSuccessfulOutput(t *testing.T) {
|
||||
path := writeAppConfigFile(t, "server:\n max_request_bytes: -1\n")
|
||||
for _, mode := range []string{"validate", "show", "sources"} {
|
||||
code, stdout, stderr := runCLICommand(t, configCommand, []string{mode, "--config", path})
|
||||
if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "validate.error") {
|
||||
t.Fatalf("%s misclassified invalid config: code=%d out=%q err=%q", mode, code, stdout, stderr)
|
||||
}
|
||||
}
|
||||
path = writeAppConfigFile(t, "{}\n")
|
||||
var stderr bytes.Buffer
|
||||
if code := configCommand([]string{"show", "--config", path}, errorOutputWriter{}, &stderr); code != ExitRuntimeError || !strings.Contains(stderr.String(), "output error") {
|
||||
t.Fatalf("writer error not reported: code=%d stderr=%q", code, stderr.String())
|
||||
}
|
||||
if code := configCommand([]string{"sources", "--config", path}, errorOutputWriter{}, io.Discard); code != ExitRuntimeError {
|
||||
t.Fatalf("source writer error returned %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigExplainUsesResolvedHistory(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(dir, "overlays"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
base := filepath.Join(dir, "config.yml")
|
||||
for path, body := range map[string]string{
|
||||
base: `promptkit_sources:
|
||||
- directory: " ./prompts/../prompts "
|
||||
server:
|
||||
addr: :8080
|
||||
`,
|
||||
filepath.Join(dir, "overlays", "local.yml"): "server:\n addr: :9090\n",
|
||||
} {
|
||||
if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
args := []string{"--config", base, "--config-overlay", "local", "--addr", ":9090"}
|
||||
for _, tc := range []struct {
|
||||
pointer string
|
||||
want []string
|
||||
}{
|
||||
{pointer: "/server/addr", want: []string{":8080", ":9090", "--addr"}},
|
||||
{pointer: "/promptkit_sources/0/directory", want: []string{"normalized application path", "prompts"}},
|
||||
{pointer: "", want: []string{"promptkit_sources", "server", "addr"}},
|
||||
} {
|
||||
code, stdout, stderr := runCLICommand(t, configCommand, append([]string{"explain", "--path", tc.pointer}, args...))
|
||||
if code != ExitOK || stderr != "" {
|
||||
t.Fatalf("explain %q failed: code=%d out=%q err=%q", tc.pointer, code, stdout, stderr)
|
||||
}
|
||||
for _, want := range tc.want {
|
||||
if !strings.Contains(stdout, want) {
|
||||
t.Fatalf("explain %q lacks %q: %q", tc.pointer, want, stdout)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigExplainRejectsUnknownOrMalformedPaths(t *testing.T) {
|
||||
path := writeAppConfigFile(t, "{}\n")
|
||||
for _, args := range [][]string{
|
||||
{"explain", "--config", path},
|
||||
{"explain", "--config", path, "--path", "not-a-pointer"},
|
||||
{"explain", "--config", path, "--path", "/missing"},
|
||||
} {
|
||||
code, stdout, stderr := runCLICommand(t, configCommand, args)
|
||||
if code != ExitRuntimeError || stdout != "" || stderr == "" {
|
||||
t.Fatalf("invalid explanation path accepted: args=%q code=%d out=%q err=%q", args, code, stdout, stderr)
|
||||
}
|
||||
}
|
||||
var stderr bytes.Buffer
|
||||
if code := configCommand([]string{"explain", "--config", path, "--path", ""}, errorOutputWriter{}, &stderr); code != ExitRuntimeError || !strings.Contains(stderr.String(), "output error") {
|
||||
t.Fatalf("explanation writer failure not reported: code=%d err=%q", code, stderr.String())
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,9 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -41,37 +43,46 @@ func TestCollectionCheckoutDiscoveryAcrossInvocations(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
config := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n", frameworkPath))
|
||||
config := filepath.Join(root, "app.yml")
|
||||
writeNativeFile(t, config, fmt.Sprintf("promptkit:\n configuration: %q\n", frameworkPath))
|
||||
input := filepath.Join(root, "transcript.md")
|
||||
if err := os.WriteFile(input, []byte("synthetic discovery transcript"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
args := func(name string) []string {
|
||||
if name == "support-repo" {
|
||||
return []string{"--config", config, "--prompt", "support/answer", "--profile", "support/fast", "--input", "transcript=" + input}
|
||||
return []string{"support/answer", "--config", config, "--profile", "support/fast", "--input-file", "transcript=" + input}
|
||||
}
|
||||
return []string{"--config", config, "--prompt", "review/answer", "--profile", "review/base"}
|
||||
return []string{"review/answer", "--config", config, "--profile", "review/base"}
|
||||
}
|
||||
check := func(name string) {
|
||||
t.Helper()
|
||||
code, out, stderr := runCLICommand(t, renderCommand, args(name))
|
||||
if code != ExitOK {
|
||||
t.Fatalf("render %s: code=%d stderr=%s", name, code, stderr)
|
||||
command := append([]string{"promptkit", "show", "prepared"}, args(name)...)
|
||||
outcome, out, stderr := executeNative(t, append(command, "--format=json")...)
|
||||
if outcome.Status != 0 {
|
||||
t.Fatalf("prepare %s: outcome=%+v stderr=%s", name, outcome, stderr)
|
||||
}
|
||||
var prepared promptkit.PreparedRun
|
||||
if err := json.Unmarshal([]byte(out), &prepared); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if name == "support-repo" {
|
||||
for _, want := range []string{"prompt: support/answer", "selected_profile_id: support/fast", "model: support-example", "max_tokens: 256", "Answer the support request", "synthetic discovery transcript"} {
|
||||
for _, want := range []string{"support/answer", "support/fast", "support-example", "Answer the support request", "synthetic discovery transcript"} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Fatalf("missing %q: %s", want, out)
|
||||
}
|
||||
}
|
||||
} else if !strings.Contains(out, "prompt: review/answer") || !strings.Contains(out, "model: review-example") || !strings.Contains(out, "Review this material") {
|
||||
if prepared.EffectiveModelParams.MaxTokens != 256 {
|
||||
t.Fatalf("inherited profile setting missing: %+v", prepared)
|
||||
}
|
||||
} else if !strings.Contains(out, "review/answer") || !strings.Contains(out, "review-example") || !strings.Contains(out, "Review this material") {
|
||||
t.Fatalf("unexpected review render: %s", out)
|
||||
}
|
||||
}
|
||||
install(tc.first)
|
||||
check(tc.first)
|
||||
code, _, _ := runCLICommand(t, renderCommand, args(tc.second))
|
||||
if code != ExitRuntimeError {
|
||||
outcome, _, _ := executeNative(t, append([]string{"promptkit", "show", "prepared"}, args(tc.second)...)...)
|
||||
if outcome.Status != 1 {
|
||||
t.Fatal("uninstalled collection unexpectedly selected")
|
||||
}
|
||||
install(tc.second)
|
||||
|
||||
42
internal/adapter/cli/examples_test.go
Normal file
42
internal/adapter/cli/examples_test.go
Normal file
@@ -0,0 +1,42 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestMaintainedApplicationExamplesUseNativeLoading(t *testing.T) {
|
||||
root, err := filepath.Abs(filepath.Join("..", "..", ".."))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Chdir(root)
|
||||
entries, err := os.ReadDir("examples")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() || !strings.HasPrefix(entry.Name(), "config") || !strings.HasSuffix(entry.Name(), ".yml") {
|
||||
continue
|
||||
}
|
||||
path := filepath.Join("examples", entry.Name())
|
||||
for _, scope := range []string{"app", "promptkit"} {
|
||||
outcome, _, diagnostics := executeNative(t, "config", "validate", "--config="+path, "--scope="+scope)
|
||||
if outcome.Status != 0 {
|
||||
t.Fatalf("%s/%s: %+v %q", path, scope, outcome, diagnostics)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, test := range []struct{ config, overlay string }{{"examples/config.yml", ""}, {"examples/config.composed.yml", "local"}, {"examples/config.collections.yml", "collection"}, {"examples/config.inheritance.yml", ""}} {
|
||||
args := []string{"promptkit", "show", "prepared", "generic.markdown_summary", "--config=" + test.config, "--profile=local-gpu", "--input-file=transcript=examples/fixtures/transcript.md", "--input-file=glossary=examples/fixtures/glossary.yml", "--format=json"}
|
||||
if test.overlay != "" {
|
||||
args = append(args, "--overlay="+test.overlay)
|
||||
}
|
||||
outcome, output, diagnostics := executeNative(t, args...)
|
||||
if outcome.Status != 0 || !strings.Contains(output, "generic.markdown_summary") || !strings.Contains(output, "Session Transcript") {
|
||||
t.Fatalf("%+v: %+v %q %q", test, outcome, output, diagnostics)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,159 +0,0 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
type frameworkConfigurationConfig struct {
|
||||
mode, configPath, configOverlay, schemaDir string
|
||||
path string
|
||||
configExplicit bool
|
||||
pathExplicit bool
|
||||
overrides []configkit.Override
|
||||
diagnostics []configkit.Diagnostic
|
||||
}
|
||||
|
||||
type frameworkConfigurationSource struct {
|
||||
ID string `json:"id"`
|
||||
Label string `json:"label"`
|
||||
Layer configkit.Layer `json:"layer"`
|
||||
Kind configkit.SourceKind `json:"kind"`
|
||||
UnderlayIndex int `json:"underlay_index"`
|
||||
Path string `json:"path,omitempty"`
|
||||
ImportedFrom *frameworkConfigurationLocation `json:"imported_from,omitempty"`
|
||||
}
|
||||
|
||||
type frameworkConfigurationLocation struct {
|
||||
SourceID string `json:"source_id"`
|
||||
Line int `json:"line"`
|
||||
Column int `json:"column"`
|
||||
}
|
||||
|
||||
func inspectFrameworkConfigurationCommand(args []string, stdout, stderr io.Writer) int {
|
||||
cfg, err := parseFrameworkConfigurationArgs(args)
|
||||
if err != nil {
|
||||
writeApplicationError(stderr, "inspect configuration parse", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
var explanationPath configkit.Path
|
||||
if cfg.mode == "explain" {
|
||||
explanationPath, err = configkit.ParsePath(cfg.path)
|
||||
if err != nil {
|
||||
writeApplicationError(stderr, "inspect configuration path", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
}
|
||||
settings, err := resolveAppSettingsForFrameworkConfiguration(cfg)
|
||||
if err != nil {
|
||||
writeApplicationError(stderr, "inspect configuration", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
if err := writeApplicationWarnings(stderr, cfg.diagnostics); err != nil {
|
||||
return ExitRuntimeError
|
||||
}
|
||||
engine, err := newEngine(settings)
|
||||
if err != nil {
|
||||
writeApplicationError(stderr, "engine", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
view, err := engine.InspectConfiguration()
|
||||
if err != nil {
|
||||
writeApplicationError(stderr, "inspect configuration", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
if err := configkit.WriteDiagnostics(stderr, view.Diagnostics()); err != nil {
|
||||
return ExitRuntimeError
|
||||
}
|
||||
if cfg.mode == "validate" {
|
||||
return ExitOK
|
||||
}
|
||||
|
||||
if cfg.mode == "explain" {
|
||||
if err := view.WriteExplanation(stdout, explanationPath); err != nil {
|
||||
fmt.Fprintf(stderr, "inspect configuration explain %q: %v\n", cfg.path, err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
return ExitOK
|
||||
}
|
||||
|
||||
switch cfg.mode {
|
||||
case "show":
|
||||
err = view.WriteEffective(stdout)
|
||||
case "sources":
|
||||
sources := frameworkConfigurationSources(view.Sources())
|
||||
err = json.NewEncoder(stdout).Encode(sources)
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "inspect configuration output error: %v\n", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
return ExitOK
|
||||
}
|
||||
|
||||
func parseFrameworkConfigurationArgs(args []string) (*frameworkConfigurationConfig, error) {
|
||||
if len(args) == 0 {
|
||||
return nil, errors.New("configuration operation is required")
|
||||
}
|
||||
cfg := &frameworkConfigurationConfig{mode: args[0]}
|
||||
switch cfg.mode {
|
||||
case "validate", "show", "sources", "explain":
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown configuration operation %q", cfg.mode)
|
||||
}
|
||||
|
||||
fs := flag.NewFlagSet("inspect configuration "+cfg.mode, flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
registerConfigSelectionFlags(fs, &cfg.configPath, &cfg.configOverlay)
|
||||
fs.StringVar(&cfg.path, "path", "", "JSON Pointer to explain (required for explain)")
|
||||
if err := fs.Parse(args[1:]); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if fs.NArg() != 0 {
|
||||
return nil, fmt.Errorf("unexpected positional args: %v", fs.Args())
|
||||
}
|
||||
cfg.configExplicit = flagWasSet(fs, "config")
|
||||
cfg.pathExplicit = flagWasSet(fs, "path")
|
||||
if cfg.mode == "explain" && !cfg.pathExplicit {
|
||||
return nil, errors.New("--path is required for explain")
|
||||
}
|
||||
cfg.overrides = applicationOverrides(fs)
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func resolveAppSettingsForFrameworkConfiguration(cfg *frameworkConfigurationConfig) (engineSettings, error) {
|
||||
settings, diagnostics, err := loadApplicationSettings(cfg.configPath, cfg.configOverlay, cfg.configExplicit, cfg.overrides)
|
||||
if err != nil {
|
||||
return engineSettings{}, err
|
||||
}
|
||||
cfg.diagnostics = diagnostics
|
||||
return engineSettings{
|
||||
promptkitSources: settings.PromptKitSources,
|
||||
schemaDir: settings.SchemaDir,
|
||||
transientRetries: settings.TransientRetries,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func frameworkConfigurationSources(sources []configkit.SourceInfo) []frameworkConfigurationSource {
|
||||
result := make([]frameworkConfigurationSource, 0, len(sources))
|
||||
for _, source := range sources {
|
||||
item := frameworkConfigurationSource{
|
||||
ID: source.ID, Label: source.Label, Layer: source.Layer,
|
||||
Kind: source.Kind, UnderlayIndex: source.UnderlayIndex, Path: source.Path,
|
||||
}
|
||||
if source.ImportedFrom != nil {
|
||||
item.ImportedFrom = &frameworkConfigurationLocation{
|
||||
SourceID: source.ImportedFrom.SourceID,
|
||||
Line: source.ImportedFrom.Line,
|
||||
Column: source.ImportedFrom.Column,
|
||||
}
|
||||
}
|
||||
result = append(result, item)
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -1,391 +0,0 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
renderformat "gitea.maximumdirect.net/eric/scriptorium/internal/format"
|
||||
configkit "go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
func TestFrameworkConfigurationCommandsNeedNoPromptOrCredential(t *testing.T) {
|
||||
configPath := writeAppConfigFile(t, "{}\n")
|
||||
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
|
||||
"configuration", "validate", "--config", configPath,
|
||||
})
|
||||
if code != ExitOK || stdout != "" || stderr != "" {
|
||||
t.Fatalf("source-free validation failed: code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
||||
}
|
||||
|
||||
t.Setenv("SCRIPTORIUM_FRAMEWORK_KEY", "")
|
||||
root := t.TempDir()
|
||||
frameworkFile := filepath.Join(root, "framework.yml")
|
||||
writeFrameworkSourceFile(t, root, "framework.yml", `backends:
|
||||
- id: local
|
||||
endpoint: http://127.0.0.1:1/v1
|
||||
api_key_env: SCRIPTORIUM_FRAMEWORK_KEY
|
||||
profiles:
|
||||
- id: protected
|
||||
backend: local
|
||||
model: offline-model
|
||||
api_key_required: true
|
||||
`)
|
||||
configPath = writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n", frameworkFile))
|
||||
for _, mode := range []string{"validate", "show", "sources"} {
|
||||
code, stdout, stderr = runCLICommand(t, inspectCommand, []string{
|
||||
"configuration", mode, "--config", configPath,
|
||||
})
|
||||
if code != ExitOK || stderr != "" {
|
||||
t.Fatalf("%s required a prompt or credential: code=%d stdout=%q stderr=%q", mode, code, stdout, stderr)
|
||||
}
|
||||
if mode == "validate" && stdout != "" {
|
||||
t.Fatalf("validation wrote success output: %q", stdout)
|
||||
}
|
||||
if mode == "show" && (!strings.Contains(stdout, "profiles:") || !strings.Contains(stdout, "offline-model")) {
|
||||
t.Fatalf("effective framework YAML is missing: %q", stdout)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFrameworkConfigurationSourcesProjectsOrderedNativeRecords(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
importDir := filepath.Join(root, "catalog")
|
||||
if err := os.MkdirAll(importDir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mainPath := writeFrameworkSourceFile(t, root, "main.yml", `config:
|
||||
imports:
|
||||
- path: catalog/fragment.yml
|
||||
required: true
|
||||
backends:
|
||||
- id: local
|
||||
endpoint: http://127.0.0.1:1/v1
|
||||
`)
|
||||
fragmentPath := writeFrameworkSourceFile(t, importDir, "fragment.yml", `profiles:
|
||||
- id: imported
|
||||
model: imported-model
|
||||
`)
|
||||
directory := t.TempDir()
|
||||
directoryPath := writeFrameworkSourceFile(t, directory, "directory.yml", `profiles:
|
||||
- id: directory
|
||||
model: directory-model
|
||||
`)
|
||||
configuredMissing := filepath.Join(root, "configured-but-replaced.yml")
|
||||
configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n", configuredMissing))
|
||||
args := []string{
|
||||
"configuration", "sources", "--config", configPath,
|
||||
"--promptkit-source", mainPath,
|
||||
"--promptkit-source", directory,
|
||||
}
|
||||
code, stdout, stderr := runCLICommand(t, inspectCommand, args)
|
||||
if code != ExitOK || stderr != "" || !strings.HasSuffix(stdout, "\n") {
|
||||
t.Fatalf("framework sources command failed: code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
||||
}
|
||||
var sources []frameworkConfigurationSource
|
||||
if err := json.Unmarshal([]byte(stdout), &sources); err != nil {
|
||||
t.Fatalf("decode framework source DTOs: %v\n%s", err, stdout)
|
||||
}
|
||||
var rawSources []map[string]any
|
||||
if err := json.Unmarshal([]byte(stdout), &rawSources); err != nil {
|
||||
t.Fatalf("decode framework source fields: %v", err)
|
||||
}
|
||||
mainIndex := findFrameworkSource(t, sources, mainPath)
|
||||
fragmentIndex := findFrameworkSource(t, sources, fragmentPath)
|
||||
directoryIndex := findFrameworkSource(t, sources, directoryPath)
|
||||
main := sources[mainIndex]
|
||||
fragment := sources[fragmentIndex]
|
||||
directorySource := sources[directoryIndex]
|
||||
if !(mainIndex < fragmentIndex && fragmentIndex < directoryIndex) {
|
||||
t.Fatalf("captured source order changed: main=%d fragment=%d directory=%d", mainIndex, fragmentIndex, directoryIndex)
|
||||
}
|
||||
if main.ID == "" || main.Label == "" || main.Layer != configkit.LayerUnderlay || main.Kind != configkit.SourceKindFile || main.UnderlayIndex < 1 {
|
||||
t.Fatalf("explicit file source fields are incomplete: %+v", main)
|
||||
}
|
||||
for _, key := range []string{"id", "label", "layer", "kind", "underlay_index", "path"} {
|
||||
if _, exists := rawSources[mainIndex][key]; !exists {
|
||||
t.Fatalf("source DTO omitted %q: %#v", key, rawSources[mainIndex])
|
||||
}
|
||||
}
|
||||
for _, key := range []string{"ID", "Label", "Layer", "Kind", "UnderlayIndex", "filesystem"} {
|
||||
if _, exists := rawSources[mainIndex][key]; exists {
|
||||
t.Fatalf("source DTO exposed implementation field %q: %#v", key, rawSources[mainIndex])
|
||||
}
|
||||
}
|
||||
if fragment.Layer != configkit.LayerUnderlay || fragment.Kind != configkit.SourceKindFile || fragment.ImportedFrom == nil ||
|
||||
fragment.ImportedFrom.SourceID != main.ID || fragment.ImportedFrom.Line < 1 || fragment.ImportedFrom.Column < 1 {
|
||||
t.Fatalf("import source fields are incomplete: %+v", fragment)
|
||||
}
|
||||
if importedFrom, ok := rawSources[fragmentIndex]["imported_from"].(map[string]any); !ok || importedFrom["source_id"] != main.ID || importedFrom["line"] == nil || importedFrom["column"] == nil {
|
||||
t.Fatalf("import location DTO fields are incomplete: %#v", rawSources[fragmentIndex])
|
||||
}
|
||||
if directorySource.Layer != configkit.LayerUnderlay || directorySource.Kind != configkit.SourceKindFile || directorySource.UnderlayIndex <= main.UnderlayIndex {
|
||||
t.Fatalf("directory-selected source fields/order are incomplete: %+v (main %+v)", directorySource, main)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFrameworkConfigurationShowFailsBeforeOutputForInvalidFinalDefinition(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
frameworkFile := writeFrameworkSourceFile(t, root, "invalid.yml", `prompts:
|
||||
- id: unused
|
||||
messages:
|
||||
- role: user
|
||||
content: "{{"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)
|
||||
configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n", frameworkFile))
|
||||
code, stdout, stderr := runCLICommand(t, configCommand, []string{"validate", "--config", configPath})
|
||||
if code != ExitOK || stdout != "" || stderr != "" {
|
||||
t.Fatalf("application validation loaded framework definitions: code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
||||
}
|
||||
for _, args := range [][]string{
|
||||
{"configuration", "validate", "--config", configPath},
|
||||
{"configuration", "show", "--config", configPath},
|
||||
{"configuration", "sources", "--config", configPath},
|
||||
{"configuration", "explain", "--config", configPath, "--path", ""},
|
||||
} {
|
||||
code, stdout, stderr = runCLICommand(t, inspectCommand, args)
|
||||
if code != ExitRuntimeError || stdout != "" || stderr == "" {
|
||||
t.Fatalf("invalid final framework definition was reported as success: args=%q code=%d stdout=%q stderr=%q", args, code, stdout, stderr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFrameworkConfigurationCommandsRejectExtraArgumentsAndReportWriterErrors(t *testing.T) {
|
||||
configPath := writeAppConfigFile(t, "{}\n")
|
||||
for _, args := range [][]string{
|
||||
{"configuration"},
|
||||
{"configuration", "explain"},
|
||||
{"configuration", "show", "unexpected"},
|
||||
{"configuration", "sources", "--format", "json"},
|
||||
} {
|
||||
code, stdout, stderr := runCLICommand(t, inspectCommand, args)
|
||||
if code != ExitRuntimeError || stdout != "" || stderr == "" {
|
||||
t.Fatalf("invalid inspection arguments were accepted: args=%q code=%d stdout=%q stderr=%q", args, code, stdout, stderr)
|
||||
}
|
||||
}
|
||||
|
||||
writerError := errors.New("synthetic framework output failure")
|
||||
var stderr bytes.Buffer
|
||||
code := inspectCommand([]string{"configuration", "show", "--config", configPath}, frameworkConfigurationFailingWriter{err: writerError}, &stderr)
|
||||
if code != ExitRuntimeError || !strings.Contains(stderr.String(), writerError.Error()) {
|
||||
t.Fatalf("effective-output writer failure was not reported: code=%d stderr=%q", code, stderr.String())
|
||||
}
|
||||
stderr.Reset()
|
||||
code = inspectCommand([]string{"configuration", "explain", "--config", configPath, "--path", ""}, frameworkConfigurationFailingWriter{err: writerError}, &stderr)
|
||||
if code != ExitRuntimeError || !strings.Contains(stderr.String(), writerError.Error()) {
|
||||
t.Fatalf("explanation writer failure was not reported: code=%d stderr=%q", code, stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestFrameworkConfigurationExplainUsesLiteralJSONPointersAndNativeHistory(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
lower := writeFrameworkSourceFile(t, root, "lower.yml", `collections:
|
||||
support:
|
||||
backends:
|
||||
- id: local
|
||||
endpoint: http://127.0.0.1:1/v1
|
||||
api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE
|
||||
extra_params:
|
||||
token: backend-token-displaced-canary
|
||||
password: backend-password-canary
|
||||
visible_option: backend-visible-value
|
||||
provider.option: dotted-visible-value
|
||||
profiles:
|
||||
- id: summary
|
||||
backend: support/local
|
||||
model: lower-model
|
||||
api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE
|
||||
extra_params:
|
||||
token: profile-token-displaced-canary
|
||||
password: profile-password-displaced-canary
|
||||
visible_option: profile-visible-value
|
||||
provider.option: dotted-visible-value
|
||||
prompts:
|
||||
- id: explainable
|
||||
default_profile: support/summary
|
||||
messages:
|
||||
- role: user
|
||||
content: hello
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)
|
||||
upperRoot := t.TempDir()
|
||||
upper := writeFrameworkSourceFile(t, upperRoot, "upper.yml", `collections:
|
||||
support:
|
||||
backends:
|
||||
- id: local
|
||||
endpoint: http://127.0.0.1:1/v1
|
||||
api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE
|
||||
extra_params:
|
||||
token: backend-token-effective-canary
|
||||
password: backend-password-effective-canary
|
||||
visible_option: backend-visible-value
|
||||
provider.option: dotted-visible-value
|
||||
profiles:
|
||||
- id: summary
|
||||
backend: support/local
|
||||
model: effective-model
|
||||
api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE
|
||||
extra_params:
|
||||
token: profile-token-effective-canary
|
||||
password: profile-password-effective-canary
|
||||
visible_option: profile-visible-value
|
||||
provider.option: dotted-visible-value
|
||||
`)
|
||||
configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n - file: %q\n", lower, upper))
|
||||
const secret = "environment-value-must-never-be-emitted"
|
||||
t.Setenv("SCRIPTORIUM_EXPLAIN_REFERENCE", secret)
|
||||
|
||||
for _, tc := range []struct {
|
||||
path string
|
||||
want []string
|
||||
unwant []string
|
||||
}{
|
||||
{
|
||||
path: "/collections/support/backends/0/extra_params/token",
|
||||
unwant: []string{"backend-token-displaced-canary", "backend-token-effective-canary"},
|
||||
},
|
||||
{
|
||||
path: "/collections/support/backends/0/extra_params/password",
|
||||
unwant: []string{"backend-password-displaced-canary", "backend-password-effective-canary"},
|
||||
},
|
||||
{
|
||||
path: "/collections/support/profiles/0/model",
|
||||
want: []string{"effective-model", "lower-model"},
|
||||
unwant: []string{},
|
||||
},
|
||||
{
|
||||
path: "/collections/support/profiles/0/extra_params/token",
|
||||
unwant: []string{"profile-token-displaced-canary", "profile-token-effective-canary"},
|
||||
},
|
||||
{
|
||||
path: "/collections/support/profiles/0/extra_params/password",
|
||||
unwant: []string{"profile-password-displaced-canary", "profile-password-effective-canary"},
|
||||
},
|
||||
{
|
||||
path: "/collections/support/profiles/0/api_key_env",
|
||||
want: []string{"SCRIPTORIUM_EXPLAIN_REFERENCE"},
|
||||
unwant: []string{secret},
|
||||
},
|
||||
{
|
||||
path: "/collections/support/profiles/0/extra_params/provider.option",
|
||||
want: []string{"dotted-visible-value"},
|
||||
unwant: []string{"provider/option"},
|
||||
},
|
||||
{
|
||||
path: "",
|
||||
want: []string{"profiles:", "effective-model"},
|
||||
unwant: []string{"profile-token-displaced-canary", "backend-password-effective-canary"},
|
||||
},
|
||||
} {
|
||||
args := []string{"configuration", "explain", "--config", configPath, "--path", tc.path}
|
||||
code, stdout, stderr := runCLICommand(t, inspectCommand, args)
|
||||
if code != ExitOK || stderr != "" {
|
||||
t.Fatalf("explain %q failed: code=%d stdout=%q stderr=%q", tc.path, code, stdout, stderr)
|
||||
}
|
||||
if stdout == "" {
|
||||
t.Fatalf("explain %q produced no output", tc.path)
|
||||
}
|
||||
for _, value := range tc.want {
|
||||
if !strings.Contains(stdout, value) {
|
||||
t.Errorf("explain %q omitted %q: %s", tc.path, value, stdout)
|
||||
}
|
||||
}
|
||||
for _, value := range tc.unwant {
|
||||
if strings.Contains(stdout, value) {
|
||||
t.Errorf("explain %q disclosed %q: %s", tc.path, value, stdout)
|
||||
}
|
||||
}
|
||||
if strings.Contains(stdout, secret) {
|
||||
t.Errorf("explain %q disclosed the resolved environment value: %s", tc.path, stdout)
|
||||
}
|
||||
}
|
||||
|
||||
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{"configuration", "show", "--config", configPath})
|
||||
if code != ExitOK || stderr != "" {
|
||||
t.Fatalf("framework show failed: code=%d stderr=%q", code, stderr)
|
||||
}
|
||||
for _, value := range []string{"SCRIPTORIUM_EXPLAIN_REFERENCE", "backend-visible-value", "profile-visible-value", "dotted-visible-value"} {
|
||||
if !strings.Contains(stdout, value) {
|
||||
t.Errorf("effective report omitted visible configuration %q: %s", value, stdout)
|
||||
}
|
||||
}
|
||||
for _, value := range []string{
|
||||
"backend-token-displaced-canary", "backend-token-effective-canary",
|
||||
"backend-password-displaced-canary", "backend-password-effective-canary",
|
||||
"profile-token-displaced-canary", "profile-token-effective-canary",
|
||||
"profile-password-displaced-canary", "profile-password-effective-canary", secret,
|
||||
} {
|
||||
if strings.Contains(stdout, value) {
|
||||
t.Errorf("effective report disclosed classified or resolved value %q: %s", value, stdout)
|
||||
}
|
||||
}
|
||||
|
||||
profileCode, profileOutput, profileError := runCLICommand(t, inspectCommand, []string{
|
||||
"profile", "--config", configPath, "--profile", "support/summary", "--format", "json",
|
||||
})
|
||||
if profileCode != ExitOK || profileError != "" {
|
||||
t.Fatalf("existing profile inspection failed: code=%d stdout=%q stderr=%q", profileCode, profileOutput, profileError)
|
||||
}
|
||||
var profile renderformat.ProfileInspection
|
||||
if err := json.Unmarshal([]byte(profileOutput), &profile); err != nil {
|
||||
t.Fatalf("decode profile inspection: %v\n%s", err, profileOutput)
|
||||
}
|
||||
if pointerValue(profile.ProfileModelParams.Model) != "effective-model" || pointerValue(profile.ProfileModelParams.ExtraParams)["visible_option"] != "profile-visible-value" {
|
||||
t.Fatalf("existing profile inspection changed configured provider data: %+v", profile.ProfileModelParams)
|
||||
}
|
||||
if strings.Contains(profileOutput, secret) {
|
||||
t.Fatalf("profile inspection emitted resolved environment data: %s", profileOutput)
|
||||
}
|
||||
|
||||
code, rendered, stderr := runCLICommand(t, renderCommand, []string{"--config", configPath, "--prompt", "support/explainable"})
|
||||
if code != ExitOK || stderr != "" {
|
||||
t.Fatalf("prepared output failed: code=%d stdout=%q stderr=%q", code, rendered, stderr)
|
||||
}
|
||||
if !strings.Contains(rendered, "profile-visible-value") || strings.Contains(rendered, secret) {
|
||||
t.Fatalf("prepared output did not preserve ordinary provider data or disclosed environment data: %s", rendered)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFrameworkConfigurationExplainRejectsMissingMalformedAndUnknownPaths(t *testing.T) {
|
||||
configPath := writeAppConfigFile(t, "{}\n")
|
||||
for _, args := range [][]string{
|
||||
{"configuration", "explain"},
|
||||
{"configuration", "explain", "--path", "not-a-pointer"},
|
||||
{"configuration", "explain", "--path", "/profiles/team~2summary/model"},
|
||||
{"configuration", "explain", "--config", configPath, "--path", "/profiles/missing/model"},
|
||||
} {
|
||||
code, stdout, stderr := runCLICommand(t, inspectCommand, args)
|
||||
if code != ExitRuntimeError || stdout != "" || stderr == "" {
|
||||
t.Fatalf("invalid explanation path was accepted: args=%q code=%d stdout=%q stderr=%q", args, code, stdout, stderr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func findFrameworkSource(t *testing.T, sources []frameworkConfigurationSource, path string) int {
|
||||
t.Helper()
|
||||
for index, source := range sources {
|
||||
if filepath.Clean(source.Path) == filepath.Clean(path) {
|
||||
return index
|
||||
}
|
||||
}
|
||||
t.Fatalf("selected source path %q was not present: %+v", path, sources)
|
||||
return -1
|
||||
}
|
||||
|
||||
type frameworkConfigurationFailingWriter struct {
|
||||
err error
|
||||
}
|
||||
|
||||
func (writer frameworkConfigurationFailingWriter) Write([]byte) (int, error) {
|
||||
return 0, writer.err
|
||||
}
|
||||
@@ -1,17 +0,0 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func writeFrameworkDiagnostics(output io.Writer, engine *promptkit.Engine) error {
|
||||
view, err := engine.InspectConfiguration()
|
||||
if err != nil {
|
||||
return fmt.Errorf("inspect captured PromptKit diagnostics: %w", err)
|
||||
}
|
||||
return configkit.WriteDiagnostics(output, view.Diagnostics())
|
||||
}
|
||||
@@ -1,386 +0,0 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config"
|
||||
configkit "go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestFrameworkSourceGroupsComposeAndValidateEffectiveDefinitions(t *testing.T) {
|
||||
conflicting := t.TempDir()
|
||||
writeFrameworkSourceFile(t, conflicting, "first.yml", `profiles:
|
||||
- id: selected
|
||||
model: first
|
||||
`)
|
||||
writeFrameworkSourceFile(t, conflicting, "second.yml", `profiles:
|
||||
- id: selected
|
||||
model: second
|
||||
`)
|
||||
if _, err := newEngine(engineSettings{promptkitSources: []appconfig.SourceSelection{{Directory: conflicting}}}); err == nil {
|
||||
t.Fatal("conflicting files in one source group unexpectedly constructed an engine")
|
||||
}
|
||||
|
||||
lower := t.TempDir()
|
||||
upper := t.TempDir()
|
||||
empty := t.TempDir()
|
||||
writeFrameworkSourceFile(t, lower, "lower.yml", `profiles:
|
||||
- id: selected
|
||||
endpoint: not-a-url
|
||||
model: lower-model
|
||||
temperature: 10
|
||||
`)
|
||||
writeFrameworkSourceFile(t, upper, "upper.yml", `profiles:
|
||||
- id: selected
|
||||
endpoint: http://127.0.0.1:1/v1
|
||||
model: upper-model
|
||||
temperature: 0.5
|
||||
`)
|
||||
engine := newFrameworkSourceTestEngine(t, []appconfig.SourceSelection{
|
||||
{Directory: lower}, {Directory: empty}, {Directory: upper},
|
||||
}, "")
|
||||
inspection, err := engine.InspectProfile(context.Background(), "selected")
|
||||
if err != nil {
|
||||
t.Fatalf("inspect profile after later source replacement: %v", err)
|
||||
}
|
||||
if pointerValue(inspection.Endpoint) != "http://127.0.0.1:1/v1" || pointerValue(inspection.Model) != "upper-model" || pointerValue(inspection.Temperature) != 0.5 {
|
||||
t.Fatalf("later source did not replace lower values before validation: %+v", inspection)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFrameworkSourceConstructionUsesCombinedDocumentAndRejectsInvalidSources(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
combined := filepath.Join(root, "combined.yml")
|
||||
writeFrameworkSourceFile(t, root, "combined.yml", `backends:
|
||||
- id: local
|
||||
endpoint: http://127.0.0.1:1/v1
|
||||
profiles:
|
||||
- id: selected
|
||||
backend: local
|
||||
model: combined-model
|
||||
prompts:
|
||||
- id: combined.prompt
|
||||
default_profile: selected
|
||||
messages:
|
||||
- role: user
|
||||
content: hello
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)
|
||||
client := &frameworkSourceCaptureClient{}
|
||||
engine := newFrameworkSourceTestEngine(t, []appconfig.SourceSelection{{File: combined}}, "", promptkit.WithLLMClient(client))
|
||||
if _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "combined.prompt"}); err != nil {
|
||||
t.Fatalf("run definitions from one combined document: %v", err)
|
||||
}
|
||||
if len(client.requests) != 1 || client.requests[0].Target.BackendID != "local" || client.requests[0].Target.Model != "combined-model" {
|
||||
t.Fatalf("combined backend/profile/prompt values did not reach generation: %+v", client.requests)
|
||||
}
|
||||
|
||||
malformed := filepath.Join(root, "malformed.yml")
|
||||
writeFrameworkSourceFile(t, root, "malformed.yml", "not: [valid\n")
|
||||
if _, err := newEngine(engineSettings{promptkitSources: []appconfig.SourceSelection{{File: malformed}}}); err == nil {
|
||||
t.Fatal("malformed framework source unexpectedly constructed an engine")
|
||||
}
|
||||
|
||||
invalidUnused := filepath.Join(root, "invalid-unused.yml")
|
||||
writeFrameworkSourceFile(t, root, "invalid-unused.yml", `prompts:
|
||||
- id: unused.prompt
|
||||
messages:
|
||||
- role: user
|
||||
content: "{{"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)
|
||||
if _, err := newEngine(engineSettings{promptkitSources: []appconfig.SourceSelection{{File: invalidUnused}}}); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||
t.Fatalf("invalid unused definition did not fail as framework configuration: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFrameworkRejectsObsoleteVersionKeyedPrompt(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
path := writeFrameworkSourceFile(t, root, "obsolete.yml", `prompts:
|
||||
old-prompt:
|
||||
"1":
|
||||
messages:
|
||||
- role: user
|
||||
content: hello
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)
|
||||
if _, err := newEngine(engineSettings{promptkitSources: []appconfig.SourceSelection{{File: path}}}); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||
t.Fatalf("obsolete version-keyed definition was not rejected by PromptKit: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFrameworkImportsRetainResourceOriginsAndCaptureAtConstruction(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
catalogDir := filepath.Join(root, "catalog")
|
||||
if err := os.MkdirAll(catalogDir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
schemaDir := filepath.Join(root, "schemas")
|
||||
if err := os.MkdirAll(schemaDir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mainFile := filepath.Join(root, "main.yml")
|
||||
writeFrameworkSourceFile(t, root, "main.yml", `config:
|
||||
imports:
|
||||
- path: catalog/base.yml
|
||||
required: true
|
||||
`)
|
||||
writeFrameworkSourceFile(t, catalogDir, "base.yml", `profiles:
|
||||
- id: profile-primary
|
||||
endpoint: http://127.0.0.1:1/v1
|
||||
model: imported-model
|
||||
prompts:
|
||||
- id: team-prompt
|
||||
default_profile: profile-primary
|
||||
inputs:
|
||||
- name: source
|
||||
required: true
|
||||
messages:
|
||||
- role: user
|
||||
content_file: ./body.md
|
||||
- role: user
|
||||
content: 'Artifact: {{input "source"}}'
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: response.schema.json
|
||||
`)
|
||||
writeFrameworkSourceFile(t, catalogDir, "body.md", "Captured message for {{.audience}}")
|
||||
artifactPath := filepath.Join(root, "request-artifact.txt")
|
||||
writeFrameworkSourceFile(t, root, "request-artifact.txt", "initial artifact")
|
||||
writeFrameworkSourceFile(t, schemaDir, "response.schema.json", `{"type":"object","title":"initial schema"}`)
|
||||
selections := []appconfig.SourceSelection{{File: mainFile}}
|
||||
engine := newFrameworkSourceTestEngine(t, selections, schemaDir)
|
||||
|
||||
writeFrameworkSourceFile(t, catalogDir, "body.md", "edited message for {{.audience}}")
|
||||
writeFrameworkSourceFile(t, root, "request-artifact.txt", "prepared artifact")
|
||||
writeFrameworkSourceFile(t, schemaDir, "response.schema.json", `{"type":"object","title":"prepared schema"}`)
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "team-prompt",
|
||||
Inputs: map[string]promptkit.ArtifactRef{"source": promptkit.File(artifactPath)},
|
||||
Vars: map[string]string{"audience": "reader"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare inherited imported prompt: %v", err)
|
||||
}
|
||||
if len(prepared.Messages) != 2 || prepared.Messages[0].Content != "Captured message for reader" || prepared.Messages[1].Content != "Artifact: prepared artifact" {
|
||||
t.Fatalf("resource origin or capture/preparation behavior changed: %+v", prepared.Messages)
|
||||
}
|
||||
if prepared.SelectedProfileID != "profile-primary" {
|
||||
t.Fatalf("profile ID was not preserved: %+v", prepared)
|
||||
}
|
||||
if prepared.StructuredOutput == nil || prepared.StructuredOutput.JSONSchema == nil || prepared.StructuredOutput.JSONSchema.Schema.(map[string]any)["title"] != "prepared schema" {
|
||||
t.Fatalf("schema was not loaded during preparation: %+v", prepared.StructuredOutput)
|
||||
}
|
||||
|
||||
newEngine := newFrameworkSourceTestEngine(t, selections, schemaDir)
|
||||
newPrepared, err := newEngine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "team-prompt",
|
||||
Inputs: map[string]promptkit.ArtifactRef{"source": promptkit.File(artifactPath)},
|
||||
Vars: map[string]string{"audience": "reader"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare with newly captured message: %v", err)
|
||||
}
|
||||
if newPrepared.Messages[0].Content != "edited message for reader" {
|
||||
t.Fatalf("new engine did not observe the message edit: %+v", newPrepared.Messages[0])
|
||||
}
|
||||
|
||||
configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\nschema_dir: %q\n", mainFile, schemaDir))
|
||||
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||
"--config", configPath,
|
||||
"--prompt", "team-prompt",
|
||||
"--input", "source=" + artifactPath,
|
||||
"--var", "audience=reader",
|
||||
"--format", "json",
|
||||
})
|
||||
if code != ExitOK {
|
||||
t.Fatalf("render exact literal IDs through CLI: code=%d stderr=%q", code, stderr)
|
||||
}
|
||||
var rendered promptkit.PreparedRun
|
||||
if err := json.Unmarshal([]byte(stdout), &rendered); err != nil {
|
||||
t.Fatalf("decode rendered literal-ID fixture: %v", err)
|
||||
}
|
||||
if rendered.PromptID != "team-prompt" || rendered.SelectedProfileID != "profile-primary" {
|
||||
t.Fatalf("CLI changed exact prompt/profile IDs: %+v", rendered)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeProfileInheritanceAndPresenceReachTheClientBoundary(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
frameworkFile := filepath.Join(root, "framework.yml")
|
||||
writeFrameworkSourceFile(t, root, "framework.yml", `backends:
|
||||
- id: first
|
||||
endpoint: https://first.example/v1
|
||||
api_key_env: FIRST_BACKEND_KEY
|
||||
extra_params:
|
||||
source: first
|
||||
nested:
|
||||
first_backend: true
|
||||
- id: second
|
||||
endpoint: https://second.example/v1
|
||||
api_key_env: SECOND_BACKEND_KEY
|
||||
extra_params:
|
||||
source: second
|
||||
nested:
|
||||
second_backend: true
|
||||
profiles:
|
||||
- id: parent
|
||||
backend: first
|
||||
model: parent-model
|
||||
temperature: 0.7
|
||||
top_p: 0.6
|
||||
max_tokens: 45
|
||||
timeout_seconds: 30
|
||||
service_tier: flex
|
||||
extra_params:
|
||||
nested:
|
||||
parent_profile: true
|
||||
retained: "yes"
|
||||
- id: backend-child
|
||||
extends: {id: parent}
|
||||
backend: second
|
||||
model: child-model
|
||||
- id: presence-child
|
||||
extends: {id: parent}
|
||||
backend: second
|
||||
api_key_env: null
|
||||
temperature: null
|
||||
max_tokens: 0
|
||||
timeout_seconds: 0
|
||||
extra_params:
|
||||
nested:
|
||||
explicit_false: false
|
||||
- id: map-null
|
||||
backend: second
|
||||
model: map-model
|
||||
extra_params: null
|
||||
- id: map-empty
|
||||
backend: second
|
||||
model: map-model
|
||||
extra_params: {}
|
||||
prompts:
|
||||
- id: execution.prompt
|
||||
messages:
|
||||
- role: user
|
||||
content: hello
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)
|
||||
client := &frameworkSourceCaptureClient{}
|
||||
engine := newFrameworkSourceTestEngine(t, []appconfig.SourceSelection{{File: frameworkFile}}, "", promptkit.WithLLMClient(client))
|
||||
|
||||
for _, profileID := range []string{"backend-child", "presence-child", "map-null", "map-empty"} {
|
||||
if _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "execution.prompt", ProfileID: profileID}); err != nil {
|
||||
t.Fatalf("run profile %q: %v", profileID, err)
|
||||
}
|
||||
}
|
||||
if len(client.requests) != 4 {
|
||||
t.Fatalf("expected four provider-bound requests, got %d", len(client.requests))
|
||||
}
|
||||
|
||||
backendChild := client.requests[0].Target
|
||||
wantBackendParams := map[string]any{"source": "second", "nested": map[string]any{
|
||||
"second_backend": true, "parent_profile": true, "retained": "yes",
|
||||
}}
|
||||
if backendChild.BackendID != "second" || backendChild.Endpoint != "https://second.example/v1" || backendChild.APIKeyEnv != "SECOND_BACKEND_KEY" || backendChild.Model != "child-model" ||
|
||||
backendChild.Temperature != 0.7 || backendChild.TopP != 0.6 || backendChild.MaxTokens != 45 || backendChild.TimeoutSeconds != 30 || backendChild.ServiceTier != "flex" || !reflect.DeepEqual(backendChild.ExtraParams, wantBackendParams) {
|
||||
t.Fatalf("backend change lost backend-derived or explicit inherited values: %+v", backendChild)
|
||||
}
|
||||
|
||||
presenceRequest := client.requests[1]
|
||||
presenceTarget := presenceRequest.Target
|
||||
if presenceTarget.APIKeyEnv != "SECOND_BACKEND_KEY" || presenceTarget.Temperature != 0 || presenceTarget.MaxTokens != 0 || presenceTarget.TimeoutSeconds != 0 ||
|
||||
!presenceRequest.TargetPresence.MaxTokens || !presenceRequest.TargetPresence.TimeoutSeconds || presenceRequest.TargetPresence.Temperature || !presenceRequest.TargetPresence.TopP {
|
||||
t.Fatalf("explicit null/zero presence did not reach the injected client: target=%+v presence=%+v", presenceTarget, presenceRequest.TargetPresence)
|
||||
}
|
||||
wantPresenceParams := map[string]any{"source": "second", "nested": map[string]any{
|
||||
"second_backend": true, "parent_profile": true, "retained": "yes", "explicit_false": false,
|
||||
}}
|
||||
if !reflect.DeepEqual(presenceTarget.ExtraParams, wantPresenceParams) {
|
||||
t.Fatalf("recursive provider-map retention changed: %#v", presenceTarget.ExtraParams)
|
||||
}
|
||||
if !reflect.DeepEqual(client.requests[2].Target.ExtraParams, map[string]any{"source": "second", "nested": map[string]any{"second_backend": true}}) {
|
||||
t.Fatalf("whole-map null did not fall back to backend defaults: %#v", client.requests[2].Target.ExtraParams)
|
||||
}
|
||||
if !reflect.DeepEqual(client.requests[3].Target.ExtraParams, map[string]any{"source": "second", "nested": map[string]any{"second_backend": true}}) {
|
||||
t.Fatalf("empty provider map unexpectedly cleared backend defaults: %#v", client.requests[3].Target.ExtraParams)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCLICompletesPartialFrameworkProfileWithRequestOverrides(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
frameworkFile := writeFrameworkSourceFile(t, root, "partial.yml", `profiles:
|
||||
- id: partial
|
||||
prompts:
|
||||
- id: partial.prompt
|
||||
default_profile: partial
|
||||
messages:
|
||||
- role: user
|
||||
content: hello
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)
|
||||
provider := newTestLLMServer("completed partial profile", nil)
|
||||
defer provider.Close()
|
||||
configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n", frameworkFile))
|
||||
code, stdout, stderr := runCLICommand(t, runCommand, []string{
|
||||
"--config", configPath,
|
||||
"--prompt", "partial.prompt",
|
||||
"--llm-base-url", provider.URL + "/v1",
|
||||
"--model", "request-model",
|
||||
})
|
||||
if code != ExitOK || stdout != "completed partial profile" {
|
||||
t.Fatalf("request overrides did not complete the partial profile: code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func newFrameworkSourceTestEngine(t *testing.T, selections []appconfig.SourceSelection, schemaDir string, options ...promptkit.Option) *promptkit.Engine {
|
||||
t.Helper()
|
||||
engine, err := newEngine(engineSettings{promptkitSources: selections, schemaDir: schemaDir}, options...)
|
||||
if err != nil {
|
||||
var loadError *configkit.LoadError
|
||||
if errors.As(err, &loadError) {
|
||||
t.Fatalf("construct engine from selected framework sources: %v (%+v)", err, loadError.Diagnostics())
|
||||
}
|
||||
t.Fatalf("construct engine from selected framework sources: %v", err)
|
||||
}
|
||||
return engine
|
||||
}
|
||||
|
||||
func writeFrameworkSourceFile(t *testing.T, directory, name, contents string) string {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(directory, 0o755); err != nil {
|
||||
t.Fatalf("create framework source directory: %v", err)
|
||||
}
|
||||
path := filepath.Join(directory, name)
|
||||
if err := os.WriteFile(path, []byte(contents), 0o644); err != nil {
|
||||
t.Fatalf("write framework source %q: %v", path, err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
type frameworkSourceCaptureClient struct {
|
||||
requests []promptkit.GenerateRequest
|
||||
}
|
||||
|
||||
func (client *frameworkSourceCaptureClient) Generate(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
client.requests = append(client.requests, request)
|
||||
return &promptkit.GenerateResponse{Content: "ok"}, nil
|
||||
}
|
||||
@@ -1,212 +0,0 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
appformat "gitea.maximumdirect.net/eric/scriptorium/internal/format"
|
||||
"go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
type promptInspectionConfig struct {
|
||||
configPath, promptID, schemaDir, outputPath string
|
||||
configOverlay string
|
||||
configExplicit bool
|
||||
outputFormat appformat.OutputFormat
|
||||
overrides []configkit.Override
|
||||
diagnostics []configkit.Diagnostic
|
||||
}
|
||||
|
||||
type profileInspectionConfig struct {
|
||||
configPath, profileID, schemaDir, outputPath string
|
||||
configOverlay string
|
||||
configExplicit bool
|
||||
outputFormat appformat.OutputFormat
|
||||
overrides []configkit.Override
|
||||
diagnostics []configkit.Diagnostic
|
||||
}
|
||||
|
||||
func inspectCommand(args []string, stdout, stderr io.Writer) int {
|
||||
if len(args) == 0 {
|
||||
fmt.Fprintln(stderr, "inspect parse error: inspection mode is required")
|
||||
return ExitRuntimeError
|
||||
}
|
||||
if args[0] == "profile" {
|
||||
return inspectProfileCommand(args[1:], stdout, stderr)
|
||||
}
|
||||
if args[0] == "configuration" {
|
||||
return inspectFrameworkConfigurationCommand(args[1:], stdout, stderr)
|
||||
}
|
||||
if args[0] != "prompt" {
|
||||
fmt.Fprintln(stderr, "inspect parse error: unknown inspection mode")
|
||||
return ExitRuntimeError
|
||||
}
|
||||
cfg, err := parsePromptInspectionArgs(args[1:])
|
||||
if err != nil {
|
||||
writeApplicationError(stderr, "inspect parse", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
settings, err := resolveAppSettingsForPromptInspection(cfg)
|
||||
if err != nil {
|
||||
writeApplicationError(stderr, "inspect", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
if err := writeApplicationWarnings(stderr, cfg.diagnostics); err != nil {
|
||||
return ExitRuntimeError
|
||||
}
|
||||
engine, err := newEngine(settings)
|
||||
if err != nil {
|
||||
writeApplicationError(stderr, "engine", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
if err := writeFrameworkDiagnostics(stderr, engine); err != nil {
|
||||
fmt.Fprintf(stderr, "engine diagnostic write error: %v\n", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
inspection, err := engine.InspectPrompt(context.Background(), cfg.promptID)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "inspect error: %v\n", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
data, err := appformat.FormatPromptInspection(inspection, cfg.outputFormat)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "inspect error: %v\n", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
if err := writeOutput(stdout, cfg.outputPath, data); err != nil {
|
||||
fmt.Fprintf(stderr, "output write error: %v\n", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
return ExitOK
|
||||
}
|
||||
|
||||
func inspectProfileCommand(args []string, stdout, stderr io.Writer) int {
|
||||
cfg, err := parseProfileInspectionArgs(args)
|
||||
if err != nil {
|
||||
writeApplicationError(stderr, "inspect parse", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
settings, err := resolveAppSettingsForProfileInspection(cfg)
|
||||
if err != nil {
|
||||
writeApplicationError(stderr, "inspect", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
if err := writeApplicationWarnings(stderr, cfg.diagnostics); err != nil {
|
||||
return ExitRuntimeError
|
||||
}
|
||||
engine, err := newEngine(settings)
|
||||
if err != nil {
|
||||
writeApplicationError(stderr, "engine", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
if err := writeFrameworkDiagnostics(stderr, engine); err != nil {
|
||||
fmt.Fprintf(stderr, "engine diagnostic write error: %v\n", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
inspection, err := engine.InspectProfile(context.Background(), cfg.profileID)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "inspect error: %v\n", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
data, err := appformat.FormatProfileInspection(inspection, cfg.outputFormat)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "inspect error: %v\n", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
if err := writeOutput(stdout, cfg.outputPath, data); err != nil {
|
||||
fmt.Fprintf(stderr, "output write error: %v\n", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
return ExitOK
|
||||
}
|
||||
|
||||
func parseProfileInspectionArgs(args []string) (*profileInspectionConfig, error) {
|
||||
cfg := &profileInspectionConfig{outputFormat: appformat.DefaultOutputFormat}
|
||||
fs := flag.NewFlagSet("inspect profile", flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
registerConfigSelectionFlags(fs, &cfg.configPath, &cfg.configOverlay)
|
||||
var promptkitSources promptKitSourceFlagValue
|
||||
fs.Var(&promptkitSources, "promptkit-source", "selected PromptKit source path (repeatable)")
|
||||
fs.StringVar(&cfg.schemaDir, "schema-dir", "", "base directory for validation schemas")
|
||||
fs.StringVar(&cfg.profileID, "profile", "", "profile ID to inspect")
|
||||
rawFormat := ""
|
||||
fs.StringVar(&rawFormat, "format", "", "output format: text or json")
|
||||
fs.StringVar(&cfg.outputPath, "out", "", "optional output file path")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cfg.configExplicit = flagWasSet(fs, "config")
|
||||
cfg.overrides = applicationOverrides(fs)
|
||||
if fs.NArg() > 0 {
|
||||
return nil, fmt.Errorf("unexpected positional args: %v", fs.Args())
|
||||
}
|
||||
if strings.TrimSpace(cfg.profileID) == "" {
|
||||
return nil, errors.New("--profile is required")
|
||||
}
|
||||
format, err := appformat.ParseOutputFormat(rawFormat)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cfg.outputFormat = format
|
||||
if cfg.outputPath != "" {
|
||||
cfg.outputPath = filepath.Clean(cfg.outputPath)
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func resolveAppSettingsForProfileInspection(cfg *profileInspectionConfig) (engineSettings, error) {
|
||||
settings, diagnostics, err := loadApplicationSettings(cfg.configPath, cfg.configOverlay, cfg.configExplicit, cfg.overrides)
|
||||
if err != nil {
|
||||
return engineSettings{}, err
|
||||
}
|
||||
cfg.diagnostics = diagnostics
|
||||
return engineSettings{promptkitSources: settings.PromptKitSources, schemaDir: settings.SchemaDir, transientRetries: settings.TransientRetries}, nil
|
||||
}
|
||||
|
||||
func parsePromptInspectionArgs(args []string) (*promptInspectionConfig, error) {
|
||||
cfg := &promptInspectionConfig{outputFormat: appformat.DefaultOutputFormat}
|
||||
fs := flag.NewFlagSet("inspect prompt", flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
registerConfigSelectionFlags(fs, &cfg.configPath, &cfg.configOverlay)
|
||||
var promptkitSources promptKitSourceFlagValue
|
||||
fs.Var(&promptkitSources, "promptkit-source", "selected PromptKit source path (repeatable)")
|
||||
fs.StringVar(&cfg.schemaDir, "schema-dir", "", "base directory for validation schemas")
|
||||
fs.StringVar(&cfg.promptID, "prompt", "", "prompt ID to inspect")
|
||||
rawFormat := ""
|
||||
fs.StringVar(&rawFormat, "format", "", "output format: text or json")
|
||||
fs.StringVar(&cfg.outputPath, "out", "", "optional output file path")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cfg.configExplicit = flagWasSet(fs, "config")
|
||||
cfg.overrides = applicationOverrides(fs)
|
||||
if fs.NArg() > 0 {
|
||||
return nil, fmt.Errorf("unexpected positional args: %v", fs.Args())
|
||||
}
|
||||
if strings.TrimSpace(cfg.promptID) == "" {
|
||||
return nil, errors.New("--prompt is required")
|
||||
}
|
||||
format, err := appformat.ParseOutputFormat(rawFormat)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cfg.outputFormat = format
|
||||
if cfg.outputPath != "" {
|
||||
cfg.outputPath = filepath.Clean(cfg.outputPath)
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func resolveAppSettingsForPromptInspection(cfg *promptInspectionConfig) (engineSettings, error) {
|
||||
settings, diagnostics, err := loadApplicationSettings(cfg.configPath, cfg.configOverlay, cfg.configExplicit, cfg.overrides)
|
||||
if err != nil {
|
||||
return engineSettings{}, err
|
||||
}
|
||||
cfg.diagnostics = diagnostics
|
||||
return engineSettings{promptkitSources: settings.PromptKitSources, schemaDir: settings.SchemaDir, transientRetries: settings.TransientRetries}, nil
|
||||
}
|
||||
@@ -1,636 +0,0 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
httpadapter "gitea.maximumdirect.net/eric/scriptorium/internal/adapter/http"
|
||||
appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config"
|
||||
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
||||
renderformat "gitea.maximumdirect.net/eric/scriptorium/internal/format"
|
||||
"go.maximumdirect.net/commandkit"
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
const (
|
||||
ExitOK = 0
|
||||
ExitRuntimeError = 1
|
||||
ExitValidationFailed = 2
|
||||
)
|
||||
|
||||
type runConfig struct {
|
||||
configPath string
|
||||
|
||||
promptkitSources []appconfig.SourceSelection
|
||||
promptID string
|
||||
profileID string
|
||||
sessionID string
|
||||
inputRaw listFlag
|
||||
varRaw listFlag
|
||||
outputPath string
|
||||
executionReportPath string
|
||||
llmBaseURL string
|
||||
apiKeyEnv string
|
||||
model string
|
||||
temperature float64
|
||||
maxTokens int
|
||||
topP float64
|
||||
reasoningEffort string
|
||||
schemaDir string
|
||||
timeout time.Duration
|
||||
runTimeout time.Duration
|
||||
transientRetries int
|
||||
configuredTransientRetries *int
|
||||
diagnostics []configkit.Diagnostic
|
||||
configOverlay string
|
||||
|
||||
defaultRenderFormat renderformat.PreparedRunOutputFormat
|
||||
|
||||
llmBaseURLSet bool
|
||||
apiKeyEnvSet bool
|
||||
modelSet bool
|
||||
temperatureSet bool
|
||||
maxTokensSet bool
|
||||
topPSet bool
|
||||
reasoningEffortSet bool
|
||||
timeoutSet bool
|
||||
transientRetriesSet bool
|
||||
}
|
||||
|
||||
type renderConfig struct {
|
||||
runConfig
|
||||
outputFormat renderformat.PreparedRunOutputFormat
|
||||
}
|
||||
|
||||
type serveConfig struct {
|
||||
configPath string
|
||||
|
||||
addr string
|
||||
promptkitSources []appconfig.SourceSelection
|
||||
schemaDir string
|
||||
artifactRoot string
|
||||
maxRequestBytes int64
|
||||
maxArtifactBytes int64
|
||||
maxResponseBytes int64
|
||||
transientRetries *int
|
||||
diagnostics []configkit.Diagnostic
|
||||
configOverlay string
|
||||
}
|
||||
|
||||
type commonCommandSettings struct {
|
||||
promptkitSources []appconfig.SourceSelection
|
||||
schemaDir string
|
||||
serverAddr string
|
||||
artifactRoot string
|
||||
maxRequestBytes int64
|
||||
maxArtifactBytes int64
|
||||
maxResponseBytes int64
|
||||
defaultRenderFormat renderformat.PreparedRunOutputFormat
|
||||
transientRetries *int
|
||||
diagnostics []configkit.Diagnostic
|
||||
}
|
||||
|
||||
type listFlag []string
|
||||
|
||||
func (l *listFlag) String() string {
|
||||
return strings.Join(*l, ",")
|
||||
}
|
||||
|
||||
func (l *listFlag) Set(v string) error {
|
||||
*l = append(*l, v)
|
||||
return nil
|
||||
}
|
||||
|
||||
func Run(args []string, stdout, stderr io.Writer) int {
|
||||
return Execute(context.Background(), commandkit.Identity{Name: "scriptorium", Version: "dev"}, args, stdout, stderr).Status
|
||||
}
|
||||
|
||||
func runCommand(args []string, stdout, stderr io.Writer) int {
|
||||
cfg, err := parseRunArgs(args)
|
||||
if err != nil {
|
||||
writeApplicationError(stderr, "run parse", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
if err := writeApplicationWarnings(stderr, cfg.diagnostics); err != nil {
|
||||
return ExitRuntimeError
|
||||
}
|
||||
|
||||
req, err := buildRunRequestFromConfig(cfg)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "run parse error: %v\n", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
|
||||
engine, err := newEngine(cfg.engineSettings())
|
||||
if err != nil {
|
||||
writeApplicationError(stderr, "engine", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
if err := writeFrameworkDiagnostics(stderr, engine); err != nil {
|
||||
fmt.Fprintf(stderr, "engine diagnostic write error: %v\n", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
return executeRun(cfg, req, engine.Run, stdout, stderr)
|
||||
}
|
||||
|
||||
|
||||
func renderCommand(args []string, stdout, stderr io.Writer) int {
|
||||
cfg, err := parseRenderArgs(args)
|
||||
if err != nil {
|
||||
writeApplicationError(stderr, "render parse", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
if err := writeApplicationWarnings(stderr, cfg.diagnostics); err != nil {
|
||||
return ExitRuntimeError
|
||||
}
|
||||
|
||||
req, err := buildRunRequestFromConfig(&cfg.runConfig)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "render parse error: %v\n", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
|
||||
engine, err := newEngine(cfg.runConfig.engineSettings())
|
||||
if err != nil {
|
||||
writeApplicationError(stderr, "engine", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
if err := writeFrameworkDiagnostics(stderr, engine); err != nil {
|
||||
fmt.Fprintf(stderr, "engine diagnostic write error: %v\n", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
|
||||
prepared, prepErr := engine.Prepare(context.Background(), req)
|
||||
if prepErr != nil {
|
||||
fmt.Fprintf(stderr, "render error: %v\n", prepErr)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
|
||||
out, err := renderformat.FormatPreparedRun(prepared, cfg.outputFormat)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "render format error: %v\n", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
|
||||
if err := writeOutput(stdout, cfg.outputPath, out); err != nil {
|
||||
fmt.Fprintf(stderr, "output write error: %v\n", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
return ExitOK
|
||||
}
|
||||
|
||||
func serveCommand(args []string, stderr io.Writer) int {
|
||||
cfg, err := parseServeArgs(args)
|
||||
if err != nil {
|
||||
writeApplicationError(stderr, "serve parse", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
if err := writeApplicationWarnings(stderr, cfg.diagnostics); err != nil {
|
||||
return ExitRuntimeError
|
||||
}
|
||||
|
||||
artifactReader, err := httpadapter.NewRestrictedArtifactReader(cfg.artifactRoot, cfg.maxArtifactBytes)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "artifact root error: %v\n", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
|
||||
engine, err := newEngine(cfg.engineSettings(), promptkit.WithArtifactReader(artifactReader))
|
||||
if err != nil {
|
||||
writeApplicationError(stderr, "engine", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
if err := writeFrameworkDiagnostics(stderr, engine); err != nil {
|
||||
fmt.Fprintf(stderr, "engine diagnostic write error: %v\n", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
|
||||
h := httpadapter.NewHandlerWithOptions(engine, httpadapter.HandlerOptions{
|
||||
MaxRequestBytes: cfg.maxRequestBytes,
|
||||
MaxResponseBytes: cfg.maxResponseBytes,
|
||||
})
|
||||
srv := &http.Server{
|
||||
Addr: cfg.addr,
|
||||
Handler: h,
|
||||
ReadHeaderTimeout: defaults.HTTPReadHeaderTimeoutDefault,
|
||||
}
|
||||
|
||||
fmt.Fprintf(stderr, "serving on %s\n", cfg.addr)
|
||||
if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
fmt.Fprintf(stderr, "server error: %v\n", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
return ExitOK
|
||||
}
|
||||
|
||||
func parseRunArgs(args []string) (*runConfig, error) {
|
||||
cfg := &runConfig{}
|
||||
fs := flag.NewFlagSet("run", flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
|
||||
registerConfigSelectionFlags(fs, &cfg.configPath, &cfg.configOverlay)
|
||||
registerExecutionRequestFlags(fs, cfg)
|
||||
fs.DurationVar(&cfg.runTimeout, "run-timeout", 0, "whole-run execution deadline")
|
||||
fs.StringVar(&cfg.executionReportPath, "execution-report", "", "JSON execution report file path")
|
||||
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := finalizeExecutionRequestConfig(fs, cfg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if cfg.runTimeout < 0 {
|
||||
return nil, errors.New("--run-timeout must be greater than or equal to 0")
|
||||
}
|
||||
if flagWasSet(fs, "execution-report") {
|
||||
if cfg.executionReportPath == "" {
|
||||
return nil, errors.New("--execution-report requires a nonempty path")
|
||||
}
|
||||
cfg.executionReportPath = filepath.Clean(cfg.executionReportPath)
|
||||
if cfg.outputPath != "" {
|
||||
collision, err := outputPathsCollide(cfg.outputPath, cfg.executionReportPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if collision {
|
||||
return nil, errors.New("--out and --execution-report select the same file")
|
||||
}
|
||||
}
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func outputPathsCollide(first, second string) (bool, error) {
|
||||
firstAbs, err := filepath.Abs(first)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
secondAbs, err := filepath.Abs(second)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if firstAbs == secondAbs {
|
||||
return true, nil
|
||||
}
|
||||
firstInfo, firstErr := os.Stat(firstAbs)
|
||||
secondInfo, secondErr := os.Stat(secondAbs)
|
||||
return firstErr == nil && secondErr == nil && os.SameFile(firstInfo, secondInfo), nil
|
||||
}
|
||||
|
||||
func parseRenderArgs(args []string) (*renderConfig, error) {
|
||||
cfg := &renderConfig{
|
||||
outputFormat: renderformat.DefaultPreparedRunOutputFormat,
|
||||
}
|
||||
fs := flag.NewFlagSet("render", flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
|
||||
registerConfigSelectionFlags(fs, &cfg.configPath, &cfg.configOverlay)
|
||||
registerExecutionRequestFlags(fs, &cfg.runConfig)
|
||||
|
||||
var rawFormat string
|
||||
fs.StringVar(&rawFormat, "format", "", "render output format (text|json)")
|
||||
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := finalizeExecutionRequestConfig(fs, &cfg.runConfig); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if flagWasSet(fs, "format") {
|
||||
format, err := renderformat.ParsePreparedRunOutputFormat(rawFormat)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cfg.outputFormat = format
|
||||
} else {
|
||||
cfg.outputFormat = cfg.defaultRenderFormat
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func parseServeArgs(args []string) (*serveConfig, error) {
|
||||
cfg := &serveConfig{}
|
||||
fs := flag.NewFlagSet("serve", flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
|
||||
registerConfigSelectionFlags(fs, &cfg.configPath, &cfg.configOverlay)
|
||||
fs.StringVar(&cfg.addr, "addr", "", "HTTP listen address")
|
||||
fs.StringVar(&cfg.artifactRoot, "artifact-root", "", "base directory for HTTP file input artifacts")
|
||||
fs.Int64Var(&cfg.maxRequestBytes, "max-request-bytes", 0, "maximum HTTP request body bytes; 0 disables the limit")
|
||||
fs.Int64Var(&cfg.maxArtifactBytes, "max-artifact-bytes", 0, "maximum HTTP file artifact bytes; 0 disables the limit")
|
||||
fs.Int64Var(&cfg.maxResponseBytes, "max-response-bytes", 0, "maximum HTTP response body bytes; 0 disables the limit")
|
||||
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if fs.NArg() > 0 {
|
||||
return nil, fmt.Errorf("unexpected positional args: %v", fs.Args())
|
||||
}
|
||||
|
||||
settings, err := resolveCommonSettings(fs, cfg.configPath, cfg.configOverlay)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cfg.promptkitSources = settings.promptkitSources
|
||||
cfg.schemaDir = settings.schemaDir
|
||||
cfg.addr = settings.serverAddr
|
||||
cfg.artifactRoot = settings.artifactRoot
|
||||
cfg.maxRequestBytes = settings.maxRequestBytes
|
||||
cfg.maxArtifactBytes = settings.maxArtifactBytes
|
||||
cfg.maxResponseBytes = settings.maxResponseBytes
|
||||
cfg.transientRetries = settings.transientRetries
|
||||
cfg.diagnostics = settings.diagnostics
|
||||
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func registerExecutionRequestFlags(fs *flag.FlagSet, cfg *runConfig) {
|
||||
fs.StringVar(&cfg.promptID, "prompt", "", "prompt ID to run")
|
||||
fs.StringVar(&cfg.profileID, "profile", "", "optional execution profile ID; if omitted, prompt default_profile is used")
|
||||
fs.StringVar(&cfg.sessionID, "session-id", "", "optional session ID")
|
||||
fs.Var(&cfg.inputRaw, "input", "input mapping(s): name=path (repeatable, comma-separated)")
|
||||
fs.Var(&cfg.varRaw, "var", "variable mapping(s): name=value (repeatable, comma-separated)")
|
||||
fs.StringVar(&cfg.outputPath, "out", "", "optional output file path")
|
||||
fs.StringVar(&cfg.llmBaseURL, "llm-base-url", "", "OpenAI-compatible base URL including /v1")
|
||||
fs.StringVar(&cfg.apiKeyEnv, "api-key-env", "", "environment variable name containing API key")
|
||||
fs.StringVar(&cfg.model, "model", "", "model name")
|
||||
fs.Float64Var(&cfg.temperature, "temperature", 0, "optional temperature override")
|
||||
fs.IntVar(&cfg.maxTokens, "max-tokens", 0, "optional max tokens override")
|
||||
fs.Float64Var(&cfg.topP, "top-p", 0, "optional top_p override")
|
||||
fs.StringVar(&cfg.reasoningEffort, "reasoning-effort", "", "optional reasoning effort override")
|
||||
fs.DurationVar(&cfg.timeout, "timeout", 0, "LLM request timeout")
|
||||
fs.IntVar(&cfg.transientRetries, "transient-retries", 0, "additional transient generation attempts")
|
||||
fs.StringVar(&cfg.promptID, "prompt-id", "", "deprecated alias for --prompt")
|
||||
fs.StringVar(&cfg.profileID, "profile-id", "", "deprecated alias for --profile")
|
||||
}
|
||||
|
||||
func finalizeExecutionRequestConfig(fs *flag.FlagSet, cfg *runConfig) error {
|
||||
if fs.NArg() > 0 {
|
||||
return fmt.Errorf("unexpected positional args: %v", fs.Args())
|
||||
}
|
||||
|
||||
settings, err := resolveCommonSettings(fs, cfg.configPath, cfg.configOverlay)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cfg.promptkitSources = settings.promptkitSources
|
||||
cfg.schemaDir = settings.schemaDir
|
||||
cfg.defaultRenderFormat = settings.defaultRenderFormat
|
||||
cfg.configuredTransientRetries = settings.transientRetries
|
||||
cfg.diagnostics = settings.diagnostics
|
||||
|
||||
if strings.TrimSpace(cfg.promptID) == "" {
|
||||
return errors.New("--prompt is required")
|
||||
}
|
||||
if cfg.outputPath != "" {
|
||||
cfg.outputPath = filepath.Clean(cfg.outputPath)
|
||||
}
|
||||
cfg.llmBaseURLSet = flagWasSet(fs, "llm-base-url")
|
||||
cfg.apiKeyEnvSet = flagWasSet(fs, "api-key-env")
|
||||
cfg.modelSet = flagWasSet(fs, "model")
|
||||
cfg.temperatureSet = flagWasSet(fs, "temperature")
|
||||
cfg.maxTokensSet = flagWasSet(fs, "max-tokens")
|
||||
cfg.topPSet = flagWasSet(fs, "top-p")
|
||||
cfg.reasoningEffortSet = flagWasSet(fs, "reasoning-effort")
|
||||
cfg.timeoutSet = flagWasSet(fs, "timeout")
|
||||
cfg.transientRetriesSet = flagWasSet(fs, "transient-retries")
|
||||
if cfg.transientRetriesSet && cfg.transientRetries < 0 {
|
||||
return errors.New("--transient-retries must be greater than or equal to 0")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func registerConfigSelectionFlags(fs *flag.FlagSet, target, overlay *string) {
|
||||
fs.StringVar(
|
||||
target,
|
||||
"config",
|
||||
"",
|
||||
fmt.Sprintf(
|
||||
"application config path (default search: %s, then %s)",
|
||||
appconfig.DefaultConfigPathLocal,
|
||||
appconfig.DefaultConfigPath,
|
||||
),
|
||||
)
|
||||
fs.StringVar(overlay, "config-overlay", "", "selected application configuration overlay")
|
||||
}
|
||||
|
||||
func resolveCommonSettings(fs *flag.FlagSet, configPath, overlay string) (commonCommandSettings, error) {
|
||||
settings, diagnostics, err := loadApplicationSettings(configPath, overlay, flagWasSet(fs, "config"), applicationOverrides(fs))
|
||||
if err != nil {
|
||||
return commonCommandSettings{}, err
|
||||
}
|
||||
return commonCommandSettings{
|
||||
promptkitSources: settings.PromptKitSources,
|
||||
schemaDir: settings.SchemaDir,
|
||||
serverAddr: settings.ServerAddr,
|
||||
artifactRoot: settings.ArtifactRoot,
|
||||
maxRequestBytes: settings.MaxRequestBytes,
|
||||
maxArtifactBytes: settings.MaxArtifactBytes,
|
||||
maxResponseBytes: settings.MaxResponseBytes,
|
||||
defaultRenderFormat: renderformat.DefaultPreparedRunOutputFormat,
|
||||
transientRetries: settings.TransientRetries,
|
||||
diagnostics: diagnostics,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type engineSettings struct {
|
||||
promptkitSources []appconfig.SourceSelection
|
||||
schemaDir string
|
||||
transientRetries *int
|
||||
}
|
||||
|
||||
func (c runConfig) engineSettings() engineSettings {
|
||||
return engineSettings{promptkitSources: c.promptkitSources, schemaDir: c.schemaDir, transientRetries: c.configuredTransientRetries}
|
||||
}
|
||||
|
||||
func (c serveConfig) engineSettings() engineSettings {
|
||||
return engineSettings{promptkitSources: c.promptkitSources, schemaDir: c.schemaDir, transientRetries: c.transientRetries}
|
||||
}
|
||||
|
||||
|
||||
func buildRunRequestFromConfig(cfg *runConfig) (promptkit.RunRequest, error) {
|
||||
var inputMappings map[string]string
|
||||
var err error
|
||||
if len(cfg.inputRaw) > 0 {
|
||||
inputMappings, err = parseMappings(cfg.inputRaw, false)
|
||||
if err != nil {
|
||||
return promptkit.RunRequest{}, fmt.Errorf("input parse error: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
varMappings := map[string]string{}
|
||||
if len(cfg.varRaw) > 0 {
|
||||
varMappings, err = parseMappings(cfg.varRaw, false)
|
||||
if err != nil {
|
||||
return promptkit.RunRequest{}, fmt.Errorf("var parse error: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
var inputs map[string]promptkit.ArtifactRef
|
||||
if len(inputMappings) > 0 {
|
||||
inputs = make(map[string]promptkit.ArtifactRef, len(inputMappings))
|
||||
for name, path := range inputMappings {
|
||||
inputs[name] = promptkit.File(path)
|
||||
}
|
||||
}
|
||||
|
||||
var modelOverride *promptkit.ExecutionTargetOverride
|
||||
if cfg.llmBaseURLSet || cfg.modelSet || cfg.temperatureSet || cfg.maxTokensSet || cfg.topPSet || cfg.reasoningEffortSet || cfg.apiKeyEnvSet || cfg.timeoutSet {
|
||||
modelOverride = &promptkit.ExecutionTargetOverride{
|
||||
Endpoint: cfg.llmBaseURL,
|
||||
Model: cfg.model,
|
||||
APIKeyEnv: cfg.apiKeyEnv,
|
||||
}
|
||||
if cfg.temperatureSet {
|
||||
modelOverride.Temperature = &cfg.temperature
|
||||
}
|
||||
if cfg.maxTokensSet {
|
||||
modelOverride.MaxTokens = &cfg.maxTokens
|
||||
}
|
||||
if cfg.topPSet {
|
||||
modelOverride.TopP = &cfg.topP
|
||||
}
|
||||
if cfg.reasoningEffortSet {
|
||||
modelOverride.ReasoningEffort = &cfg.reasoningEffort
|
||||
}
|
||||
if cfg.timeoutSet {
|
||||
timeoutSeconds := int(cfg.timeout.Seconds())
|
||||
modelOverride.TimeoutSeconds = &timeoutSeconds
|
||||
}
|
||||
}
|
||||
|
||||
var transientRetries *int
|
||||
if cfg.transientRetriesSet {
|
||||
transientRetries = &cfg.transientRetries
|
||||
}
|
||||
return promptkit.RunRequest{
|
||||
PromptID: cfg.promptID,
|
||||
ProfileID: cfg.profileID,
|
||||
SessionID: cfg.sessionID,
|
||||
Inputs: inputs,
|
||||
Vars: varMappings,
|
||||
Execution: modelOverride,
|
||||
TransientRetries: transientRetries,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func parseMappings(raw []string, allowEmptyValue bool) (map[string]string, error) {
|
||||
out := make(map[string]string)
|
||||
for _, entry := range raw {
|
||||
for _, piece := range strings.Split(entry, ",") {
|
||||
piece = strings.TrimSpace(piece)
|
||||
if piece == "" {
|
||||
continue
|
||||
}
|
||||
key, value, err := parseMapping(piece)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !allowEmptyValue && strings.TrimSpace(value) == "" {
|
||||
return nil, fmt.Errorf("mapping %q has empty value", piece)
|
||||
}
|
||||
out[key] = value
|
||||
}
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil, errors.New("no valid mappings provided")
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func parseMapping(value string) (string, string, error) {
|
||||
idx := strings.IndexRune(value, '=')
|
||||
if idx <= 0 {
|
||||
return "", "", fmt.Errorf("invalid mapping %q, expected name=value", value)
|
||||
}
|
||||
key := strings.TrimSpace(value[:idx])
|
||||
val := strings.TrimSpace(value[idx+1:])
|
||||
if key == "" {
|
||||
return "", "", fmt.Errorf("invalid mapping %q, empty name", value)
|
||||
}
|
||||
return key, val, nil
|
||||
}
|
||||
|
||||
func flagWasSet(fs *flag.FlagSet, name string) bool {
|
||||
set := false
|
||||
fs.Visit(func(f *flag.Flag) {
|
||||
if f.Name == name {
|
||||
set = true
|
||||
}
|
||||
})
|
||||
return set
|
||||
}
|
||||
|
||||
func writeOutput(stdout io.Writer, outputPath string, body []byte) error {
|
||||
if outputPath == "" {
|
||||
_, err := stdout.Write(body)
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(outputPath, body, 0644)
|
||||
}
|
||||
|
||||
func determineExitCode(runErr error, result *promptkit.RunResult) int {
|
||||
if runErr != nil {
|
||||
return ExitRuntimeError
|
||||
}
|
||||
if result != nil && result.Validation.Status == promptkit.ValidationFailed {
|
||||
return ExitValidationFailed
|
||||
}
|
||||
return ExitOK
|
||||
}
|
||||
|
||||
func printSummary(stderr io.Writer, res *promptkit.RunResult) {
|
||||
if res == nil {
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(stderr, "prompt=%s selected_profile=%s model=%s validation=%s mode=%s validation_errors=%d prompt_hash=%s inputs=%d usage=%d/%d/%d",
|
||||
res.PromptID,
|
||||
res.SelectedProfileID,
|
||||
res.ModelName,
|
||||
res.Validation.Status,
|
||||
res.Validation.Mode,
|
||||
len(res.Validation.Errors),
|
||||
res.RenderedPromptHash,
|
||||
len(res.InputHashes),
|
||||
res.Usage.PromptTokens,
|
||||
res.Usage.CompletionTokens,
|
||||
res.Usage.TotalTokens,
|
||||
)
|
||||
if res.Usage.CachedTokens != 0 || res.Usage.CacheWriteTokens != 0 {
|
||||
fmt.Fprintf(stderr, " cached_tokens=%d cache_write_tokens=%d", res.Usage.CachedTokens, res.Usage.CacheWriteTokens)
|
||||
}
|
||||
if res.SelectedBackendID != "" {
|
||||
fmt.Fprintf(stderr, " backend=%s", res.SelectedBackendID)
|
||||
}
|
||||
fmt.Fprintln(stderr)
|
||||
}
|
||||
|
||||
func runErrorMessage(err error) string {
|
||||
if errors.Is(err, promptkit.ErrCapacityExceeded) {
|
||||
return "run error: model backend capacity is exhausted"
|
||||
}
|
||||
return fmt.Sprintf("run error: %v", err)
|
||||
}
|
||||
|
||||
func printUsage(w io.Writer) {
|
||||
fmt.Fprintln(w, "usage: scriptorium <run|render|serve|inspect|config> ...")
|
||||
fmt.Fprintln(w, " run: scriptorium run [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--schema-dir DIR] --prompt ID [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--out path] [--execution-report path] [--timeout 10m] [--run-timeout 10m] [--transient-retries N]")
|
||||
fmt.Fprintln(w, " render: scriptorium render [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--schema-dir DIR] --prompt ID [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--format text|json] [--out path] [--timeout 10m] [--transient-retries N]")
|
||||
fmt.Fprintf(w, " serve: scriptorium serve [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--addr %s] [--schema-dir DIR] [--artifact-root DIR] [--max-request-bytes N] [--max-artifact-bytes N] [--max-response-bytes N]\n", defaults.HTTPAddrDefault)
|
||||
fmt.Fprintln(w, " inspect prompt: scriptorium inspect prompt --prompt ID [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--schema-dir DIR] [--format text|json] [--out path]")
|
||||
fmt.Fprintln(w, " inspect profile: scriptorium inspect profile --profile ID [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--schema-dir DIR] [--format text|json] [--out path]")
|
||||
fmt.Fprintln(w, " inspect configuration: scriptorium inspect configuration <validate|show|sources|explain> [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--schema-dir DIR] [--path JSON_POINTER (required for explain)]")
|
||||
fmt.Fprintln(w, " config: scriptorium config <validate|show|sources|explain> [--config PATH] [--config-overlay NAME] [application overrides] [--path /pointer for explain]")
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -153,7 +153,7 @@ import (
|
||||
_ "gitea.maximumdirect.net/eric/scriptorium/internal/adapter/http"
|
||||
_ "gitea.maximumdirect.net/eric/scriptorium/internal/config"
|
||||
_ "gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
||||
_ "gitea.maximumdirect.net/eric/scriptorium/internal/format"
|
||||
_ "gitea.maximumdirect.net/eric/scriptorium/internal/report"
|
||||
)
|
||||
`
|
||||
if err := os.WriteFile(sourcePath, []byte(source), 0o644); err != nil {
|
||||
|
||||
@@ -1,179 +0,0 @@
|
||||
package format
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
type PromptInspection struct {
|
||||
PromptID string `json:"prompt_id"`
|
||||
PromptHash string `json:"prompt_hash"`
|
||||
DefaultProfileID *string `json:"default_profile_id,omitempty"`
|
||||
Inputs *[]PromptInspectionInput `json:"inputs,omitempty"`
|
||||
OutputContract OutputContract `json:"output_contract"`
|
||||
}
|
||||
|
||||
type PromptInspectionInput struct {
|
||||
Name string `json:"name"`
|
||||
Required *bool `json:"required,omitempty"`
|
||||
ContentType *string `json:"content_type,omitempty"`
|
||||
Description *string `json:"description,omitempty"`
|
||||
}
|
||||
|
||||
type OutputContract struct {
|
||||
Format string `json:"format"`
|
||||
ValidationMode string `json:"validation_mode"`
|
||||
SchemaPath *string `json:"schema_path,omitempty"`
|
||||
RepairAttempts *int `json:"repair_attempts,omitempty"`
|
||||
}
|
||||
|
||||
type ProfileInspection struct {
|
||||
ProfileID string `json:"profile_id"`
|
||||
ProfileModelParams ProfileModelParams `json:"profile_model_params"`
|
||||
APIKeyRequired *bool `json:"api_key_required,omitempty"`
|
||||
}
|
||||
|
||||
type ProfileModelParams struct {
|
||||
BackendID *string `json:"backend_id,omitempty"`
|
||||
Endpoint *string `json:"endpoint,omitempty"`
|
||||
Model *string `json:"model,omitempty"`
|
||||
Temperature *float64 `json:"temperature,omitempty"`
|
||||
MaxTokens *int `json:"max_tokens,omitempty"`
|
||||
TopP *float64 `json:"top_p,omitempty"`
|
||||
TimeoutSeconds *int `json:"timeout_seconds,omitempty"`
|
||||
ServiceTier *string `json:"service_tier,omitempty"`
|
||||
ReasoningEffort *string `json:"reasoning_effort,omitempty"`
|
||||
APIKeyEnv *string `json:"api_key_env,omitempty"`
|
||||
ExtraParams *map[string]any `json:"extra_params,omitempty"`
|
||||
}
|
||||
|
||||
func FormatPromptInspection(value *promptkit.PromptInspection, outputFormat OutputFormat) ([]byte, error) {
|
||||
if value == nil {
|
||||
return nil, errors.New("prompt inspection is nil")
|
||||
}
|
||||
dto := PromptInspection{
|
||||
PromptID: value.PromptID, PromptHash: value.PromptHash, DefaultProfileID: value.DefaultProfileID,
|
||||
OutputContract: OutputContract{Format: string(value.OutputContract.Format), ValidationMode: string(value.OutputContract.ValidationMode), SchemaPath: value.OutputContract.SchemaPath, RepairAttempts: value.OutputContract.RepairAttempts},
|
||||
}
|
||||
if value.Inputs != nil {
|
||||
inputs := make([]PromptInspectionInput, len(*value.Inputs))
|
||||
for i, input := range *value.Inputs {
|
||||
inputs[i] = PromptInspectionInput{Name: input.Name, Required: input.Required, ContentType: input.ContentType, Description: input.Description}
|
||||
}
|
||||
dto.Inputs = &inputs
|
||||
}
|
||||
switch outputFormat {
|
||||
case OutputFormatJSON:
|
||||
data, err := json.MarshalIndent(dto, "", " ")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return append(data, '\n'), nil
|
||||
case OutputFormatText:
|
||||
var b bytes.Buffer
|
||||
fmt.Fprintf(&b, "prompt_id: %s\nprompt_hash: %s\n", dto.PromptID, dto.PromptHash)
|
||||
if dto.DefaultProfileID != nil {
|
||||
fmt.Fprintf(&b, "default_profile_id: %s\n", *dto.DefaultProfileID)
|
||||
}
|
||||
if dto.Inputs != nil {
|
||||
if len(*dto.Inputs) == 0 {
|
||||
fmt.Fprintln(&b, "inputs: []")
|
||||
} else {
|
||||
fmt.Fprintln(&b, "inputs:")
|
||||
for _, input := range *dto.Inputs {
|
||||
fmt.Fprintf(&b, " - name: %s\n", input.Name)
|
||||
if input.Required != nil {
|
||||
fmt.Fprintf(&b, " required: %t\n", *input.Required)
|
||||
}
|
||||
if input.ContentType != nil {
|
||||
fmt.Fprintf(&b, " content_type: %s\n", *input.ContentType)
|
||||
}
|
||||
if input.Description != nil {
|
||||
fmt.Fprintf(&b, " description: %s\n", strconv.Quote(*input.Description))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(&b, "output_contract:\n format: %s\n validation_mode: %s\n", dto.OutputContract.Format, dto.OutputContract.ValidationMode)
|
||||
if dto.OutputContract.SchemaPath != nil {
|
||||
fmt.Fprintf(&b, " schema_path: %s\n", *dto.OutputContract.SchemaPath)
|
||||
}
|
||||
if dto.OutputContract.RepairAttempts != nil {
|
||||
fmt.Fprintf(&b, " repair_attempts: %d\n", *dto.OutputContract.RepairAttempts)
|
||||
}
|
||||
return b.Bytes(), nil
|
||||
default:
|
||||
return nil, fmt.Errorf("%w: %q", ErrUnknownPreparedRunFormat, outputFormat)
|
||||
}
|
||||
}
|
||||
|
||||
func FormatProfileInspection(value *promptkit.ProfileInspection, outputFormat OutputFormat) ([]byte, error) {
|
||||
if value == nil {
|
||||
return nil, errors.New("profile inspection is nil")
|
||||
}
|
||||
dto := ProfileInspection{ProfileID: value.ProfileID, APIKeyRequired: value.APIKeyRequired, ProfileModelParams: ProfileModelParams{
|
||||
BackendID: value.Backend, Endpoint: value.Endpoint, Model: value.Model, Temperature: value.Temperature,
|
||||
MaxTokens: value.MaxTokens, TopP: value.TopP, TimeoutSeconds: value.TimeoutSeconds, ServiceTier: value.ServiceTier,
|
||||
ReasoningEffort: value.ReasoningEffort, APIKeyEnv: value.APIKeyEnv, ExtraParams: value.ExtraParams,
|
||||
}}
|
||||
switch outputFormat {
|
||||
case OutputFormatJSON:
|
||||
data, err := json.MarshalIndent(dto, "", " ")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return append(data, '\n'), nil
|
||||
case OutputFormatText:
|
||||
var b bytes.Buffer
|
||||
fmt.Fprintf(&b, "profile_id: %s\nprofile_model_params:\n", dto.ProfileID)
|
||||
p := dto.ProfileModelParams
|
||||
if p.BackendID != nil {
|
||||
fmt.Fprintf(&b, " backend_id: %s\n", *p.BackendID)
|
||||
}
|
||||
if p.Endpoint != nil {
|
||||
fmt.Fprintf(&b, " endpoint: %s\n", *p.Endpoint)
|
||||
}
|
||||
if p.Model != nil {
|
||||
fmt.Fprintf(&b, " model: %s\n", *p.Model)
|
||||
}
|
||||
if p.Temperature != nil {
|
||||
fmt.Fprintf(&b, " temperature: %g\n", *p.Temperature)
|
||||
}
|
||||
if p.MaxTokens != nil {
|
||||
fmt.Fprintf(&b, " max_tokens: %d\n", *p.MaxTokens)
|
||||
}
|
||||
if p.TopP != nil {
|
||||
fmt.Fprintf(&b, " top_p: %g\n", *p.TopP)
|
||||
}
|
||||
if p.TimeoutSeconds != nil {
|
||||
fmt.Fprintf(&b, " timeout_seconds: %d\n", *p.TimeoutSeconds)
|
||||
}
|
||||
if p.ServiceTier != nil {
|
||||
fmt.Fprintf(&b, " service_tier: %s\n", *p.ServiceTier)
|
||||
}
|
||||
if p.ReasoningEffort != nil {
|
||||
fmt.Fprintf(&b, " reasoning_effort: %s\n", *p.ReasoningEffort)
|
||||
}
|
||||
if p.APIKeyEnv != nil {
|
||||
fmt.Fprintf(&b, " api_key_env: %s\n", *p.APIKeyEnv)
|
||||
}
|
||||
if p.ExtraParams != nil {
|
||||
paramsJSON, err := json.Marshal(p.ExtraParams)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fmt.Fprintf(&b, " extra_params: %s\n", paramsJSON)
|
||||
}
|
||||
if dto.APIKeyRequired != nil {
|
||||
fmt.Fprintf(&b, "api_key_required: %t\n", *dto.APIKeyRequired)
|
||||
}
|
||||
return b.Bytes(), nil
|
||||
default:
|
||||
return nil, fmt.Errorf("%w: %q", ErrUnknownPreparedRunFormat, outputFormat)
|
||||
}
|
||||
}
|
||||
@@ -1,95 +0,0 @@
|
||||
package format
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func ptr[T any](value T) *T { return &value }
|
||||
|
||||
func TestPromptInspectionOptionalFieldsAndDisclosure(t *testing.T) {
|
||||
inputs := []promptkit.PromptInputInspection{{Name: "first", Required: ptr(false), ContentType: ptr(""), Description: ptr("")}, {Name: "second"}}
|
||||
value := &promptkit.PromptInspection{PromptID: "support/p", PromptHash: "hash", DefaultProfileID: ptr(""), Inputs: &inputs,
|
||||
Messages: []promptkit.PromptMessageInspection{{Role: "user", Content: "private message"}}, Description: ptr("private description"), SessionID: ptr("private session"),
|
||||
OutputContract: promptkit.PromptOutputInspection{Format: "text", ValidationMode: promptkit.ValidationNone, SchemaPath: ptr(""), RepairAttempts: ptr(0)}}
|
||||
for _, format := range []OutputFormat{OutputFormatText, OutputFormatJSON} {
|
||||
output, err := FormatPromptInspection(value, format)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s := string(output)
|
||||
for _, forbidden := range []string{"private message", "private description", "private session", "prompt_version"} {
|
||||
if strings.Contains(s, forbidden) {
|
||||
t.Fatalf("inspection exposed %q: %s", forbidden, s)
|
||||
}
|
||||
}
|
||||
if strings.Index(s, "first") > strings.Index(s, "second") {
|
||||
t.Fatalf("input order changed: %s", s)
|
||||
}
|
||||
if !strings.Contains(s, "default_profile_id") || !strings.Contains(s, "repair_attempts") || !strings.Contains(s, "required") {
|
||||
t.Fatalf("present empty values missing: %s", s)
|
||||
}
|
||||
}
|
||||
value.DefaultProfileID, value.Inputs = nil, nil
|
||||
value.OutputContract.SchemaPath, value.OutputContract.RepairAttempts = nil, nil
|
||||
for _, format := range []OutputFormat{OutputFormatText, OutputFormatJSON} {
|
||||
output, err := FormatPromptInspection(value, format)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, absent := range []string{"default_profile_id", "inputs", "schema_path", "repair_attempts"} {
|
||||
if strings.Contains(string(output), absent) {
|
||||
t.Fatalf("absent %s rendered: %s", absent, output)
|
||||
}
|
||||
}
|
||||
}
|
||||
empty := []promptkit.PromptInputInspection{}
|
||||
value.Inputs = &empty
|
||||
output, err := FormatPromptInspection(value, OutputFormatJSON)
|
||||
if err != nil || !strings.Contains(string(output), `"inputs": []`) {
|
||||
t.Fatalf("empty inputs lost: %s %v", output, err)
|
||||
}
|
||||
if _, err := FormatPromptInspection(nil, OutputFormatText); err == nil {
|
||||
t.Fatal("nil prompt accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProfileInspectionPreservesOnlyPresentValues(t *testing.T) {
|
||||
params := map[string]any{}
|
||||
value := &promptkit.ProfileInspection{ProfileID: "support/fast", Backend: ptr("backend"), Model: ptr(""), Temperature: ptr(0.0), MaxTokens: ptr(0), APIKeyRequired: ptr(false), ExtraParams: ¶ms}
|
||||
for _, format := range []OutputFormat{OutputFormatText, OutputFormatJSON} {
|
||||
output, err := FormatProfileInspection(value, format)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s := string(output)
|
||||
for _, key := range []string{"profile_model_params", "backend_id", "model", "temperature", "max_tokens", "api_key_required", "extra_params"} {
|
||||
if !strings.Contains(s, key) {
|
||||
t.Fatalf("present %s missing: %s", key, s)
|
||||
}
|
||||
}
|
||||
for _, key := range []string{"effective_model_params", "endpoint", "api_key_env", "top_p"} {
|
||||
if strings.Contains(s, key) {
|
||||
t.Fatalf("absent %s appeared: %s", key, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
output, err := FormatProfileInspection(value, OutputFormatJSON)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var decoded map[string]any
|
||||
if err := json.Unmarshal(output, &decoded); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p := decoded["profile_model_params"].(map[string]any)
|
||||
if p["backend_id"] != "backend" || p["model"] != "" || p["temperature"] != float64(0) || decoded["api_key_required"] != false || len(p["extra_params"].(map[string]any)) != 0 {
|
||||
t.Fatalf("wrong values: %#v", decoded)
|
||||
}
|
||||
if _, err := FormatProfileInspection(nil, OutputFormatText); err == nil {
|
||||
t.Fatal("nil profile accepted")
|
||||
}
|
||||
}
|
||||
@@ -1,210 +0,0 @@
|
||||
// Package format formats already-prepared public data for adapters.
|
||||
package format
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
var ErrUnknownPreparedRunFormat = errors.New("unknown prepared run format")
|
||||
|
||||
// OutputFormat selects an application-owned textual or JSON representation.
|
||||
type OutputFormat string
|
||||
|
||||
const (
|
||||
OutputFormatText OutputFormat = "text"
|
||||
OutputFormatJSON OutputFormat = "json"
|
||||
|
||||
DefaultOutputFormat OutputFormat = OutputFormatText
|
||||
)
|
||||
|
||||
// PreparedRunOutputFormat is the output format for prepared render data.
|
||||
type PreparedRunOutputFormat = OutputFormat
|
||||
|
||||
const (
|
||||
PreparedRunFormatText = OutputFormatText
|
||||
PreparedRunFormatJSON = OutputFormatJSON
|
||||
|
||||
DefaultPreparedRunOutputFormat = DefaultOutputFormat
|
||||
)
|
||||
|
||||
// PreparedRunFormatter serializes a prepared run without performing use case work.
|
||||
type PreparedRunFormatter interface {
|
||||
Format(prepared *promptkit.PreparedRun) ([]byte, error)
|
||||
}
|
||||
|
||||
// ParsePreparedRunOutputFormat parses a format name.
|
||||
func ParsePreparedRunOutputFormat(raw string) (PreparedRunOutputFormat, error) {
|
||||
return ParseOutputFormat(raw)
|
||||
}
|
||||
|
||||
// ParseOutputFormat parses the shared inspection and prepared-run format names.
|
||||
func ParseOutputFormat(raw string) (OutputFormat, error) {
|
||||
switch OutputFormat(strings.ToLower(strings.TrimSpace(raw))) {
|
||||
case "":
|
||||
return DefaultOutputFormat, nil
|
||||
case OutputFormatText:
|
||||
return OutputFormatText, nil
|
||||
case OutputFormatJSON:
|
||||
return OutputFormatJSON, nil
|
||||
default:
|
||||
return "", fmt.Errorf("%w: %q (supported: %s, %s)", ErrUnknownPreparedRunFormat, raw, OutputFormatText, OutputFormatJSON)
|
||||
}
|
||||
}
|
||||
|
||||
// FormatterForPreparedRun returns a formatter strategy for the chosen format.
|
||||
func FormatterForPreparedRun(outputFormat PreparedRunOutputFormat) (PreparedRunFormatter, error) {
|
||||
switch outputFormat {
|
||||
case PreparedRunFormatText:
|
||||
return textPreparedRunFormatter{}, nil
|
||||
case PreparedRunFormatJSON:
|
||||
return jsonPreparedRunFormatter{}, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("%w: %q", ErrUnknownPreparedRunFormat, outputFormat)
|
||||
}
|
||||
}
|
||||
|
||||
// FormatPreparedRun formats a prepared run using the selected format.
|
||||
func FormatPreparedRun(prepared *promptkit.PreparedRun, outputFormat PreparedRunOutputFormat) ([]byte, error) {
|
||||
formatter, err := FormatterForPreparedRun(outputFormat)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return formatter.Format(prepared)
|
||||
}
|
||||
|
||||
// FormatPreparedRunByName parses a format name and formats a prepared run.
|
||||
func FormatPreparedRunByName(prepared *promptkit.PreparedRun, rawFormat string) ([]byte, error) {
|
||||
outputFormat, err := ParsePreparedRunOutputFormat(rawFormat)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return FormatPreparedRun(prepared, outputFormat)
|
||||
}
|
||||
|
||||
type jsonPreparedRunFormatter struct{}
|
||||
|
||||
func (jsonPreparedRunFormatter) Format(prepared *promptkit.PreparedRun) ([]byte, error) {
|
||||
if prepared == nil {
|
||||
return nil, errors.New("prepared run is nil")
|
||||
}
|
||||
return json.MarshalIndent(prepared, "", " ")
|
||||
}
|
||||
|
||||
type textPreparedRunFormatter struct{}
|
||||
|
||||
func (textPreparedRunFormatter) Format(prepared *promptkit.PreparedRun) ([]byte, error) {
|
||||
if prepared == nil {
|
||||
return nil, errors.New("prepared run is nil")
|
||||
}
|
||||
|
||||
var b bytes.Buffer
|
||||
fmt.Fprintf(&b, "prompt: %s\n", prepared.PromptID)
|
||||
fmt.Fprintf(&b, "selected_profile_id: %s\n", prepared.SelectedProfileID)
|
||||
if prepared.SelectedBackendID != "" {
|
||||
fmt.Fprintf(&b, "selected_backend_id: %s\n", prepared.SelectedBackendID)
|
||||
}
|
||||
if prepared.PromptHash != "" {
|
||||
fmt.Fprintf(&b, "prompt_hash: %s\n", prepared.PromptHash)
|
||||
}
|
||||
if prepared.SessionID != "" {
|
||||
fmt.Fprintf(&b, "session_id: %s\n", prepared.SessionID)
|
||||
}
|
||||
fmt.Fprintf(&b, "rendered_prompt_hash: %s\n", prepared.RenderedPromptHash)
|
||||
|
||||
target := prepared.EffectiveModelParams
|
||||
fmt.Fprintln(&b, "effective_model_params:")
|
||||
fmt.Fprintf(&b, " endpoint: %s\n", target.Endpoint)
|
||||
fmt.Fprintf(&b, " model: %s\n", target.Model)
|
||||
fmt.Fprintf(&b, " temperature: %g\n", target.Temperature)
|
||||
fmt.Fprintf(&b, " max_tokens: %d\n", target.MaxTokens)
|
||||
fmt.Fprintf(&b, " top_p: %g\n", target.TopP)
|
||||
fmt.Fprintf(&b, " timeout_seconds: %d\n", target.TimeoutSeconds)
|
||||
if target.ServiceTier != "" {
|
||||
fmt.Fprintf(&b, " service_tier: %s\n", target.ServiceTier)
|
||||
}
|
||||
if target.ReasoningEffort != "" {
|
||||
fmt.Fprintf(&b, " reasoning_effort: %s\n", target.ReasoningEffort)
|
||||
}
|
||||
if target.APIKeyEnv != "" {
|
||||
fmt.Fprintf(&b, " api_key_env: %s\n", target.APIKeyEnv)
|
||||
}
|
||||
if len(target.ExtraParams) > 0 {
|
||||
fmt.Fprintln(&b, " extra_params:")
|
||||
keys := make([]string, 0, len(target.ExtraParams))
|
||||
for k := range target.ExtraParams {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
renderedValue, err := formatExtraParamTextValue(target.ExtraParams[k])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to format extra_params.%s: %w", k, err)
|
||||
}
|
||||
fmt.Fprintf(&b, " %s: %s\n", k, renderedValue)
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Fprintln(&b, "input_hashes:")
|
||||
inputKeys := make([]string, 0, len(prepared.InputHashes))
|
||||
for k := range prepared.InputHashes {
|
||||
inputKeys = append(inputKeys, k)
|
||||
}
|
||||
sort.Strings(inputKeys)
|
||||
for _, k := range inputKeys {
|
||||
fmt.Fprintf(&b, " %s: %s\n", k, prepared.InputHashes[k])
|
||||
}
|
||||
|
||||
fmt.Fprintln(&b, "messages:")
|
||||
roleOrder := make([]string, 0)
|
||||
byRole := make(map[string][]promptkit.RenderedMessage)
|
||||
for _, msg := range prepared.Messages {
|
||||
if _, exists := byRole[msg.Role]; !exists {
|
||||
roleOrder = append(roleOrder, msg.Role)
|
||||
}
|
||||
byRole[msg.Role] = append(byRole[msg.Role], msg)
|
||||
}
|
||||
for _, role := range roleOrder {
|
||||
fmt.Fprintf(&b, " %s:\n", role)
|
||||
messages := byRole[role]
|
||||
for i, msg := range messages {
|
||||
fmt.Fprintf(&b, " - message: %d\n", i+1)
|
||||
if msg.CacheControl != nil {
|
||||
fmt.Fprintf(&b, " cache_control: %s", msg.CacheControl.Type)
|
||||
if msg.CacheControl.TTL != "" {
|
||||
fmt.Fprintf(&b, " ttl=%s", msg.CacheControl.TTL)
|
||||
}
|
||||
fmt.Fprintln(&b)
|
||||
}
|
||||
fmt.Fprintln(&b, " content: |")
|
||||
content := msg.Content
|
||||
if content == "" {
|
||||
fmt.Fprintln(&b, " ")
|
||||
continue
|
||||
}
|
||||
for _, line := range strings.Split(content, "\n") {
|
||||
fmt.Fprintf(&b, " %s\n", line)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return b.Bytes(), nil
|
||||
}
|
||||
|
||||
func formatExtraParamTextValue(value any) (string, error) {
|
||||
if s, ok := value.(string); ok {
|
||||
return s, nil
|
||||
}
|
||||
|
||||
b, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(b), nil
|
||||
}
|
||||
@@ -1,375 +0,0 @@
|
||||
package format
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestTextFormatterIncludesPreparedRunDetails(t *testing.T) {
|
||||
prepared := samplePreparedRun()
|
||||
|
||||
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
s := string(out)
|
||||
if strings.Contains(s, "prompt_version") {
|
||||
t.Fatalf("unexpected prompt version in text output: %s", s)
|
||||
}
|
||||
|
||||
for _, want := range []string{
|
||||
"prompt: prompt.id",
|
||||
"selected_profile_id: local-fast",
|
||||
"selected_backend_id: local",
|
||||
"endpoint: http://llm/v1",
|
||||
"model: gpt-test",
|
||||
"temperature: 0.4",
|
||||
"max_tokens: 256",
|
||||
"top_p: 0.8",
|
||||
"timeout_seconds: 45",
|
||||
"service_tier: priority",
|
||||
"reasoning_effort: medium",
|
||||
"api_key_env: SCRIPTORIUM_API_KEY",
|
||||
"prompt_hash: prompt-hash",
|
||||
"rendered_prompt_hash: rendered-hash",
|
||||
"glossary: hash-glossary",
|
||||
"transcript: hash-transcript",
|
||||
"messages:",
|
||||
" system:",
|
||||
" user:",
|
||||
"System guidance.",
|
||||
"Summarize the transcript.",
|
||||
"Include key entities.",
|
||||
"Second user message.",
|
||||
} {
|
||||
if !strings.Contains(s, want) {
|
||||
t.Fatalf("expected text output to include %q, got:\n%s", want, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTextFormatterRendersExtraParamsDeterministically(t *testing.T) {
|
||||
prepared := samplePreparedRun()
|
||||
prepared.EffectiveModelParams.ExtraParams = map[string]any{
|
||||
"z_string": "enabled",
|
||||
"b_number": 42,
|
||||
"a_object": map[string]any{
|
||||
"nested": "value",
|
||||
"count": 2,
|
||||
},
|
||||
"c_array": []any{"first", 3, false},
|
||||
}
|
||||
|
||||
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
s := string(out)
|
||||
|
||||
want := strings.Join([]string{
|
||||
" extra_params:",
|
||||
" a_object: {\"count\":2,\"nested\":\"value\"}",
|
||||
" b_number: 42",
|
||||
" c_array: [\"first\",3,false]",
|
||||
" z_string: enabled",
|
||||
}, "\n")
|
||||
if !strings.Contains(s, want) {
|
||||
t.Fatalf("expected deterministic extra_params block %q, got:\n%s", want, s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTextFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) {
|
||||
const secret = "super-secret-api-key"
|
||||
t.Setenv("SCRIPTORIUM_API_KEY", secret)
|
||||
|
||||
out, err := FormatPreparedRun(samplePreparedRun(), PreparedRunFormatText)
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if strings.Contains(string(out), secret) {
|
||||
t.Fatalf("text output should not include resolved api key value: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTextFormatterDoesNotIncludeDirectAPIKeyValue(t *testing.T) {
|
||||
const directKey = "direct-format-key"
|
||||
// PreparedRun intentionally has no field for direct API keys.
|
||||
prepared := samplePreparedRun()
|
||||
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if strings.Contains(string(out), directKey) {
|
||||
t.Fatalf("text output should not include direct api key value: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTextFormatterIncludesMessageCacheControlBeforeContent(t *testing.T) {
|
||||
prepared := samplePreparedRun()
|
||||
prepared.Messages = []promptkit.RenderedMessage{
|
||||
{
|
||||
Role: "system",
|
||||
Content: "System guidance.",
|
||||
CacheControl: &promptkit.CacheControl{
|
||||
Type: promptkit.CacheControlEphemeral,
|
||||
TTL: "1h",
|
||||
},
|
||||
},
|
||||
{Role: "user", Content: "Summarize the transcript."},
|
||||
}
|
||||
|
||||
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
s := string(out)
|
||||
if !strings.Contains(s, " system:\n - message: 1\n cache_control: ephemeral ttl=1h\n content: |") {
|
||||
t.Fatalf("expected system message cache control before content, got:\n%s", s)
|
||||
}
|
||||
if strings.Count(s, "cache_control:") != 1 {
|
||||
t.Fatalf("expected exactly one cache_control line, got:\n%s", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTextFormatterIncludesSessionIDWhenPresent(t *testing.T) {
|
||||
prepared := samplePreparedRun()
|
||||
prepared.SessionID = "session-123"
|
||||
|
||||
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if !strings.Contains(string(out), "session_id: session-123\n") {
|
||||
t.Fatalf("expected session_id in text output, got:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTextFormatterOmitsEmptyCacheControlTTL(t *testing.T) {
|
||||
prepared := samplePreparedRun()
|
||||
prepared.Messages = []promptkit.RenderedMessage{
|
||||
{
|
||||
Role: "system",
|
||||
Content: "System guidance.",
|
||||
CacheControl: &promptkit.CacheControl{
|
||||
Type: promptkit.CacheControlEphemeral,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
s := string(out)
|
||||
if !strings.Contains(s, " cache_control: ephemeral\n") {
|
||||
t.Fatalf("expected cache_control line without ttl, got:\n%s", s)
|
||||
}
|
||||
if strings.Contains(s, "ttl=") {
|
||||
t.Fatalf("expected empty ttl to be omitted, got:\n%s", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJSONFormatterEmitsValidJSONAndIncludesPreparedRunFields(t *testing.T) {
|
||||
prepared := samplePreparedRun()
|
||||
prepared.SessionID = "session-123"
|
||||
prepared.EffectiveModelParams.ExtraParams = map[string]any{
|
||||
"number": 42,
|
||||
"nested": map[string]any{
|
||||
"enabled": true,
|
||||
},
|
||||
}
|
||||
|
||||
out, err := FormatPreparedRun(prepared, PreparedRunFormatJSON)
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
|
||||
var decoded map[string]any
|
||||
if err := json.Unmarshal(out, &decoded); err != nil {
|
||||
t.Fatalf("expected valid json output, got %v", err)
|
||||
}
|
||||
|
||||
if decoded["prompt_id"] != "prompt.id" {
|
||||
t.Fatalf("expected prompt_id in json output, got %#v", decoded["prompt_id"])
|
||||
}
|
||||
if _, ok := decoded["prompt_version"]; ok {
|
||||
t.Fatalf("unexpected prompt_version in json output: %#v", decoded)
|
||||
}
|
||||
if decoded["selected_profile_id"] != "local-fast" {
|
||||
t.Fatalf("expected selected_profile_id in json output, got %#v", decoded["selected_profile_id"])
|
||||
}
|
||||
if decoded["rendered_prompt_hash"] != "rendered-hash" {
|
||||
t.Fatalf("expected rendered_prompt_hash in json output, got %#v", decoded["rendered_prompt_hash"])
|
||||
}
|
||||
if decoded["session_id"] != "session-123" {
|
||||
t.Fatalf("expected session_id in json output, got %#v", decoded["session_id"])
|
||||
}
|
||||
modelParams, ok := decoded["effective_model_params"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected effective_model_params in json output, got %#v", decoded)
|
||||
}
|
||||
extraParams, ok := modelParams["extra_params"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected extra_params in json output, got %#v", modelParams["extra_params"])
|
||||
}
|
||||
if extraParams["number"] != float64(42) {
|
||||
t.Fatalf("unexpected numeric extra param in json output: %#v", extraParams["number"])
|
||||
}
|
||||
nested, ok := extraParams["nested"].(map[string]any)
|
||||
if !ok || nested["enabled"] != true {
|
||||
t.Fatalf("unexpected nested extra param in json output: %#v", extraParams["nested"])
|
||||
}
|
||||
if _, ok := decoded["input_hashes"]; !ok {
|
||||
t.Fatalf("expected input_hashes in json output, got %#v", decoded)
|
||||
}
|
||||
if _, ok := decoded["messages"]; !ok {
|
||||
t.Fatalf("expected messages in json output, got %#v", decoded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJSONFormatterIncludesMessageCacheControlOnlyWhenPresent(t *testing.T) {
|
||||
prepared := samplePreparedRun()
|
||||
prepared.Messages = []promptkit.RenderedMessage{
|
||||
{
|
||||
Role: "system",
|
||||
Content: "System guidance.",
|
||||
CacheControl: &promptkit.CacheControl{
|
||||
Type: promptkit.CacheControlEphemeral,
|
||||
TTL: "1h",
|
||||
},
|
||||
},
|
||||
{Role: "user", Content: "Summarize the transcript."},
|
||||
}
|
||||
|
||||
out, err := FormatPreparedRun(prepared, PreparedRunFormatJSON)
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
|
||||
var decoded struct {
|
||||
Messages []map[string]any `json:"messages"`
|
||||
}
|
||||
if err := json.Unmarshal(out, &decoded); err != nil {
|
||||
t.Fatalf("expected valid json output, got %v", err)
|
||||
}
|
||||
if len(decoded.Messages) != 2 {
|
||||
t.Fatalf("expected 2 messages, got %d", len(decoded.Messages))
|
||||
}
|
||||
|
||||
cacheControl, ok := decoded.Messages[0]["cache_control"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected first message cache_control, got %#v", decoded.Messages[0])
|
||||
}
|
||||
if cacheControl["type"] != string(promptkit.CacheControlEphemeral) || cacheControl["ttl"] != "1h" {
|
||||
t.Fatalf("unexpected cache_control payload: %#v", cacheControl)
|
||||
}
|
||||
if _, ok := decoded.Messages[1]["cache_control"]; ok {
|
||||
t.Fatalf("expected second message to omit cache_control, got %#v", decoded.Messages[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestJSONFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) {
|
||||
const secret = "super-secret-api-key"
|
||||
t.Setenv("SCRIPTORIUM_API_KEY", secret)
|
||||
|
||||
out, err := FormatPreparedRun(samplePreparedRun(), PreparedRunFormatJSON)
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if strings.Contains(string(out), secret) {
|
||||
t.Fatalf("json output should not include resolved api key value: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJSONFormatterDoesNotIncludeDirectAPIKeyValue(t *testing.T) {
|
||||
const directKey = "direct-format-key"
|
||||
// PreparedRun intentionally has no field for direct API keys.
|
||||
prepared := samplePreparedRun()
|
||||
out, err := FormatPreparedRun(prepared, PreparedRunFormatJSON)
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if strings.Contains(string(out), directKey) {
|
||||
t.Fatalf("json output should not include direct api key value: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParsePreparedRunOutputFormatRecognizesSupportedNames(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
want PreparedRunOutputFormat
|
||||
}{
|
||||
{name: "default empty", input: "", want: DefaultPreparedRunOutputFormat},
|
||||
{name: "text", input: "text", want: PreparedRunFormatText},
|
||||
{name: "json", input: "json", want: PreparedRunFormatJSON},
|
||||
{name: "trim and case", input: " JSON ", want: PreparedRunFormatJSON},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got, err := ParsePreparedRunOutputFormat(tc.input)
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if got != tc.want {
|
||||
t.Fatalf("expected %q, got %q", tc.want, got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParsePreparedRunOutputFormatUnknownFails(t *testing.T) {
|
||||
_, err := ParsePreparedRunOutputFormat("yaml")
|
||||
if err == nil {
|
||||
t.Fatal("expected error for unknown format")
|
||||
}
|
||||
if !errors.Is(err, ErrUnknownPreparedRunFormat) {
|
||||
t.Fatalf("expected ErrUnknownPreparedRunFormat, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatPreparedRunByNameUnknownFailsClearly(t *testing.T) {
|
||||
_, err := FormatPreparedRunByName(samplePreparedRun(), "yaml")
|
||||
if err == nil {
|
||||
t.Fatal("expected unknown format error")
|
||||
}
|
||||
if !errors.Is(err, ErrUnknownPreparedRunFormat) {
|
||||
t.Fatalf("expected ErrUnknownPreparedRunFormat, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func samplePreparedRun() *promptkit.PreparedRun {
|
||||
return &promptkit.PreparedRun{
|
||||
PromptID: "prompt.id",
|
||||
PromptHash: "prompt-hash",
|
||||
SelectedProfileID: "local-fast",
|
||||
SelectedBackendID: "local",
|
||||
EffectiveModelParams: promptkit.ExecutionTarget{
|
||||
BackendID: "local",
|
||||
Endpoint: "http://llm/v1",
|
||||
Model: "gpt-test",
|
||||
Temperature: 0.4,
|
||||
MaxTokens: 256,
|
||||
TopP: 0.8,
|
||||
TimeoutSeconds: 45,
|
||||
ServiceTier: "priority",
|
||||
ReasoningEffort: "medium",
|
||||
APIKeyEnv: "SCRIPTORIUM_API_KEY",
|
||||
},
|
||||
InputHashes: map[string]string{
|
||||
"transcript": "hash-transcript",
|
||||
"glossary": "hash-glossary",
|
||||
},
|
||||
RenderedPromptHash: "rendered-hash",
|
||||
Messages: []promptkit.RenderedMessage{
|
||||
{Role: "system", Content: "System guidance."},
|
||||
{Role: "user", Content: "Summarize the transcript.\nInclude key entities."},
|
||||
{Role: "user", Content: "Second user message."},
|
||||
},
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user