From 79154882b89a5d9a44a86e6120732a7544f74161 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 4 Oct 2026 13:47:00 +0000 Subject: [PATCH] Add scoped PromptKit configuration reports --- docs/cli.md | 28 ++++++-- docs/roadmap/implementation.md | 10 +++ internal/cli/configkit_commands.go | 59 ++++++++++++++- internal/cli/configkit_commands_test.go | 96 +++++++++++++++++++++++++ internal/cli/run.go | 4 +- 5 files changed, 187 insertions(+), 10 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index 92c591cd..cce964d7 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -13,6 +13,8 @@ notarius help notarius --version notarius run --input path/to/source.json [--json] [flags] notarius config validate [--config path/to/config.yml] [--config-overlay name] [--pipeline pipeline-id] [--only lane-a,lane-b] +notarius config effective [--scope notarius|promptkit] [--config path/to/config.yml] [--config-overlay name] +notarius config explain [--scope notarius|promptkit] [--config path/to/config.yml] [--config-overlay name] notarius pipelines list [--config path/to/config.yml] [--config-overlay name] [--json] ~~~ @@ -188,8 +190,16 @@ notarius config effective [--config path/to/config.yml] [--config-overlay name] ~~~ This command writes the selected composed configuration as deterministic, -redacted YAML. Standard output contains only that YAML: it does not validate a -pipeline, prepare modules, materialize references, or contact a provider. +redacted YAML. By default, it reports the Notarius configuration. Set +**--scope promptkit** to report PromptKit's captured effective configuration; +**--scope notarius** is equivalent to the default. Only the PromptKit scope +constructs the PromptKit inspection engine, using the selected configuration +and registered application assets. Engine construction validates and captures +the definitions. Standard output contains only the selected report: the +command does not validate a pipeline, prepare modules, materialize references, +resolve credentials, or contact a provider. PromptKit output can include +ordinary inline prompt and provider-parameter data; credential-shaped provider +parameters are redacted. ## config explain @@ -197,11 +207,15 @@ pipeline, prepare modules, materialize references, or contact a provider. notarius config explain [--config path/to/config.yml] [--config-overlay name] ~~~ -This command writes ConfigKit's redacted, human-readable explanation for one -canonical JSON Pointer. Supply `""` explicitly to explain the root. A malformed -pointer is a syntax error; a well-formed unknown path is a command failure. -The explanation can describe a value displaced by a selected overlay. Standard -output contains only the explanation. +This command writes a redacted, human-readable explanation for one canonical +JSON Pointer. Supply `""` explicitly to explain the root. By default, it +explains Notarius configuration; **--scope promptkit** selects PromptKit +configuration history, while **--scope notarius** is equivalent to the default. +A malformed pointer is a syntax error; a well-formed unknown path is a command +failure. The explanation can describe values displaced by source overlays or +imports. Standard output contains only the selected explanation. PromptKit +scope constructs and validates its captured definitions, and its report can +include ordinary inline prompt and provider-parameter data. ## pipelines list diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 58248314..99ef1d58 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -576,6 +576,16 @@ its requested report without transcript input, credentials, or provider work. Validation: `GOWORK=off go test ./internal/cli`; `git diff --check`. +> Completion: complete. Added opt-in `promptkit` scope for effective and +> explanation reports while preserving the default and explicit `notarius` +> output. The PromptKit path builds its inspector from the selected normalized +> configuration and registered assets, and reports malformed/unknown/history +> selectors and writer failures with the established exit classes. CLI +> coverage verifies imported and fallback origins, escaped pointers, +> credential redaction, and no LLM factory invocation. Updated root help and +> [CLI reference](../cli.md). `GOWORK=off go test ./internal/cli` and +> `git diff --check` pass. + Prompt size: two scoped branches, not resolved-profile parsing. ## Stage 12 — Add Resolved Profile Inspection To The CLI diff --git a/internal/cli/configkit_commands.go b/internal/cli/configkit_commands.go index fb6c90db..f38b3922 100644 --- a/internal/cli/configkit_commands.go +++ b/internal/cli/configkit_commands.go @@ -3,6 +3,7 @@ package cli import ( "context" "encoding/json" + "errors" "flag" "fmt" "io" @@ -10,6 +11,7 @@ import ( configkit "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/notarius/internal/core/config" + frameworkllm "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" ) func runConfigEffective(args []string, stdout, stderr io.Writer, opts Options) int { @@ -17,6 +19,7 @@ func runConfigEffective(args []string, stdout, stderr io.Writer, opts Options) i fs.SetOutput(io.Discard) configPath := fs.String("config", "", "config file path") overlay := fs.String("config-overlay", "", "configuration overlay") + scope := fs.String("scope", "notarius", "configuration scope: notarius or promptkit") if err := fs.Parse(args); err != nil { fmt.Fprintf(stderr, "notarius: %v\n", err) return 2 @@ -25,12 +28,28 @@ func runConfigEffective(args []string, stdout, stderr io.Writer, opts Options) i fmt.Fprintf(stderr, "notarius: unexpected argument %q\n", fs.Arg(0)) return 2 } + if *scope != "notarius" && *scope != "promptkit" { + fmt.Fprintf(stderr, "notarius: invalid configuration scope %q (want notarius or promptkit)\n", *scope) + return 2 + } loaded, _, err := loadConfigKit(*configPath, *overlay, nil, opts) if err != nil { writeConfigLoadError(stderr, err) return 1 } - if err := loaded.WriteEffective(stdout); err != nil { + if *scope == "notarius" { + if err := loaded.WriteEffective(stdout); err != nil { + fmt.Fprintf(stderr, "notarius: write effective configuration: %v\n", err) + return 1 + } + return 0 + } + inspector, err := newPromptKitInspection(loaded.Config, opts) + if err != nil { + fmt.Fprintf(stderr, "notarius: inspect PromptKit configuration: %v\n", err) + return 1 + } + if err := inspector.WriteEffectiveConfiguration(stdout); err != nil { fmt.Fprintf(stderr, "notarius: write effective configuration: %v\n", err) return 1 } @@ -42,6 +61,7 @@ func runConfigExplain(args []string, stdout, stderr io.Writer, opts Options) int fs.SetOutput(io.Discard) configPath := fs.String("config", "", "config file path") overlay := fs.String("config-overlay", "", "configuration overlay") + scope := fs.String("scope", "notarius", "configuration scope: notarius or promptkit") if err := fs.Parse(args); err != nil { fmt.Fprintf(stderr, "notarius: %v\n", err) return 2 @@ -50,7 +70,37 @@ func runConfigExplain(args []string, stdout, stderr io.Writer, opts Options) int fmt.Fprintln(stderr, "notarius: config explain requires exactly one JSON Pointer") return 2 } + if *scope != "notarius" && *scope != "promptkit" { + fmt.Fprintf(stderr, "notarius: invalid configuration scope %q (want notarius or promptkit)\n", *scope) + return 2 + } pointer := fs.Arg(0) + if *scope == "promptkit" { + loaded, _, err := loadConfigKit(*configPath, *overlay, nil, opts) + if err != nil { + writeConfigLoadError(stderr, err) + return 1 + } + inspector, err := newPromptKitInspection(loaded.Config, opts) + if err != nil { + fmt.Fprintf(stderr, "notarius: inspect PromptKit configuration: %v\n", err) + return 1 + } + found, err := inspector.WriteConfigurationExplanation(stdout, pointer) + if err != nil { + if errors.Is(err, frameworkllm.ErrInvalidPromptKitConfigurationPointer) { + fmt.Fprintf(stderr, "notarius: %q is not a canonical JSON Pointer\n", pointer) + return 2 + } + fmt.Fprintf(stderr, "notarius: write PromptKit configuration explanation: %v\n", err) + return 1 + } + if !found { + fmt.Fprintf(stderr, "notarius: PromptKit configuration pointer %q is unknown\n", pointer) + return 1 + } + return 0 + } parsed, err := configkit.ParsePath(pointer) if err != nil || parsed.String() != pointer { fmt.Fprintf(stderr, "notarius: %q is not a canonical JSON Pointer\n", pointer) @@ -68,6 +118,13 @@ func runConfigExplain(args []string, stdout, stderr io.Writer, opts Options) int return 0 } +func newPromptKitInspection(cfg config.Config, opts Options) (*frameworkllm.PromptKitProfileInspector, error) { + return frameworkllm.NewPromptKitProfileInspector(frameworkllm.PromptKitProfileInspectorConfig{ + Source: promptKitProfileSourceConfig(cfg), + Assets: opts.promptKitAssets, + }) +} + func runConfigValidate(args []string, stdout, stderr io.Writer, opts Options) int { fs := flag.NewFlagSet("config validate", flag.ContinueOnError) fs.SetOutput(io.Discard) diff --git a/internal/cli/configkit_commands_test.go b/internal/cli/configkit_commands_test.go index 4b33b533..36630ee6 100644 --- a/internal/cli/configkit_commands_test.go +++ b/internal/cli/configkit_commands_test.go @@ -276,6 +276,102 @@ func TestConfigInspectionCommandsReportWriterFailures(t *testing.T) { } } +func TestPromptKitScopedInspectionCommandsUseSelectedSources(t *testing.T) { + t.Parallel() + root := t.TempDir() + configPath := filepath.Join(root, "notarius.yml") + writeConfigKitFile(t, configPath, "version: 5\npromptkit:\n profile_file: profiles/base.yml\n") + writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "promptkit:\n profile_file: "+filepath.Join(root, "profiles", "imported.yml")+"\n") + writeConfigKitFile(t, filepath.Join(root, "profiles", "base.yml"), `profiles: + base: + endpoint: https://example.invalid/v1 + model: base-model + extra_params: + Authorization: base-auth-secret + ordinary: visible-value +`) + writeConfigKitFile(t, filepath.Join(root, "profiles", "imported.yml"), `configkit: + imports: + - path: base.yml +profiles: + "escaped/id": + model: imported-model + dnd-extraction: + model: imported-override +`) + assets, err := productionPromptAssets() + if err != nil { + t.Fatal(err) + } + factoryCalls := 0 + opts := commandContractOptions(t) + opts.promptKitAssets = assets + opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { + factoryCalls++ + return nil, nil, errors.New("inspection must not construct an LLM client") + } + var stdout, stderr bytes.Buffer + if code := RunWithOptions([]string{"config", "effective", "--config", configPath}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "pipelines:") || strings.Contains(stdout.String(), "profiles:") || stderr.Len() != 0 { + t.Fatalf("default effective report = %d, %q, %q", code, stdout.String(), stderr.String()) + } + defaultReport := stdout.String() + stdout.Reset() + stderr.Reset() + if code := RunWithOptions([]string{"config", "effective", "--scope", "notarius", "--config", configPath}, &stdout, &stderr, opts); code != 0 || stdout.String() != defaultReport || stderr.Len() != 0 { + t.Fatalf("explicit Notarius effective report = %d, %q, %q", code, stdout.String(), stderr.String()) + } + stdout.Reset() + stderr.Reset() + if code := RunWithOptions([]string{"config", "effective", "--scope", "promptkit", "--config", configPath, "--config-overlay", "development"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "imported-model") || strings.Contains(stdout.String(), "base-auth-secret") || !strings.Contains(stdout.String(), "visible-value") || stderr.Len() != 0 { + t.Fatalf("PromptKit effective report = %d, %q, %q", code, stdout.String(), stderr.String()) + } + stdout.Reset() + stderr.Reset() + if code := RunWithOptions([]string{"config", "explain", "--scope", "promptkit", "--config", configPath, "--config-overlay", "development", "/profiles/escaped~1id/model"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "imported-model") || !strings.Contains(stdout.String(), "imported.yml") || stderr.Len() != 0 { + t.Fatalf("PromptKit imported explanation = %d, %q, %q", code, stdout.String(), stderr.String()) + } + stdout.Reset() + stderr.Reset() + if code := RunWithOptions([]string{"config", "explain", "--scope", "promptkit", "--config", configPath, "--config-overlay", "development", "/profiles/base/model"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "base-model") || !strings.Contains(stdout.String(), "base.yml") || stderr.Len() != 0 { + t.Fatalf("PromptKit imported base explanation = %d, %q, %q", code, stdout.String(), stderr.String()) + } + stdout.Reset() + stderr.Reset() + if code := RunWithOptions([]string{"config", "explain", "--scope", "promptkit", "--config", configPath, "--config-overlay", "development", "/profiles/dnd-extraction/model"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "imported-override") || !strings.Contains(stdout.String(), "dnd-extraction.yaml") || stderr.Len() != 0 { + t.Fatalf("PromptKit historical fallback explanation = %d, %q, %q", code, stdout.String(), stderr.String()) + } + if factoryCalls != 0 { + t.Fatalf("inspection invoked LLM client factory %d times", factoryCalls) + } +} + +func TestPromptKitScopedInspectionClassifiesSelectorsAndWriterErrors(t *testing.T) { + t.Parallel() + path := writeCommandConfig(t, "session", "other") + opts := commandContractOptions(t) + for _, test := range []struct { + name string + args []string + want int + }{ + {name: "blank scope", args: []string{"config", "effective", "--scope="}, want: 2}, + {name: "unknown scope", args: []string{"config", "effective", "--scope", "unknown"}, want: 2}, + {name: "malformed pointer", args: []string{"config", "explain", "--scope", "promptkit", "--config", path, "/profiles/~2"}, want: 2}, + {name: "unknown pointer", args: []string{"config", "explain", "--scope", "promptkit", "--config", path, "/not-configured"}, want: 1}, + } { + t.Run(test.name, func(t *testing.T) { + var stdout, stderr bytes.Buffer + if code := RunWithOptions(test.args, &stdout, &stderr, opts); code != test.want || stderr.Len() == 0 { + t.Fatalf("code=%d stdout=%q stderr=%q, want exit %d and diagnostic", code, stdout.String(), stderr.String(), test.want) + } + }) + } + var stderr bytes.Buffer + if code := RunWithOptions([]string{"config", "effective", "--scope", "promptkit", "--config", path}, failingInspectionWriter{}, &stderr, opts); code != 1 || !strings.Contains(stderr.String(), "write effective configuration") { + t.Fatalf("PromptKit writer failure = %d, %q", code, stderr.String()) + } +} + type failingInspectionWriter struct{} func (failingInspectionWriter) Write([]byte) (int, error) { diff --git a/internal/cli/run.go b/internal/cli/run.go index dbf634e3..357fbdaa 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -35,8 +35,8 @@ const usage = `Usage: notarius --version notarius run --input path/to/source.json [--json] [flags] notarius config validate --config path/to/config.yml [--config-overlay name] [--pipeline pipeline-id] [--only lane-a,lane-b] - notarius config effective --config path/to/config.yml [--config-overlay name] - notarius config explain --config path/to/config.yml [--config-overlay name] + notarius config effective [--scope notarius|promptkit] --config path/to/config.yml [--config-overlay name] + notarius config explain [--scope notarius|promptkit] --config path/to/config.yml [--config-overlay name] notarius pipelines list --config path/to/config.yml [--config-overlay name] [--json] `