diff --git a/docs/cli.md b/docs/cli.md index cce964d7..9dd70b7a 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -15,6 +15,7 @@ notarius run --input path/to/source.json [--json] [flags] notarius config validate [--config path/to/config.yml] [--config-overlay name] [--pipeline pipeline-id] [--only lane-a,lane-b] notarius config effective [--scope notarius|promptkit] [--config path/to/config.yml] [--config-overlay name] notarius config explain [--scope notarius|promptkit] [--config path/to/config.yml] [--config-overlay name] +notarius config profile [--config path/to/config.yml] [--config-overlay name] notarius pipelines list [--config path/to/config.yml] [--config-overlay name] [--json] ~~~ @@ -217,6 +218,40 @@ imports. Standard output contains only the selected explanation. PromptKit scope constructs and validates its captured definitions, and its report can include ordinary inline prompt and provider-parameter data. +## config profile + +~~~ +notarius config profile [--config path/to/config.yml] [--config-overlay name] +~~~ + +This command inspects one exact, nonblank PromptKit profile ID and writes one +JSON object followed by a newline: + +The following illustrates the JSON shape; field values vary with the selected +profile and its configured backend. + +~~~json +{"profile_id":"example","api_key_required":true,"effective_model_params":{"backend_id":"openrouter","endpoint":"https://provider.example/v1","model":"example-model","temperature":0,"max_tokens":0,"top_p":0,"timeout_seconds":600,"service_tier":"","reasoning_effort":"","api_key_env":"PROVIDER_API_KEY","extra_params":null}} +~~~ + +The object reports the resolved target, including backend-completed endpoint +and credential-name settings. An empty **backend_id** identifies an +endpoint-only profile. Credential names and **api_key_required** are configured +metadata; the command does not look up credential values or report whether an +environment variable is set. Credential-shaped keys in **extra_params** are +redacted. + +This is the named profile's configured target, not a prediction of provider +defaults or the profile selected by an individual request. A request may omit +its profile and let its prompt select a default, and a run may supply a profile +override; this command applies neither behavior. Zero-valued fields show the +effective value but do not distinguish omission from an explicit zero. Use +[effective configuration](#config-effective) and +[configuration explanation](#config-explain) to inspect source presence and +history. Profile/configuration failures use exit status 1; malformed command +syntax, a blank ID, or an incorrect argument count use exit status 2. No +transcript or model call is involved. + ## pipelines list ~~~ diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 99ef1d58..0ff654cf 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -613,6 +613,17 @@ call; preflight/run behavior remains unchanged. Validation: `GOWORK=off go test ./internal/cli ./internal/framework/llm`; `git diff --check`. +> Completion: complete. Added `config profile` with exact nonblank IDs and one +> neutral JSON object/newline containing the captured resolved target and +> `APIKeyRequired` metadata. Tests cover backend-completed and endpoint-only +> targets, inheritance, null/zero values, credential-shaped parameter +> redaction, required but unset credentials, literal whitespace-bearing IDs, +> safe unknown-profile handling, writer failure, and selected overlays. The +> command performs no credential lookup or model call, and profile IDs now +> remain exact through the inspection adapter. Updated root help and the +> [CLI reference](../cli.md). `GOWORK=off go test ./internal/cli +> ./internal/framework/llm` and `git diff --check` pass. + Prompt size: one focused subcommand and its contract. ## Stage 13 — Maintain Examples And Operator Guidance diff --git a/internal/cli/configkit_commands.go b/internal/cli/configkit_commands.go index f38b3922..c4682bdf 100644 --- a/internal/cli/configkit_commands.go +++ b/internal/cli/configkit_commands.go @@ -118,6 +118,51 @@ func runConfigExplain(args []string, stdout, stderr io.Writer, opts Options) int return 0 } +func runConfigProfile(args []string, stdout, stderr io.Writer, opts Options) int { + fs := flag.NewFlagSet("config profile", flag.ContinueOnError) + fs.SetOutput(io.Discard) + configPath := fs.String("config", "", "config file path") + overlay := fs.String("config-overlay", "", "configuration overlay") + if err := fs.Parse(args); err != nil { + fmt.Fprintf(stderr, "notarius: %v\n", err) + return 2 + } + if fs.NArg() != 1 || strings.TrimSpace(fs.Arg(0)) == "" { + fmt.Fprintln(stderr, "notarius: config profile requires exactly one nonblank profile ID") + return 2 + } + profileID := fs.Arg(0) + loaded, _, err := loadConfigKit(*configPath, *overlay, nil, opts) + if err != nil { + writeConfigLoadError(stderr, err) + return 1 + } + inspector, err := newPromptKitInspection(loaded.Config, opts) + if err != nil { + fmt.Fprintf(stderr, "notarius: inspect PromptKit configuration: %v\n", err) + return 1 + } + profile, err := inspector.InspectProfile(context.Background(), profileID) + if err != nil { + fmt.Fprintf(stderr, "notarius: %v\n", err) + return 1 + } + result := struct { + ProfileID string `json:"profile_id"` + APIKeyRequired bool `json:"api_key_required"` + EffectiveModelParams frameworkllm.PromptKitEffectiveModelParams `json:"effective_model_params"` + }{ + ProfileID: profile.ProfileID, + APIKeyRequired: profile.CredentialRequired, + EffectiveModelParams: profile.EffectiveModelParams, + } + if err := json.NewEncoder(stdout).Encode(result); err != nil { + fmt.Fprintf(stderr, "notarius: write PromptKit profile inspection: %v\n", err) + return 1 + } + return 0 +} + func newPromptKitInspection(cfg config.Config, opts Options) (*frameworkllm.PromptKitProfileInspector, error) { return frameworkllm.NewPromptKitProfileInspector(frameworkllm.PromptKitProfileInspectorConfig{ Source: promptKitProfileSourceConfig(cfg), diff --git a/internal/cli/configkit_commands_test.go b/internal/cli/configkit_commands_test.go index 36630ee6..dfe87669 100644 --- a/internal/cli/configkit_commands_test.go +++ b/internal/cli/configkit_commands_test.go @@ -3,6 +3,7 @@ package cli import ( "bytes" "context" + "encoding/json" "errors" "io" "path/filepath" @@ -372,6 +373,133 @@ func TestPromptKitScopedInspectionClassifiesSelectorsAndWriterErrors(t *testing. } } +func TestConfigProfileReportsResolvedRedactedSettingsWithoutCredentialLookup(t *testing.T) { + t.Parallel() + root := t.TempDir() + profilePath := filepath.Join(root, "profiles.yml") + writeConfigKitFile(t, profilePath, `backends: + managed: + endpoint: https://provider.example.invalid/v1 + api_key_env: NOT_SET_SYNTHETIC_PROFILE_KEY +profiles: + inherited: + backend: managed + model: inherited-model + temperature: 0.4 + api_key_env: NOT_SET_SYNTHETIC_PROFILE_KEY + api_key_required: true + extra_params: + Authorization: private-authorization + ordinary: visible-value + selected: + extends: inherited + temperature: 0 + extra_params: null + endpoint-only: + endpoint: https://endpoint.example.invalid/v1 + model: endpoint-model + api_key_required: false + " literal profile ": + endpoint: https://literal.example.invalid/v1 + model: literal-model +`) + configPath := filepath.Join(root, "notarius.yml") + writeConfigKitFile(t, configPath, "version: 5\npromptkit:\n profile_file: "+profilePath+"\n") + writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "promptkit:\n profile_file: "+profilePath+"\n") + assets, err := productionPromptAssets() + if err != nil { + t.Fatal(err) + } + lookupCalls := 0 + factoryCalls := 0 + opts := commandContractOptions(t) + opts.promptKitAssets = assets + opts.LookupEnv = func(string) (string, bool) { + lookupCalls++ + return "", false + } + opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { + factoryCalls++ + return nil, nil, errors.New("profile inspection must not construct an LLM client") + } + + type profileReport struct { + ProfileID string `json:"profile_id"` + APIKeyRequired bool `json:"api_key_required"` + Target struct { + BackendID string `json:"backend_id"` + Endpoint string `json:"endpoint"` + Model string `json:"model"` + Temperature float64 `json:"temperature"` + APIKeyEnv string `json:"api_key_env"` + ExtraParams map[string]any `json:"extra_params"` + } `json:"effective_model_params"` + } + run := func(profileID string) (int, string, string, profileReport) { + t.Helper() + var stdout, stderr bytes.Buffer + code := RunWithOptions([]string{"config", "profile", "--config", configPath, "--config-overlay", "development", profileID}, &stdout, &stderr, opts) + var report profileReport + if code == 0 { + if bytes.Count(stdout.Bytes(), []byte("\n")) != 1 { + t.Fatalf("profile JSON output = %q, want one object and newline", stdout.String()) + } + if err := json.Unmarshal(stdout.Bytes(), &report); err != nil { + t.Fatalf("decode profile JSON %q: %v", stdout.String(), err) + } + } + return code, stdout.String(), stderr.String(), report + } + code, output, diagnostic, selected := run("selected") + if code != 0 || diagnostic != "" || selected.ProfileID != "selected" || !selected.APIKeyRequired || selected.Target.BackendID != "managed" || selected.Target.Endpoint != "https://provider.example.invalid/v1" || selected.Target.Model != "inherited-model" || selected.Target.Temperature != 0 || selected.Target.APIKeyEnv != "NOT_SET_SYNTHETIC_PROFILE_KEY" || selected.Target.ExtraParams != nil { + t.Fatalf("resolved inherited profile = %d %q %q %#v", code, output, diagnostic, selected) + } + code, output, diagnostic, inherited := run("inherited") + if code != 0 || diagnostic != "" || inherited.Target.ExtraParams["Authorization"] != "[REDACTED]" || inherited.Target.ExtraParams["ordinary"] != "visible-value" || strings.Contains(output, "private-authorization") { + t.Fatalf("redacted profile = %d %q %q %#v", code, output, diagnostic, inherited) + } + code, output, diagnostic, endpointOnly := run("endpoint-only") + if code != 0 || diagnostic != "" || endpointOnly.ProfileID != "endpoint-only" || endpointOnly.APIKeyRequired || endpointOnly.Target.BackendID != "" || endpointOnly.Target.Endpoint != "https://endpoint.example.invalid/v1" { + t.Fatalf("endpoint-only profile = %d %q %q %#v", code, output, diagnostic, endpointOnly) + } + code, output, diagnostic, literal := run(" literal profile ") + if code != 0 || diagnostic != "" || literal.ProfileID != " literal profile " || literal.Target.Model != "literal-model" { + t.Fatalf("literal profile ID = %d %q %q %#v", code, output, diagnostic, literal) + } + if lookupCalls != 0 || factoryCalls != 0 { + t.Fatalf("profile inspection called credential lookup %d times and LLM factory %d times", lookupCalls, factoryCalls) + } +} + +func TestConfigProfileClassifiesSyntaxAndRuntimeFailures(t *testing.T) { + t.Parallel() + configPath := writeCommandConfig(t, "session", "other") + for _, test := range []struct { + name string + args []string + want int + }{ + {name: "missing ID", args: []string{"config", "profile", "--config", configPath}, want: 2}, + {name: "blank ID", args: []string{"config", "profile", "--config", configPath, " "}, want: 2}, + {name: "multiple IDs", args: []string{"config", "profile", "--config", configPath, "one", "two"}, want: 2}, + {name: "unknown profile", args: []string{"config", "profile", "--config", configPath, "unknown-profile"}, want: 1}, + } { + t.Run(test.name, func(t *testing.T) { + var stdout, stderr bytes.Buffer + if code := RunWithOptions(test.args, &stdout, &stderr, commandContractOptions(t)); code != test.want || stderr.Len() == 0 { + t.Fatalf("code=%d stdout=%q stderr=%q, want exit %d and diagnostic", code, stdout.String(), stderr.String(), test.want) + } + }) + } + profilePath := filepath.Join(t.TempDir(), "profiles.yml") + writeConfigKitFile(t, profilePath, "profiles:\n writer:\n endpoint: https://example.invalid/v1\n model: writer-model\n") + configPath = writeCommandConfigContent(t, "version: 5\npromptkit:\n profile_file: "+profilePath+"\n") + var stderr bytes.Buffer + if code := RunWithOptions([]string{"config", "profile", "--config", configPath, "writer"}, failingInspectionWriter{}, &stderr, commandContractOptions(t)); code != 1 || !strings.Contains(stderr.String(), "write PromptKit profile inspection") { + t.Fatalf("profile writer failure = %d, %q", code, stderr.String()) + } +} + type failingInspectionWriter struct{} func (failingInspectionWriter) Write([]byte) (int, error) { diff --git a/internal/cli/run.go b/internal/cli/run.go index 357fbdaa..144a6847 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -37,6 +37,7 @@ const usage = `Usage: notarius config validate --config path/to/config.yml [--config-overlay name] [--pipeline pipeline-id] [--only lane-a,lane-b] notarius config effective [--scope notarius|promptkit] --config path/to/config.yml [--config-overlay name] notarius config explain [--scope notarius|promptkit] --config path/to/config.yml [--config-overlay name] + notarius config profile [--config path/to/config.yml] [--config-overlay name] notarius pipelines list --config path/to/config.yml [--config-overlay name] [--json] ` @@ -1059,6 +1060,8 @@ func runConfig(args []string, stdout, stderr io.Writer, opts Options) int { return runConfigEffective(args[1:], stdout, stderr, opts) case "explain": return runConfigExplain(args[1:], stdout, stderr, opts) + case "profile": + return runConfigProfile(args[1:], stdout, stderr, opts) default: fmt.Fprintf(stderr, "notarius: unknown config subcommand %q\n", args[0]) writeUsage(stderr) diff --git a/internal/framework/llm/promptkit_profiles.go b/internal/framework/llm/promptkit_profiles.go index 95b0cccd..507cf73f 100644 --- a/internal/framework/llm/promptkit_profiles.go +++ b/internal/framework/llm/promptkit_profiles.go @@ -94,7 +94,6 @@ func (i *PromptKitProfileInspector) InspectProfile(ctx context.Context, profileI if i == nil || i.engine == nil { return PromptKitProfileInspection{}, fmt.Errorf("PromptKit profile inspector must not be nil") } - profileID = strings.TrimSpace(profileID) inspection, err := i.engine.InspectProfile(ctx, profileID) if err != nil { if ctxErr := ctx.Err(); ctxErr != nil {