From 169a94f7eafaa01b312da6400671e8756b8d32f9 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 4 Oct 2026 14:04:32 +0000 Subject: [PATCH] Document PromptKit source composition and deployment --- docs/config.md | 134 +++++++++++++++++++---- docs/operations.md | 53 +++++++-- docs/roadmap/implementation.md | 11 ++ examples/llm-profiles/dnd-extraction.yml | 3 + internal/cli/example_contract_test.go | 21 +++- 5 files changed, 193 insertions(+), 29 deletions(-) diff --git a/docs/config.md b/docs/config.md index 80ab6934..df138a8a 100644 --- a/docs/config.md +++ b/docs/config.md @@ -149,7 +149,7 @@ does not independently clear other inherited fields. `extends` is now a reserved key in every input mapping, including dynamic module options and named maps. Existing ordinary data or entry names using that key must be renamed along with any consumer that reads them. Configuration -overlays, pipeline inheritance, and PromptKit profile `base_profile` +overlays, pipeline inheritance, and PromptKit's native profile `extends` inheritance are separate mechanisms. ## PromptKit Profiles @@ -185,14 +185,16 @@ file that supplies it. The complete example's Use absolute paths for services and containers when their deployment layout is not configuration-relative. -An operator source is optional. For a requested ID, PromptKit checks the -configured operator source first, then Notarius's embedded fallback profiles, -then its own built-in catalog. A matching profile is complete: it replaces a -lower-precedence definition rather than merging with it. The maintained +An operator source is optional. PromptKit's maintained catalog, Notarius's +registered prompt and fallback assets, and the selected operator source are +assembled in that order; a configured `local_backend` contributes its backend +last. Later source groups recursively overlay earlier groups. Within one group, +inputs compose strictly; directory filenames do not provide priority. For a +requested profile ID, an operator definition can partially override the +embedded fallback while retaining unspecified fields. The maintained [`dnd-extraction` LLM profile](../examples/llm-profiles/dnd-extraction.yml) -is a secret-free deployment artifact; production, development, and local -deployments can each provide a complete definition with that same workload ID. -Use workload-oriented IDs for new profiles instead of model names. +is a secret-free deployment artifact. It also demonstrates a derived +development profile that is not selected by the normal pipeline. [Operations](operations.md#promptkit-profile-deployment) owns the deployment workflow and credential-handling guidance. @@ -205,27 +207,121 @@ a positive value limits simultaneous local generations. The application-wide backend field has an environment override. Omitting **local_backend** registers nothing and preserves existing built-in and endpoint-only profile behavior. -A file-backed PromptKit profile selects the registration by its case-sensitive -backend ID: +A PromptKit file uses keyed mappings for profiles, backends, and prompts. A +profile selects a backend by its exact ID: ~~~yaml -id: local-summary -backend: local -model: example-model +profiles: + local-summary: + backend: local + model: example-model ~~~ Keep credentials out of the local-backend object. A PromptKit profile may name its credential environment variable through `api_key_env`; set that variable -only in the run environment. PromptKit owns the -[pinned profile-file format](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md), -including `base_profile` inheritance. Notarius passes profiles through without -merging them. Filesystem profiles cannot express PromptKit's in-memory -`APIKeyRequired` setting; an unset `api_key_env` is optional and may reach the -provider without authorization. +only in the run environment. `api_key_required` is supported in profile files; +inspection reports PromptKit's effective metadata but does not check whether +the named environment variable is set. PromptKit owns the +[pinned v0.12.0 format](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/formats.md), +including native `extends` inheritance and field-presence rules. Notarius +selects source groups and passes definitions through PromptKit. The [PromptKit upstream boundary](integrations/pkg-promptkit.md) identifies the supported package API, and [Operations](operations.md#operational-limits) describes the effective concurrency layers. +### Native PromptKit Sources And Migration + +The configured `profile_file` or `profile_dir` is a PromptKit configuration +source. It can contain profiles and backends, as well as prompt overlays. +PromptKit's catalog is followed by registered application prompt/fallback +assets and the selected operator source; Notarius's optional programmatic +`local` backend is applied last. Composition, recursion, exact-key inheritance, +and final-value validation are owned by PromptKit and ConfigKit. Explicit +`null`, zero, false, blank, and omission may have different meanings; an empty +map does not clear an inherited map. + +When migrating external files from PromptKit before v0.12, place backend and +profile entries beneath their `backends` and `profiles` mappings, and place +prompt versions beneath `prompts..v1`. Remove embedded prompt +`id`/`version` fields. Replace `base_profile` with native `extends`. There is +no legacy-format parser or alias layer. Keep Notarius's `version: 5` and +`promptkit.profile_file`/`profile_dir` settings unchanged. + +Imports use native ConfigKit declarations relative to the file that declares +them. A file-root layout keeps imported fragments outside a recursively +selected profile directory, avoiding selection of the same definition both by +directory discovery and by import: + +~~~yaml +# notarius.yml +version: 5 +promptkit: + profile_file: ./promptkit/entry.yml +~~~ + +~~~yaml +# promptkit/entry.yml +configkit: + imports: + - path: fragments/profiles.yml +profiles: + dnd-extraction-development: + extends: dnd-extraction + temperature: 0.2 +~~~ + +PromptKit recursively discovers YAML files in a selected directory, and its +inputs compose strictly; there is no filename-priority merger. Imports retain +their declaring source roots, so relative message/content files resolve from +the file that declares them. Use a separate file root for related imports +when directory selection would otherwise load those files a second time. + +The same unified document can declare a custom backend or select a catalog +backend without redeclaring it: + +~~~yaml +backends: + private-compatible: + endpoint: https://llm.example.invalid/v1 + api_key_env: PRIVATE_LLM_API_KEY +profiles: + private-summary: + backend: private-compatible + model: summary-model + catalog-summary: + backend: openrouter + model: openai/gpt-5.6-luna +~~~ + +To override the embedded D&D fallback for a configured local backend, supply +only the changed fields. Explicitly clear inherited settings that do not apply +to the local target: + +~~~yaml +profiles: + dnd-extraction: + backend: local + model: local-model + service_tier: null +~~~ + +The `promptkit.local_backend` configuration registers that conventional +backend. Prompt overlays use the same recursive source composition; for +example, this changes one output setting while retaining the other embedded +definition fields: + +~~~yaml +prompts: + dnd.spells: + v1: + output: + repair_attempts: 0 +~~~ + +Sequences such as prompt messages are replaced as a whole when overridden. +Prompt and schema definitions remain subject to final effective validation at +engine construction. + `notarius config validate --pipeline ` resolves the selected pipeline and inspects every explicit effective profile without contacting a provider or requiring credential values. It rejects absent, malformed, or incompatible diff --git a/docs/operations.md b/docs/operations.md index bc7c1edf..a9cb067d 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -99,24 +99,25 @@ Profile deployment has four distinct layers: | Layer | Owner | Operational role | | --- | --- | --- | -| Prompts and schemas | Notarius module families | Embedded request and structured-output definitions. They are not deployment profile files. | +| Prompts and schemas | Notarius module families, with optional operator prompt overlays | Embedded request and structured-output definitions; operator configuration may recursively overlay prompt fields. Schemas remain application assets. | | Fallback profiles | Notarius module families | Embedded application defaults, including D&D's `dnd-extraction` profile. | | Built-in profiles | PromptKit | Upstream catalog entries available when no higher-precedence source defines an ID. | -| Operator profiles | Deployment filesystem | Complete environment-specific definitions selected by `promptkit.profile_file` or `promptkit.profile_dir`. | +| Operator configuration | Deployment filesystem | Native profile/backend definitions and prompt overlays selected by `promptkit.profile_file` or `promptkit.profile_dir`. | The maintained D&D pipeline uses the workload ID `dnd-extraction`. The embedded fallback makes that ID usable without an operator file. Production, -development, and local deployments can each install a different complete -definition for the same ID, retaining the pipeline while choosing their own -model, backend, timeout, or reasoning policy. An operator definition wins over -the fallback; it is not merged with it. The configuration field and full -precedence rules are owned by [Configuration](config.md#promptkit-profiles). +development, and local deployments can each overlay settings for the same ID, +retaining inherited values they do not change. Notarius's registered fallback +configuration precedes the selected operator source, and groups compose by +native recursive overlays; a configured Notarius local backend is applied +after the operator source. The configuration fields and source precedence are +owned by [Configuration](config.md#promptkit-profiles). Use a profile source owned by the service account, keep it readable only by the intended operator, and supply provider credentials through the service environment—not in the Notarius configuration or profile YAML. The maintained -[LLM profile](../examples/llm-profiles/dnd-extraction.yml) is secret-free and -can be copied as a format starting point. Validate a deployment without a +[LLM profiles](../examples/llm-profiles/dnd-extraction.yml) are secret-free +and can be copied as a format starting point. Validate a deployment without a provider call or credentials: ~~~sh @@ -132,6 +133,40 @@ surviving declaration. The complete example's Use absolute paths such as `/etc/notarius/profiles/dnd-extraction.yml` for services and containers when that is clearer for the deployment. +### Inspect Before Deployment + +Use the three configuration reports together when reviewing an operator +source: + +~~~sh +notarius config effective --scope promptkit --config /etc/notarius/config.yml +notarius config explain --scope promptkit --config /etc/notarius/config.yml \ + /profiles/dnd-extraction/model +notarius config profile --config /etc/notarius/config.yml dnd-extraction +~~~ + +Each report command constructs an inspection engine from the selected +configuration and registered application assets. Construction validates and +captures the effective definitions; after editing a source, run the command +again to inspect a newly constructed engine. These commands do not resolve +credential values, call a provider, or prepare a request. `config profile` +reports the named profile's resolved target and effective credential metadata, +not whether a credential is available or which profile a particular prompt or +run will select. + +Effective and explanation reports can include ordinary inline prompt and +provider-parameter data. Credential-shaped provider parameter keys are +redacted, but the rest of the content is not treated as secret-free. Reports +are written only when requested; Notarius does not automatically persist them +in run output or debug bundles. Operators own their destination, access, and +retention when redirecting or storing reports. + +Configuration reports are distinct from a prepared execution or a requested +debug trace. They describe captured definitions and source history, not +rendered request messages, runtime credential values, or an individual +provider interaction. The [CLI reference](cli.md#config-effective) documents +the report syntax and output contracts. + ## Run Lifecycle Use the [run command](cli.md#run) to start a pipeline. A valid invocation loads diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 0ff654cf..084897f4 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -659,6 +659,17 @@ linked to CLI/upstream contracts rather than duplicating their manuals. Validation: `GOWORK=off go test ./internal/core/config ./internal/cli`; `git diff --check`; verify local/pinned links. +> Completion: complete. Added a derived `dnd-extraction-development` profile +> with a supported temperature override while leaving the normal pipeline on +> `dnd-extraction`. Updated the canonical configuration guide with v0.12 +> source groups, strict composition, imports, source roots, native migration, +> backend/null/prompt-overlay examples, and exact inheritance. Updated operator +> guidance with the three report workflow, capture lifetime, credential +> metadata, and report handling. Example tests retain exactly two top-level +> configurations and validate the development profile offline without provider +> credentials. `GOWORK=off go test ./internal/core/config ./internal/cli` and +> `git diff --check` pass; reviewed the new local and pinned links. + Prompt size: maintained examples and two operator documents. ## Stage 14 — Finish Developer Contracts And Record The Decision diff --git a/examples/llm-profiles/dnd-extraction.yml b/examples/llm-profiles/dnd-extraction.yml index 75e5c76a..ce0bd8fd 100644 --- a/examples/llm-profiles/dnd-extraction.yml +++ b/examples/llm-profiles/dnd-extraction.yml @@ -4,3 +4,6 @@ profiles: model: openai/gpt-5.6-luna timeout_seconds: 240 service_tier: flex + dnd-extraction-development: + extends: dnd-extraction + temperature: 0.2 diff --git a/internal/cli/example_contract_test.go b/internal/cli/example_contract_test.go index 342d9b4e..d7a95f0a 100644 --- a/internal/cli/example_contract_test.go +++ b/internal/cli/example_contract_test.go @@ -193,7 +193,7 @@ func TestMaintainedConfigurationExampleSet(t *testing.T) { t.Fatal(err) } if len(llmProfiles) != 1 || llmProfiles[0].Name() != "dnd-extraction.yml" { - t.Fatalf("maintained LLM profiles = %#v, want dnd-extraction.yml", llmProfiles) + t.Fatalf("maintained LLM profile files = %#v, want the single combined profile file", llmProfiles) } } @@ -215,6 +215,25 @@ func TestMaintainedExamplesValidateEffectiveProfilesOffline(t *testing.T) { } }) } + + var stdout, stderr strings.Builder + code := RunWithOptions([]string{ + "config", "profile", "--config", repositoryPath("examples", "dnd-complete.config.yml"), "dnd-extraction-development", + }, &stdout, &stderr, Options{}) + var profile struct { + ProfileID string `json:"profile_id"` + Target struct { + BackendID string `json:"backend_id"` + Model string `json:"model"` + Temperature float64 `json:"temperature"` + } `json:"effective_model_params"` + } + if err := json.Unmarshal([]byte(stdout.String()), &profile); err != nil { + t.Fatalf("development profile JSON = %q: %v", stdout.String(), err) + } + if code != 0 || stderr.Len() != 0 || profile.ProfileID != "dnd-extraction-development" || profile.Target.BackendID != "openrouter" || profile.Target.Model != "openai/gpt-5.6-luna" || profile.Target.Temperature != 0.2 { + t.Fatalf("development profile inspection = code %d, %q, %q, %#v", code, stdout.String(), stderr.String(), profile) + } } func exampleStepLaneIDs(resolved pipeline.ResolvedPipeline) []string {