Migrate CLI and module tests to keyed PromptKit sources

This commit is contained in:
2026-10-04 12:29:55 +00:00
parent 8a8ca6d83c
commit ec2ac3369b
23 changed files with 189 additions and 255 deletions

View File

@@ -1,11 +1,11 @@
# PromptKit Integration
Notarius pins
[`gitea.maximumdirect.net/eric/promptkit` v0.11.2](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.2)
[`gitea.maximumdirect.net/eric/promptkit` v0.12.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0)
as its in-process prompt engine. The upstream
[Go package consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.2/docs/consumers/pkg-promptkit.md)
[Go package consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/consumers/pkg-promptkit.md)
owns the public engine API, and the upstream
[format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.2/docs/formats.md)
[format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/formats.md)
owns prompt, profile, and schema file contracts.
## Supported Boundary
@@ -20,15 +20,20 @@ Notarius relies on the root `promptkit` package to:
record credential-redacted details and run that exact execution;
- return rendered debug material, validated structured output, selected
profile, backend, effective model metadata, and token usage;
- register the optional conventional `local` backend through `BackendLocal`,
`LocalBackend`, and `WithBackend`;
- compose profile and backend mappings through `ConfigurationSources`,
`ProfileSource`, and `BackendSource`, retaining filesystem roots with
`FilesystemInputs` and `DirectoryInputs`;
- register the optional conventional `local` backend with `BackendSource`,
`BackendLocal`, and `LocalBackend`;
- distinguish structured-output validation failure from execution failure; and
- identify a missing explicit profile through `ErrProfileNotFound` and backend
admission exhaustion through `ErrCapacityExceeded`.
The pinned
[`BackendLocal`, `LocalBackend`, and `WithBackend` API](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.2/backends.go)
owns the registration and backend-capacity contract.
[`BackendSource`, `BackendLocal`, and `LocalBackend` API](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/backends.go)
owns keyed backend configuration and backend-capacity semantics. The
[package consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/consumers/pkg-promptkit.md)
describes source-group assembly.
For one completion, the adapter calls `PrepareExecution`, takes a
caller-owned `Details` snapshot, and calls `RunPrepared` for that same opaque
@@ -72,7 +77,7 @@ PromptKit capacity rejection into the provider-neutral Notarius
`ErrLLMCapacityExceeded` contract. It may include the normalized selected
backend ID in safe diagnostic context, without exposing PromptKit's capacity
error type. PromptKit's built-in client retries eligible transient HTTP failures
with its v0.11.2 default of two additional attempts per generation; Notarius
with its v0.12.0 default of two additional attempts per generation; Notarius
keeps pipeline-stage retries outside that completion boundary.
The adapter projects built-in generation failures as the application-owned
@@ -84,17 +89,17 @@ exposing PromptKit error values to Notarius callers.
## Profile Sources And Compatibility
Notarius gives PromptKit the configured operator profile source, registered
application fallback profile assets, and optional backend registration through
the same construction path for inspection and execution. PromptKit owns the
resulting source precedence and strict profile parsing: a matching operator
profile is a complete replacement for a fallback or built-in profile, while an
invalid matching document fails instead of falling through. The operator
configuration and deployment workflow are defined in
application fallback profile assets, and optional backend configuration
through the same construction path for inspection and execution. PromptKit
owns source precedence, strict group composition, and recursive field overlay:
omitted operator fields retain lower-layer values, while supplied fields
replace them. A malformed final effective definition fails construction. The
operator configuration and deployment workflow are defined in
[Configuration](../config.md#promptkit-profiles) and
[Operations](../operations.md#promptkit-profile-deployment).
PromptKit owns `base_profile` resolution under its
[pinned format rules](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.2/docs/formats.md).
PromptKit and ConfigKit own `extends` inheritance under the
[pinned format rules](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/formats.md).
Notarius records the selected leaf identity and resolved target without parsing
or merging inheritance. An unset filesystem `api_key_env` is optional and may
reach the provider without authorization, which can result in a 401 or 403.
@@ -127,13 +132,13 @@ identity, precedence, credential, and capacity ownership.
External catalog profile names and settings can change between dependency
releases, including removal without aliases. Review directly selected IDs and
inherited `base_profile` values when upgrading. An application-owned alias that
inherited `extends` values when upgrading. An application-owned alias that
inherits an external profile does not freeze that profile's settings.
PromptKit's opt-in `promptkit-openrouter-standard` and
`promptkit-openrouter-auto` names provide name continuity within its
selected-and-tested dependency scope, not a fixed model or provider availability
guarantee. See the upstream
[catalog compatibility policy](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.2/docs/policy/release.md#catalog-profile-compatibility).
[catalog compatibility policy](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/policy/release.md#catalog-profile-compatibility).
Use a complete operator-owned profile with explicit backend or endpoint, model,
and settings when those choices must remain operator-controlled. The embedded
@@ -141,7 +146,7 @@ and settings when those choices must remain operator-controlled. The embedded
external catalog profile. An upgrade updates Notarius's compiled-in catalog
checkpoint identity so prior profile-source checkpoints are not reused.
Notarius supports this boundary against PromptKit v0.11.2. Its fallback source,
Notarius supports this boundary against PromptKit v0.12.0. Its native sources,
prepared-execution, inspection, and typed capacity APIs are used as public
upstream contracts; other PromptKit APIs or file-format behavior are not
implicitly supported. A dependency upgrade requires reviewing the adapter,
@@ -171,8 +176,9 @@ production prompts use PromptKit's bounded structural-repair contract; their
current declaration is one additional repair attempt. Notarius retains the
transport-neutral boundary and does not expose PromptKit types to modules.
The v0.11.2 patch updates upstream policy and licensing documentation without
changing runtime APIs, profile settings, or dependency/catalog versions.
The v0.12.0 upgrade replaces the removed option-based configuration API with
native ConfigKit-backed sources and keyed documents; release pins and notices
are listed in the [canonical attribution](../oss/attribution.md#promptkit).
See the canonical [PromptKit attribution](../oss/attribution.md#promptkit),
[OpenRouter catalog notice](../oss/attribution.md#promptkit-openrouter-catalog),
and [Rakestrawhome catalog notice](../oss/attribution.md#promptkit-rakestrawhome-catalog).

View File

@@ -325,6 +325,12 @@ Prompt size: mechanical adapter fixture migration and restoring its suite.
## Stage 5 — Restore CLI And Registered-Module Fixture Coverage
> Completion: complete. CLI and registered-module fixtures now use keyed
> sources, `GOWORK=off go test ./internal/cli ./internal/modules/...` and
> `GOWORK=off go test ./...` pass offline, and the integration reference
> describes the pinned v0.12 source API. Captured fingerprint migration remains
> scheduled for Stage 7.
Goal: close the initial format/API migration window.
Inspect `internal/cli/promptkit_profiles_test.go`, `production_contract_test.go`,

View File

@@ -19,6 +19,7 @@ import (
"testing/fstest"
"time"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkmap"
@@ -59,6 +60,7 @@ import (
scenenormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/scenedescriptions"
spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop"
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
"gitea.maximumdirect.net/eric/promptkit"
)
@@ -228,14 +230,16 @@ func TestProductionAssetsResolveDNDExtractionProfile(t *testing.T) {
components := productionTestComponents(t)
newEngine := func(profileFile string) (*promptkit.Engine, error) {
t.Helper()
options, err := components.assets.PromptKitOptions()
sources, options, err := components.assets.PromptKitConfigurationSources()
if err != nil {
return nil, err
}
if profileFile != "" {
options = append(options, promptkit.WithProfileFile(profileFile))
sources.Underlays = append(sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{
Source: configkit.File(profileFile), Imports: true,
}}})
}
return promptkit.NewEngine(promptkit.Config{}, options...)
return promptkit.NewEngine(promptkit.Config{Sources: sources}, options...)
}
t.Run("fallback", func(t *testing.T) {
@@ -258,10 +262,11 @@ func TestProductionAssetsResolveDNDExtractionProfile(t *testing.T) {
t.Run("valid operator profile wins", func(t *testing.T) {
profilePath := filepath.Join(t.TempDir(), "profiles.yaml")
if err := os.WriteFile(profilePath, []byte(`id: dnd-extraction
endpoint: http://operator.example.test/v1
model: operator-model
timeout_seconds: 75
if err := os.WriteFile(profilePath, []byte(`profiles:
dnd-extraction:
endpoint: http://operator.example.test/v1
model: operator-model
timeout_seconds: 75
`), 0o600); err != nil {
t.Fatal(err)
}
@@ -274,8 +279,8 @@ timeout_seconds: 75
t.Fatalf("InspectProfile() error = %v, want operator profile", err)
}
params := inspection.EffectiveModelParams
if params.BackendID != "" || params.Endpoint != "http://operator.example.test/v1" || params.Model != "operator-model" || params.TimeoutSeconds != 75 || params.ServiceTier != "" {
t.Fatalf("operator profile parameters = %#v, want complete replacement", params)
if params.Endpoint != "http://operator.example.test/v1" || params.Model != "operator-model" || params.TimeoutSeconds != 75 || params.ServiceTier != "flex" {
t.Fatalf("operator profile parameters = %#v, want recursive overlay over fallback profile", params)
}
})
@@ -320,17 +325,9 @@ func TestProductionPromptAssetsPrepareWithoutProviderCredentials(t *testing.T) {
if _, err := fs.ReadFile(schemaFS, filepath.Base(semanticreconcile.SchemaAssetPath)); err != nil {
t.Fatalf("generic reconciliation schema asset: %v", err)
}
options, err := components.assets.PromptKitOptions()
if err != nil {
t.Fatalf("production PromptKit options: %v", err)
}
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "assembled-prompt-test", Endpoint: "http://127.0.0.1:1/v1", Model: "test",
})))
engine, err := promptkit.NewEngine(promptkit.Config{}, options...)
if err != nil {
t.Fatalf("production prompt engine: %v", err)
}
engine := promptkitfixture.NewEngine(t, components.assets, promptkit.Config{}, map[string]promptkit.ProfileConfig{
"assembled-prompt-test": promptkitfixture.Profile("http://127.0.0.1:1/v1", "test"),
})
if engine == nil {
t.Fatal("production PromptKit engine is nil")
}
@@ -575,25 +572,27 @@ func TestProductionLLMClientFactoryUsesConfiguredLocalBackend(t *testing.T) {
defer server.Close()
profilePath := filepath.Join(t.TempDir(), "profiles.yml")
if err := os.WriteFile(profilePath, []byte(`id: local-profile
backend: local
model: local-model
if err := os.WriteFile(profilePath, []byte(`profiles:
local-profile:
backend: local
model: local-model
`), 0o600); err != nil {
t.Fatal(err)
}
assets := llm.NewAssetRegistry()
if err := assets.RegisterPromptFS(fstest.MapFS{
"production.local.yaml": {Data: []byte(`id: production.local
version: "v1"
inputs:
- name: transcript
required: true
messages:
- role: user
content: '{{ input "transcript" }}'
output:
format: json
validation_mode: json
"production.local.yaml": {Data: []byte(`prompts:
production.local:
v1:
inputs:
- name: transcript
required: true
messages:
- role: user
content: '{{ input "transcript" }}'
output:
format: json
validation_mode: json
`)},
}, "."); err != nil {
t.Fatalf("register prompt assets: %v", err)

View File

@@ -31,11 +31,12 @@ func TestExplicitPromptKitProfileValidationInspectsProfilesWithoutGeneration(t *
}
return profilePath
}
localProfile := "id: local-profile\nbackend: local\nmodel: local-model\n"
credentialProfile := `id: credential-profile
endpoint: ` + server.URL + `/v1
model: credential-model
api_key_env: NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY
localProfile := "profiles:\n local-profile:\n backend: local\n model: local-model\n"
credentialProfile := `profiles:
credential-profile:
endpoint: ` + server.URL + `/v1
model: credential-model
api_key_env: NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY
`
t.Setenv("NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY", "")
@@ -60,7 +61,7 @@ api_key_env: NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY
name: "missing local backend registration",
profilePath: writeProfile(t, "local-profile", localProfile),
profileID: "local-profile",
wantErr: []string{`PromptKit profile "local-profile" is invalid or unreadable`},
wantErr: []string{"load PromptKit profiles", "configuration is invalid or unreadable"},
},
{
name: "absent profile",
@@ -71,9 +72,9 @@ api_key_env: NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY
},
{
name: "malformed profile",
profilePath: writeProfile(t, "malformed-profile", "id: malformed-profile\nendpoint: https://provider.example/v1?credential=forbidden\nmodel: malformed-model\n"),
profilePath: writeProfile(t, "malformed-profile", "profiles:\n malformed-profile:\n endpoint: https://provider.example/v1?credential=forbidden\n model: malformed-model\n"),
profileID: "malformed-profile",
wantErr: []string{`PromptKit profile "malformed-profile" is invalid or unreadable`},
wantErr: []string{"load PromptKit profiles", "configuration is invalid or unreadable"},
rejectErr: []string{"malformed-profile.yaml", "credential=forbidden"},
},
{
@@ -149,7 +150,7 @@ api_key_env: NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY
func TestExplicitPromptKitProfileValidationUsesFallbackAssets(t *testing.T) {
assets := llm.NewAssetRegistry()
if err := assets.RegisterFallbackProfileFS(fstest.MapFS{
"profiles/fallback.yaml": {Data: []byte("id: fallback-profile\nendpoint: http://promptkit.test/v1\nmodel: fallback-model\n")},
"profiles/fallback.yaml": {Data: []byte("profiles:\n fallback-profile:\n endpoint: http://promptkit.test/v1\n model: fallback-model\n")},
}, "profiles"); err != nil {
t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err)
}
@@ -160,8 +161,8 @@ func TestExplicitPromptKitProfileValidationUsesFallbackAssets(t *testing.T) {
func TestExplicitPromptKitProfileValidationRejectsInvalidInheritanceBeforeGeneration(t *testing.T) {
for _, profiles := range []string{
"id: child\nbase_profile: missing\n",
"id: first\nbase_profile: second\n\n---\nid: second\nbase_profile: first\n",
"profiles:\n child:\n extends: missing\n",
"profiles:\n first:\n extends: second\n second:\n extends: first\n",
} {
t.Run("invalid inheritance", func(t *testing.T) {
profilePath := filepath.Join(t.TempDir(), "profiles.yaml")
@@ -169,7 +170,7 @@ func TestExplicitPromptKitProfileValidationRejectsInvalidInheritanceBeforeGenera
t.Fatal(err)
}
profileID := "child"
if strings.Contains(profiles, "id: first") {
if strings.Contains(profiles, "first:") {
profileID = "first"
}
err := validateExplicitPromptKitProfiles(context.Background(), config.Config{PromptKit: config.PromptKitConfig{ProfileFile: profilePath}}, []string{profileID}, nil)

View File

@@ -679,7 +679,7 @@ func writeRunContractProfiles(t *testing.T, ids ...string) string {
t.Helper()
dir := t.TempDir()
for _, id := range ids {
profile := fmt.Sprintf("id: %s\nendpoint: http://127.0.0.1:1/v1\nmodel: %s-model\n", id, id)
profile := fmt.Sprintf("profiles:\n %s:\n endpoint: http://127.0.0.1:1/v1\n model: %s-model\n", id, id)
if err := os.WriteFile(filepath.Join(dir, id+".yaml"), []byte(profile), 0o600); err != nil {
t.Fatal(err)
}

View File

@@ -9,6 +9,7 @@ import (
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
"gitea.maximumdirect.net/eric/promptkit"
)
@@ -17,17 +18,9 @@ func TestRegisterAssetsPreparesGenericPromptOffline(t *testing.T) {
if err := RegisterAssets(registry); err != nil {
t.Fatalf("RegisterAssets() error = %v, want nil", err)
}
options, err := registry.PromptKitOptions()
if err != nil {
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
}
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "semantic-reconciliation-test", Endpoint: "http://127.0.0.1:1/v1", Model: "offline-test-model",
})))
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
if err != nil {
t.Fatalf("NewEngine() error = %v, want nil", err)
}
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
"semantic-reconciliation-test": promptkitfixture.Profile("http://127.0.0.1:1/v1", "offline-test-model"),
})
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: PromptID, PromptVersion: PromptVersion, ProfileID: "semantic-reconciliation-test",
Inputs: map[string]promptkit.ArtifactRef{

View File

@@ -9,6 +9,7 @@ import (
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
"gitea.maximumdirect.net/eric/promptkit"
)
@@ -131,18 +132,7 @@ func prepareScenesPrompt(t *testing.T, transcript []byte, players string, party
func newScenesPromptEngine(t *testing.T, registry *llm.AssetRegistry) *promptkit.Engine {
t.Helper()
options, err := registry.PromptKitOptions()
if err != nil {
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
}
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "scene-test-profile",
Endpoint: "http://127.0.0.1:1/v1",
Model: "scene-test-model",
})))
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
if err != nil {
t.Fatalf("NewEngine() error = %v, want nil", err)
}
return engine
return promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
"scene-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "scene-test-model"),
})
}

View File

@@ -8,6 +8,7 @@ import (
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
"gitea.maximumdirect.net/eric/promptkit"
)
@@ -34,17 +35,9 @@ func TestPromptAssetsPrepareRequiredInputs(t *testing.T) {
if err := RegisterPromptAssets(registry); err != nil {
t.Fatalf("RegisterPromptAssets() error = %v, want nil", err)
}
options, err := registry.PromptKitOptions()
if err != nil {
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
}
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "combat-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "combat-test-model",
})))
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
if err != nil {
t.Fatalf("NewEngine() error = %v, want nil", err)
}
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
"combat-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "combat-test-model"),
})
transcript := `{"units":[{"sentinel":"combat-transcript"}]}`
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "combat-test-profile",

View File

@@ -8,6 +8,7 @@ import (
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
"gitea.maximumdirect.net/eric/promptkit"
)
@@ -63,18 +64,9 @@ func newEnemyEventPromptEngine(t *testing.T) *promptkit.Engine {
if err := RegisterPromptAssets(registry); err != nil {
t.Fatal(err)
}
options, err := registry.PromptKitOptions()
if err != nil {
t.Fatal(err)
}
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "dnd-extraction", Endpoint: "http://127.0.0.1:1/v1", Model: "enemy-test-model",
})))
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
if err != nil {
t.Fatal(err)
}
return engine
return promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
"dnd-extraction": promptkitfixture.Profile("http://127.0.0.1:1/v1", "enemy-test-model"),
})
}
func enemyEventPromptInputs() map[string]promptkit.ArtifactRef {

View File

@@ -8,6 +8,7 @@ import (
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
"gitea.maximumdirect.net/eric/promptkit"
)
@@ -16,17 +17,9 @@ func TestPromptAssetsPrepareItemOccurrencePrompt(t *testing.T) {
if err := RegisterPromptAssets(registry); err != nil {
t.Fatal(err)
}
options, err := registry.PromptKitOptions()
if err != nil {
t.Fatal(err)
}
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "item-occurrences-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "item-occurrences-test-model",
})))
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
if err != nil {
t.Fatal(err)
}
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
"item-occurrences-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "item-occurrences-test-model"),
})
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "item-occurrences-test-profile",
Inputs: map[string]promptkit.ArtifactRef{

View File

@@ -7,6 +7,7 @@ import (
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
"gitea.maximumdirect.net/eric/promptkit"
)
@@ -15,15 +16,9 @@ func TestRegisterPromptAssetsPreparesItemPrompt(t *testing.T) {
if err := RegisterPromptAssets(registry); err != nil {
t.Fatalf("RegisterPromptAssets() error = %v", err)
}
options, err := registry.PromptKitOptions()
if err != nil {
t.Fatalf("PromptKitOptions() error = %v", err)
}
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ID: "item-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "item-test-model"})))
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
if err != nil {
t.Fatalf("NewEngine() error = %v", err)
}
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
"item-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "item-test-model"),
})
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "item-test-profile", Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.Inline(`{"units":[{"sentinel":"item-transcript"}]}`), "players": promptkit.Inline("item-player"), "party": promptkit.Inline(" "), "glossary": promptkit.Inline(" "),
}})

View File

@@ -7,6 +7,7 @@ import (
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
"gitea.maximumdirect.net/eric/promptkit"
)
@@ -15,17 +16,9 @@ func TestRegisterPromptAssetsPreparesLocationOccurrencePrompt(t *testing.T) {
if err := RegisterPromptAssets(registry); err != nil {
t.Fatal(err)
}
options, err := registry.PromptKitOptions()
if err != nil {
t.Fatal(err)
}
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "location-occurrences-test", Endpoint: "http://127.0.0.1:1/v1", Model: "test",
})))
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
if err != nil {
t.Fatal(err)
}
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
"location-occurrences-test": promptkitfixture.Profile("http://127.0.0.1:1/v1", "test"),
})
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "location-occurrences-test",
Inputs: map[string]promptkit.ArtifactRef{

View File

@@ -7,6 +7,7 @@ import (
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
"gitea.maximumdirect.net/eric/promptkit"
)
@@ -15,15 +16,9 @@ func TestRegisterPromptAssetsPreparesLocationPrompt(t *testing.T) {
if err := RegisterPromptAssets(registry); err != nil {
t.Fatalf("RegisterPromptAssets() error = %v", err)
}
options, err := registry.PromptKitOptions()
if err != nil {
t.Fatalf("PromptKitOptions() error = %v", err)
}
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ID: "location-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "location-test-model"})))
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
if err != nil {
t.Fatalf("NewEngine() error = %v", err)
}
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
"location-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "location-test-model"),
})
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "location-test-profile", Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.Inline(`{"units":[{"sentinel":"location-transcript"}]}`), "players": promptkit.Inline("location-player"), "party": promptkit.Inline(" "), "glossary": promptkit.Inline(" "),
}})

View File

@@ -8,6 +8,7 @@ import (
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
"gitea.maximumdirect.net/eric/promptkit"
)
@@ -23,17 +24,9 @@ func TestRegisterPromptAssetsAndPrepareOccurrencePrompt(t *testing.T) {
if _, err := fs.ReadFile(schemaFS, "dnd_npc_occurrences_llm.v1.json"); err != nil {
t.Fatalf("response schema asset: %v", err)
}
options, err := registry.PromptKitOptions()
if err != nil {
t.Fatal(err)
}
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "npc-occurrences-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "npc-occurrences-test-model",
})))
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
if err != nil {
t.Fatal(err)
}
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
"npc-occurrences-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "npc-occurrences-test-model"),
})
transcript := `{"units":[{"sentinel":"occurrence-transcript"}]}`
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "npc-occurrences-test-profile",

View File

@@ -8,6 +8,7 @@ import (
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
"gitea.maximumdirect.net/eric/promptkit"
)
@@ -16,17 +17,9 @@ func TestRegisterPromptAssetsAndPrepareNPCPrompt(t *testing.T) {
if err := RegisterPromptAssets(registry); err != nil {
t.Fatalf("RegisterPromptAssets() error = %v, want nil", err)
}
options, err := registry.PromptKitOptions()
if err != nil {
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
}
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "npc-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "npc-test-model",
})))
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
if err != nil {
t.Fatalf("NewEngine() error = %v, want nil", err)
}
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
"npc-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "npc-test-model"),
})
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "npc-test-profile",
Inputs: map[string]promptkit.ArtifactRef{

View File

@@ -8,6 +8,7 @@ import (
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
"gitea.maximumdirect.net/eric/promptkit"
)
@@ -16,17 +17,9 @@ func TestRegisterPromptAssetsPreparesSceneDescriptionPrompt(t *testing.T) {
if err := RegisterPromptAssets(registry); err != nil {
t.Fatalf("RegisterPromptAssets() error = %v, want nil", err)
}
options, err := registry.PromptKitOptions()
if err != nil {
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
}
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "scene-description-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "scene-description-test-model",
})))
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
if err != nil {
t.Fatalf("NewEngine() error = %v, want nil", err)
}
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
"scene-description-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "scene-description-test-model"),
})
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "scene-description-test-profile",
Inputs: map[string]promptkit.ArtifactRef{

View File

@@ -8,6 +8,7 @@ import (
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
"gitea.maximumdirect.net/eric/promptkit"
)
@@ -88,19 +89,9 @@ func prepareSpellsPrompt(t *testing.T, transcript []byte, players string, party
if err := RegisterPromptAssets(registry); err != nil {
t.Fatalf("register spell prompt assets: %v", err)
}
options, err := registry.PromptKitOptions()
if err != nil {
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
}
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "spell-test-profile",
Endpoint: "http://127.0.0.1:1/v1",
Model: "spell-test-model",
})))
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
if err != nil {
t.Fatalf("NewEngine() error = %v, want nil", err)
}
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
"spell-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "spell-test-model"),
})
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: PromptID,
PromptVersion: SchemaVersion,

View File

@@ -19,6 +19,7 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity"
identityvalidator "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemregistry/identity"
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
"gitea.maximumdirect.net/eric/promptkit"
)
@@ -346,15 +347,9 @@ func TestRegisterPromptAssetsPreparesItemNormalizationPrompt(t *testing.T) {
if err := RegisterPromptAssets(registry); err != nil {
t.Fatal(err)
}
options, err := registry.PromptKitOptions()
if err != nil {
t.Fatal(err)
}
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ID: "item-normalize-test", Endpoint: "http://127.0.0.1:1/v1", Model: "test"})))
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
if err != nil {
t.Fatal(err)
}
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
"item-normalize-test": promptkitfixture.Profile("http://127.0.0.1:1/v1", "test"),
})
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: PromptID, PromptVersion: PromptVersion, ProfileID: "item-normalize-test", Inputs: map[string]promptkit.ArtifactRef{"candidates": promptkit.Inline(`{"candidate_number_range":{"first":1,"last":1},"candidates":[{"candidate_number":1,"label":"Rope","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`), "transcript": promptkit.Inline(`{"windows":[{"units":[]}]}`)}})
if err != nil {
t.Fatal(err)

View File

@@ -8,6 +8,7 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile"
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
"gitea.maximumdirect.net/eric/promptkit"
)
@@ -19,15 +20,9 @@ func TestRegisterPromptAssetsPreparesLocationNormalizationPrompt(t *testing.T) {
if err := RegisterPromptAssets(registry); err != nil {
t.Fatal(err)
}
options, err := registry.PromptKitOptions()
if err != nil {
t.Fatal(err)
}
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ID: "location-normalize-test", Endpoint: "http://127.0.0.1:1/v1", Model: "test"})))
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
if err != nil {
t.Fatal(err)
}
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
"location-normalize-test": promptkitfixture.Profile("http://127.0.0.1:1/v1", "test"),
})
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: PromptID, PromptVersion: PromptVersion, ProfileID: "location-normalize-test", Inputs: map[string]promptkit.ArtifactRef{"candidates": promptkit.Inline(`{"candidate_number_range":{"first":1,"last":1},"candidates":[{"candidate_number":1,"label":"The Tavern","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`), "transcript": promptkit.Inline(`{"windows":[{"units":[]}]}`)}})
if err != nil {
t.Fatal(err)

View File

@@ -8,6 +8,7 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile"
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
"gitea.maximumdirect.net/eric/promptkit"
)
@@ -22,17 +23,9 @@ func TestRegisterPromptAssetsPreparesNormalizationPrompt(t *testing.T) {
if err := RegisterPromptAssets(registry); err != nil {
t.Fatalf("RegisterPromptAssets() error = %v", err)
}
options, err := registry.PromptKitOptions()
if err != nil {
t.Fatalf("PromptKitOptions() error = %v", err)
}
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "normalize-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "normalize-test-model",
})))
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
if err != nil {
t.Fatalf("NewEngine() error = %v", err)
}
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
"normalize-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "normalize-test-model"),
})
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: PromptID, PromptVersion: PromptVersion, ProfileID: "normalize-test-profile",
Inputs: map[string]promptkit.ArtifactRef{

View File

@@ -18,6 +18,7 @@ import (
npcextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/npcregistry"
scenedescriptionextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/scenedescriptions"
spellextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
"gitea.maximumdirect.net/eric/promptkit"
)
@@ -174,18 +175,9 @@ func TestExtractionPromptComposition(t *testing.T) {
func newPromptCacheEngine(t *testing.T, registry *llm.AssetRegistry) *promptkit.Engine {
t.Helper()
options, err := registry.PromptKitOptions()
if err != nil {
t.Fatalf("PromptKitOptions() error = %v", err)
}
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "prompt-cache-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "prompt-cache-test-model",
})))
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
if err != nil {
t.Fatalf("NewEngine() error = %v", err)
}
return engine
return promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
"prompt-cache-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "prompt-cache-test-model"),
})
}
func withPromptInputs(inputs, extras map[string]promptkit.ArtifactRef) map[string]promptkit.ArtifactRef {

View File

@@ -7,6 +7,7 @@ import (
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
"gitea.maximumdirect.net/eric/promptkit"
)
@@ -67,18 +68,9 @@ func TestPromptAssetMetadataIsContentSafe(t *testing.T) {
func newCombatSemanticsPromptEngine(t *testing.T, registry *llm.AssetRegistry) *promptkit.Engine {
t.Helper()
options, err := registry.PromptKitOptions()
if err != nil {
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
}
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "combat-semantics-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "combat-semantics-test-model",
})))
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
if err != nil {
t.Fatalf("NewEngine() error = %v, want nil", err)
}
return engine
return promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
"combat-semantics-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "combat-semantics-test-model"),
})
}
func assertRenderedExactlyOnce(t *testing.T, messages []promptkit.RenderedMessage, sentinel string, wantIndex int) {

View File

@@ -0,0 +1,41 @@
// Package promptkitfixture provides small native configuration sources for
// tests that prepare registered prompt assets.
package promptkitfixture
import (
"testing"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/promptkit"
)
func String(value string) *string { return &value }
func AddProfiles(sources promptkit.ConfigurationSources, profiles map[string]promptkit.ProfileConfig) promptkit.ConfigurationSources {
if len(profiles) == 0 {
return sources
}
sources.Underlays = append(sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{
Source: promptkit.ProfileSource("test profiles", profiles),
}}})
return sources
}
func Profile(endpoint, model string) promptkit.ProfileConfig {
return promptkit.ProfileConfig{Endpoint: String(endpoint), Model: String(model)}
}
func NewEngine(t testing.TB, registry *llm.AssetRegistry, cfg promptkit.Config, profiles map[string]promptkit.ProfileConfig) *promptkit.Engine {
t.Helper()
sources, options, err := registry.PromptKitConfigurationSources()
if err != nil {
t.Fatalf("PromptKitConfigurationSources() error = %v", err)
}
cfg.Sources = AddProfiles(sources, profiles)
engine, err := promptkit.NewEngine(cfg, options...)
if err != nil {
t.Fatalf("NewEngine() error = %v", err)
}
return engine
}