Compare commits
65 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 63e26dc203 | |||
| fe44b12073 | |||
| 65b8e8f0a2 | |||
| 76364a7b7e | |||
| 1c93aa0947 | |||
| 1febd2edfa | |||
| 5e269f9760 | |||
| fb58aef8c2 | |||
| 3f9727d5e8 | |||
| 0663fe566a | |||
| 347ef05b41 | |||
| 068a4b1b5a | |||
| b396581d21 | |||
| 6f0a75e7cf | |||
| d88a01cec0 | |||
| c9744ad2e6 | |||
| 74e0f454e9 | |||
| 244f258413 | |||
| 5bab35f096 | |||
| 8136b4814f | |||
| 349ace0528 | |||
| 716bc62991 | |||
| 61ceab786c | |||
| 2d1fa57d9b | |||
| 6da4d070e5 | |||
| ae0e410875 | |||
| 79088a3546 | |||
| 2c8b63187c | |||
| 2b278c4074 | |||
| db32d31708 | |||
| b9899ea3cc | |||
| 66fa5585c1 | |||
| e690cae24f | |||
| 6700ed2188 | |||
| 4aba4f5672 | |||
| 169a94f7ea | |||
| db8acbea43 | |||
| 79154882b8 | |||
| c848befb53 | |||
| a904e4d841 | |||
| 7ea9fe12b9 | |||
| deaa3560bc | |||
| ef504eabb2 | |||
| ec2ac3369b | |||
| 8a8ca6d83c | |||
| 30509f9d33 | |||
| 28aab5e9a4 | |||
| ea403c8094 | |||
| f5b066beee | |||
| aae01a59ce | |||
| 71cf084e8b | |||
| e327a5eef7 | |||
| af62b3bacc | |||
| e1097cc7a5 | |||
| 3f226c91d0 | |||
| 64509947df | |||
| 29c7c165a5 | |||
| 4ef69314e1 | |||
| 1d1434b118 | |||
| f4779fede9 | |||
| 01c08ec8e4 | |||
| d7ea0fd32d | |||
| ec99d48619 | |||
| 9e74df0b6e | |||
| 6e5e277426 |
@@ -4,6 +4,6 @@ when:
|
||||
|
||||
steps:
|
||||
- name: validate-release
|
||||
image: golang:1.25.5
|
||||
image: golang:1.27.0
|
||||
commands:
|
||||
- ./scripts/check-release-source.sh "$CI_COMMIT_TAG"
|
||||
|
||||
@@ -30,6 +30,10 @@ demonstrates all implemented D&D lanes and the supporting campaign references.
|
||||
|
||||
## Install A Source Release
|
||||
|
||||
Source builds require Go at or above the consumer minimum declared in
|
||||
[`go.mod`](go.mod). Go 1.27.0 is required by the selected ConfigKit
|
||||
dependency.
|
||||
|
||||
Install a pinned source release with Go:
|
||||
|
||||
~~~
|
||||
|
||||
@@ -1,45 +1,46 @@
|
||||
id: dnd.combat_turns
|
||||
version: "v1"
|
||||
default_profile: dnd-extraction
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
- name: players
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: party
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: glossary
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: npc_registry
|
||||
required: false
|
||||
content_type: application/json
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ./sharedassets/common-dnd-system.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-identity.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-references.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-extraction-evidence.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-npc-registry.md
|
||||
- role: user
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd_combat_turns_llm.v1.json
|
||||
repair_attempts: 1
|
||||
prompts:
|
||||
dnd.combat_turns:
|
||||
v1:
|
||||
default_profile: dnd-extraction
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
- name: players
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: party
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: glossary
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: npc_registry
|
||||
required: false
|
||||
content_type: application/json
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ./sharedassets/common-dnd-system.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-identity.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-references.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-extraction-evidence.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-npc-registry.md
|
||||
- role: user
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd_combat_turns_llm.v1.json
|
||||
repair_attempts: 1
|
||||
|
||||
@@ -1,53 +1,54 @@
|
||||
id: dnd.enemy_events
|
||||
version: "v1"
|
||||
default_profile: dnd-extraction
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
- name: players
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: party
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: glossary
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: npc_registry
|
||||
required: true
|
||||
content_type: application/json
|
||||
- name: combat_turns
|
||||
required: true
|
||||
content_type: application/json
|
||||
- name: npc_occurrences
|
||||
required: true
|
||||
content_type: application/json
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ./sharedassets/common-dnd-system.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-identity.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-references.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-extraction-evidence.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-npc-registry.md
|
||||
- role: user
|
||||
content_file: ./combat-grounding.md
|
||||
- role: user
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd_enemy_events_llm.v1.json
|
||||
repair_attempts: 1
|
||||
prompts:
|
||||
dnd.enemy_events:
|
||||
v1:
|
||||
default_profile: dnd-extraction
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
- name: players
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: party
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: glossary
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: npc_registry
|
||||
required: true
|
||||
content_type: application/json
|
||||
- name: combat_turns
|
||||
required: true
|
||||
content_type: application/json
|
||||
- name: npc_occurrences
|
||||
required: true
|
||||
content_type: application/json
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ./sharedassets/common-dnd-system.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-identity.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-references.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-extraction-evidence.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-npc-registry.md
|
||||
- role: user
|
||||
content_file: ./combat-grounding.md
|
||||
- role: user
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd_enemy_events_llm.v1.json
|
||||
repair_attempts: 1
|
||||
|
||||
@@ -1,45 +1,46 @@
|
||||
id: dnd.item_occurrences
|
||||
version: "v1"
|
||||
default_profile: dnd-extraction
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
- name: players
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: party
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: glossary
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: item_registry
|
||||
required: true
|
||||
content_type: application/json
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ./sharedassets/common-dnd-system.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-identity.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-references.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-extraction-evidence.md
|
||||
- role: user
|
||||
content_file: ./item-registry.md
|
||||
- role: user
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd_item_occurrences_llm.v1.json
|
||||
repair_attempts: 1
|
||||
prompts:
|
||||
dnd.item_occurrences:
|
||||
v1:
|
||||
default_profile: dnd-extraction
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
- name: players
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: party
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: glossary
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: item_registry
|
||||
required: true
|
||||
content_type: application/json
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ./sharedassets/common-dnd-system.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-identity.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-references.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-extraction-evidence.md
|
||||
- role: user
|
||||
content_file: ./item-registry.md
|
||||
- role: user
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd_item_occurrences_llm.v1.json
|
||||
repair_attempts: 1
|
||||
|
||||
@@ -1,40 +1,41 @@
|
||||
id: dnd.item_registry
|
||||
version: "v1"
|
||||
default_profile: dnd-extraction
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
- name: players
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: party
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: glossary
|
||||
required: false
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ./sharedassets/common-dnd-system.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-identity.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-references.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-extraction-evidence.md
|
||||
- role: user
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd_item_registry_llm.v1.json
|
||||
repair_attempts: 1
|
||||
prompts:
|
||||
dnd.item_registry:
|
||||
v1:
|
||||
default_profile: dnd-extraction
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
- name: players
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: party
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: glossary
|
||||
required: false
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ./sharedassets/common-dnd-system.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-identity.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-references.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-extraction-evidence.md
|
||||
- role: user
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd_item_registry_llm.v1.json
|
||||
repair_attempts: 1
|
||||
|
||||
@@ -1,30 +1,31 @@
|
||||
id: dnd.item_registry.normalize
|
||||
version: "v1"
|
||||
default_profile: dnd-extraction
|
||||
inputs:
|
||||
- name: candidates
|
||||
required: true
|
||||
content_type: application/json
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ./sharedassets/common-dnd-system.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/protocol.md
|
||||
- role: user
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/candidates.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/transcript-windows.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: semantic_reconciliation_llm.v1.json
|
||||
repair_attempts: 1
|
||||
prompts:
|
||||
dnd.item_registry.normalize:
|
||||
v1:
|
||||
default_profile: dnd-extraction
|
||||
inputs:
|
||||
- name: candidates
|
||||
required: true
|
||||
content_type: application/json
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ./sharedassets/common-dnd-system.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/protocol.md
|
||||
- role: user
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/candidates.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/transcript-windows.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: semantic_reconciliation_llm.v1.json
|
||||
repair_attempts: 1
|
||||
|
||||
@@ -1,45 +1,46 @@
|
||||
id: dnd.location_occurrences
|
||||
version: "v1"
|
||||
default_profile: dnd-extraction
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
- name: players
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: party
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: glossary
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: location_registry
|
||||
required: true
|
||||
content_type: application/json
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ./sharedassets/common-dnd-system.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-identity.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-references.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-extraction-evidence.md
|
||||
- role: user
|
||||
content_file: ./location-registry.md
|
||||
- role: user
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd_location_occurrences_llm.v1.json
|
||||
repair_attempts: 1
|
||||
prompts:
|
||||
dnd.location_occurrences:
|
||||
v1:
|
||||
default_profile: dnd-extraction
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
- name: players
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: party
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: glossary
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: location_registry
|
||||
required: true
|
||||
content_type: application/json
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ./sharedassets/common-dnd-system.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-identity.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-references.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-extraction-evidence.md
|
||||
- role: user
|
||||
content_file: ./location-registry.md
|
||||
- role: user
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd_location_occurrences_llm.v1.json
|
||||
repair_attempts: 1
|
||||
|
||||
@@ -1,40 +1,41 @@
|
||||
id: dnd.location_registry
|
||||
version: "v1"
|
||||
default_profile: dnd-extraction
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
- name: players
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: party
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: glossary
|
||||
required: false
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ./sharedassets/common-dnd-system.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-identity.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-references.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-extraction-evidence.md
|
||||
- role: user
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd_location_registry_llm.v1.json
|
||||
repair_attempts: 1
|
||||
prompts:
|
||||
dnd.location_registry:
|
||||
v1:
|
||||
default_profile: dnd-extraction
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
- name: players
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: party
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: glossary
|
||||
required: false
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ./sharedassets/common-dnd-system.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-identity.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-references.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-extraction-evidence.md
|
||||
- role: user
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd_location_registry_llm.v1.json
|
||||
repair_attempts: 1
|
||||
|
||||
@@ -1,30 +1,31 @@
|
||||
id: dnd.location_registry.normalize
|
||||
version: "v1"
|
||||
default_profile: dnd-extraction
|
||||
inputs:
|
||||
- name: candidates
|
||||
required: true
|
||||
content_type: application/json
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ./sharedassets/common-dnd-system.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/protocol.md
|
||||
- role: user
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/candidates.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/transcript-windows.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: semantic_reconciliation_llm.v1.json
|
||||
repair_attempts: 1
|
||||
prompts:
|
||||
dnd.location_registry.normalize:
|
||||
v1:
|
||||
default_profile: dnd-extraction
|
||||
inputs:
|
||||
- name: candidates
|
||||
required: true
|
||||
content_type: application/json
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ./sharedassets/common-dnd-system.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/protocol.md
|
||||
- role: user
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/candidates.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/transcript-windows.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: semantic_reconciliation_llm.v1.json
|
||||
repair_attempts: 1
|
||||
|
||||
@@ -1,45 +1,46 @@
|
||||
id: dnd.npc_occurrences
|
||||
version: "v1"
|
||||
default_profile: dnd-extraction
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
- name: players
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: party
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: glossary
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: npc_registry
|
||||
required: true
|
||||
content_type: application/json
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ./sharedassets/common-dnd-system.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-identity.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-references.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-extraction-evidence.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-npc-registry.md
|
||||
- role: user
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd_npc_occurrences_llm.v1.json
|
||||
repair_attempts: 1
|
||||
prompts:
|
||||
dnd.npc_occurrences:
|
||||
v1:
|
||||
default_profile: dnd-extraction
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
- name: players
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: party
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: glossary
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: npc_registry
|
||||
required: true
|
||||
content_type: application/json
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ./sharedassets/common-dnd-system.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-identity.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-references.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-extraction-evidence.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-npc-registry.md
|
||||
- role: user
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd_npc_occurrences_llm.v1.json
|
||||
repair_attempts: 1
|
||||
|
||||
@@ -1,40 +1,41 @@
|
||||
id: dnd.npc_registry
|
||||
version: "v1"
|
||||
default_profile: dnd-extraction
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
- name: players
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: party
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: glossary
|
||||
required: false
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ./sharedassets/common-dnd-system.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-identity.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-references.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-extraction-evidence.md
|
||||
- role: user
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd_npc_registry_llm.v1.json
|
||||
repair_attempts: 1
|
||||
prompts:
|
||||
dnd.npc_registry:
|
||||
v1:
|
||||
default_profile: dnd-extraction
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
- name: players
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: party
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: glossary
|
||||
required: false
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ./sharedassets/common-dnd-system.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-identity.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-references.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-extraction-evidence.md
|
||||
- role: user
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd_npc_registry_llm.v1.json
|
||||
repair_attempts: 1
|
||||
|
||||
@@ -1,30 +1,31 @@
|
||||
id: dnd.npc_registry.normalize
|
||||
version: "v1"
|
||||
default_profile: dnd-extraction
|
||||
inputs:
|
||||
- name: candidates
|
||||
required: true
|
||||
content_type: application/json
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ./sharedassets/common-dnd-system.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/protocol.md
|
||||
- role: user
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/candidates.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/transcript-windows.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: semantic_reconciliation_llm.v1.json
|
||||
repair_attempts: 1
|
||||
prompts:
|
||||
dnd.npc_registry.normalize:
|
||||
v1:
|
||||
default_profile: dnd-extraction
|
||||
inputs:
|
||||
- name: candidates
|
||||
required: true
|
||||
content_type: application/json
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ./sharedassets/common-dnd-system.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/protocol.md
|
||||
- role: user
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/candidates.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/transcript-windows.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: semantic_reconciliation_llm.v1.json
|
||||
repair_attempts: 1
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
id: dnd-extraction
|
||||
backend: openrouter
|
||||
model: openai/gpt-5.6-luna
|
||||
timeout_seconds: 240
|
||||
service_tier: flex
|
||||
profiles:
|
||||
dnd-extraction:
|
||||
backend: openrouter
|
||||
model: openai/gpt-5.6-luna
|
||||
timeout_seconds: 240
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,40 +1,41 @@
|
||||
id: dnd.scene_descriptions
|
||||
version: "v1"
|
||||
default_profile: dnd-extraction
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
- name: players
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: party
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: glossary
|
||||
required: false
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ./sharedassets/common-dnd-system.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-identity.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-references.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-scene-combat-policy.md
|
||||
- role: user
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd_scene_descriptions_llm.v1.json
|
||||
repair_attempts: 1
|
||||
prompts:
|
||||
dnd.scene_descriptions:
|
||||
v1:
|
||||
default_profile: dnd-extraction
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
- name: players
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: party
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: glossary
|
||||
required: false
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ./sharedassets/common-dnd-system.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-identity.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-references.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-scene-combat-policy.md
|
||||
- role: user
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd_scene_descriptions_llm.v1.json
|
||||
repair_attempts: 1
|
||||
|
||||
@@ -1,23 +1,24 @@
|
||||
id: dnd.scene_descriptions.validate_combat
|
||||
version: "v1"
|
||||
default_profile: dnd-extraction
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ./sharedassets/common-dnd-system.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-scene-combat-policy.md
|
||||
- role: user
|
||||
content_file: ./instructions.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd_scene_combat_semantics_llm.v1.json
|
||||
repair_attempts: 1
|
||||
prompts:
|
||||
dnd.scene_descriptions.validate_combat:
|
||||
v1:
|
||||
default_profile: dnd-extraction
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ./sharedassets/common-dnd-system.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-scene-combat-policy.md
|
||||
- role: user
|
||||
content_file: ./instructions.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd_scene_combat_semantics_llm.v1.json
|
||||
repair_attempts: 1
|
||||
|
||||
@@ -1,34 +1,35 @@
|
||||
id: dnd.scenes
|
||||
version: "v1"
|
||||
default_profile: dnd-extraction
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
- name: players
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: party
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: glossary
|
||||
required: false
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ./sharedassets/common-dnd-system.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-references.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./instructions.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-transcript-full.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd_scenes_llm.v1.json
|
||||
repair_attempts: 1
|
||||
prompts:
|
||||
dnd.scenes:
|
||||
v1:
|
||||
default_profile: dnd-extraction
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
- name: players
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: party
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: glossary
|
||||
required: false
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ./sharedassets/common-dnd-system.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-references.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./instructions.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-transcript-full.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd_scenes_llm.v1.json
|
||||
repair_attempts: 1
|
||||
|
||||
@@ -1,50 +1,51 @@
|
||||
id: dnd.spells
|
||||
version: "v1"
|
||||
default_profile: dnd-extraction
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
- name: spell_catalog
|
||||
required: true
|
||||
content_type: application/json
|
||||
- name: players
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: party
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: glossary
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: npc_registry
|
||||
required: false
|
||||
content_type: application/json
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ./sharedassets/common-dnd-system.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-identity.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-references.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-extraction-evidence.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-npc-registry.md
|
||||
- role: user
|
||||
content_file: ./spell-catalog.md
|
||||
- role: user
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd_spells_llm.v1.json
|
||||
repair_attempts: 1
|
||||
prompts:
|
||||
dnd.spells:
|
||||
v1:
|
||||
default_profile: dnd-extraction
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
- name: spell_catalog
|
||||
required: true
|
||||
content_type: application/json
|
||||
- name: players
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: party
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: glossary
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: npc_registry
|
||||
required: false
|
||||
content_type: application/json
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ./sharedassets/common-dnd-system.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-identity.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-references.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-extraction-evidence.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-npc-registry.md
|
||||
- role: user
|
||||
content_file: ./spell-catalog.md
|
||||
- role: user
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd_spells_llm.v1.json
|
||||
repair_attempts: 1
|
||||
|
||||
@@ -1,27 +1,28 @@
|
||||
id: generic.semantic_reconciliation
|
||||
version: "v1"
|
||||
inputs:
|
||||
- name: candidates
|
||||
required: true
|
||||
content_type: application/json
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ./system.md
|
||||
- role: user
|
||||
content_file: ./protocol.md
|
||||
- role: user
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./candidates.md
|
||||
- role: user
|
||||
content_file: ./transcript-windows.md
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: semantic_reconciliation_llm.v1.json
|
||||
repair_attempts: 1
|
||||
prompts:
|
||||
generic.semantic_reconciliation:
|
||||
v1:
|
||||
inputs:
|
||||
- name: candidates
|
||||
required: true
|
||||
content_type: application/json
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ./system.md
|
||||
- role: user
|
||||
content_file: ./protocol.md
|
||||
- role: user
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./candidates.md
|
||||
- role: user
|
||||
content_file: ./transcript-windows.md
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: semantic_reconciliation_llm.v1.json
|
||||
repair_attempts: 1
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# ADR-0010: Use workload-oriented LLM profile defaults
|
||||
|
||||
**Status:** Accepted
|
||||
**Status:** Accepted; profile source replacement behavior superseded by
|
||||
[ADR-0019](0019-adopt-native-promptkit-configuration.md)
|
||||
**Date:** 2026-08-03
|
||||
|
||||
## Context
|
||||
|
||||
92
docs/adr/0019-adopt-native-promptkit-configuration.md
Normal file
92
docs/adr/0019-adopt-native-promptkit-configuration.md
Normal file
@@ -0,0 +1,92 @@
|
||||
# ADR-0019: Adopt native PromptKit configuration sources
|
||||
|
||||
**Status:** Accepted
|
||||
**Date:** 2026-10-04
|
||||
|
||||
## Context
|
||||
|
||||
PromptKit v0.12 replaces its former source options with ConfigKit-backed
|
||||
configuration sources, keyed documents, recursive overlays, and native
|
||||
inheritance. Notarius must choose how application-owned prompt and fallback
|
||||
assets, operator configuration, and the optional local backend reach PromptKit
|
||||
without recreating source parsing or inheritance in the adapter. Runtime,
|
||||
preflight, and operator inspection must agree about the effective definitions.
|
||||
|
||||
Notarius also captures checkpoint identity for the PromptKit configuration
|
||||
used by an execution engine. That identity must follow the engine's captured
|
||||
definition state rather than re-reading mutable source files, and must not
|
||||
publish credentials, prompt bodies, provider parameters, or source paths.
|
||||
|
||||
## Decision
|
||||
|
||||
Use PromptKit's public v0.12 native source APIs as the only PromptKit
|
||||
configuration boundary. Assemble PromptKit catalogs, registered application
|
||||
prompt and fallback inputs, the optional operator file or recursively selected
|
||||
directory, and the optional Notarius local backend registration in that order.
|
||||
Use native ConfigKit imports and PromptKit/ConfigKit composition behavior;
|
||||
retain each filesystem input's resource context. Inputs within a group compose
|
||||
strictly, and later groups recursively overlay earlier ones. PromptKit owns
|
||||
definition parsing, exact-key `extends` inheritance, and final effective-value
|
||||
validation. Notarius does not provide a legacy parser, compatibility alias,
|
||||
filename-priority merger, whole-profile replacement shim, or alternate
|
||||
inheritance engine.
|
||||
|
||||
Execution, explicit-profile preflight, and configuration inspection use the
|
||||
same source construction. Each constructed engine captures its effective
|
||||
definitions. Edits to source files take effect for a newly constructed engine,
|
||||
not an existing one. Derive the private versioned
|
||||
`promptkit_configuration` checkpoint identity from public captured inspection
|
||||
and definition APIs, including effective configuration, profile targets and
|
||||
credential-policy metadata, captured prompt hashes, and the PromptKit and
|
||||
selected catalog identities. Retain only the digest as checkpoint metadata;
|
||||
keep all digest inputs private. Exclude local-backend concurrency settings
|
||||
because they affect scheduling rather than execution semantics.
|
||||
|
||||
Keep PromptKit types and source assembly inside the LLM integration boundary.
|
||||
The CLI may project copied profile fields into a Notarius-owned neutral JSON
|
||||
contract. PromptKit's effective and explanation reports remain upstream
|
||||
redacted configuration and source-history views. Neither inspection path reads
|
||||
credential values or contacts a provider.
|
||||
|
||||
## Compatibility
|
||||
|
||||
The migration to v0.12 keyed configuration is an intentional external file
|
||||
format change. Operators must migrate `id`/`version` embedded fields and
|
||||
`base_profile` to native keyed documents and `extends`. Notarius does not
|
||||
silently accept the former option-based API or file format. Notarius's own
|
||||
version 5 configuration contract and loader decision in
|
||||
[ADR-0018](0018-adopt-configkit.md) remain in force; this decision concerns
|
||||
PromptKit definitions and source composition.
|
||||
|
||||
The workload-oriented profile identity and D&D fallback ownership in
|
||||
[ADR-0010](0010-workload-oriented-llm-profile-defaults.md) remain in force.
|
||||
Only that ADR's former claim that a matching operator profile completely
|
||||
replaces the fallback has been superseded: native recursive overlays retain
|
||||
unspecified inherited fields.
|
||||
|
||||
## Alternatives considered
|
||||
|
||||
- Keep the removed PromptKit option-based configuration API or add a
|
||||
compatibility parser. This would fork the supported upstream contract and
|
||||
preserve legacy format machinery.
|
||||
- Merge profile documents inside Notarius or restore whole-profile
|
||||
replacement. This would duplicate upstream composition behavior and make
|
||||
inspection, preflight, and runtime easier to diverge.
|
||||
- Re-read and hash source files for checkpoint identity. This could describe
|
||||
mutable files rather than the definitions captured by the engine and would
|
||||
duplicate upstream discovery and composition.
|
||||
- Expose upstream inspection structs directly as CLI output. This would bind
|
||||
the public Notarius contract to dependency-owned Go types.
|
||||
|
||||
## Consequences
|
||||
|
||||
PromptKit remains the owner of its native configuration semantics, while
|
||||
Notarius owns which registered and operator sources are assembled and when an
|
||||
engine is reconstructed. A valid later overlay can supersede an earlier
|
||||
definition; validation applies to the final effective definitions rather than
|
||||
as a separate audit of every superseded value. Configuration reports can
|
||||
contain ordinary prompt and provider parameters, so operators own the
|
||||
destination and retention of explicitly requested reports even though
|
||||
credential-shaped fields are redacted. PromptKit upgrades require review of
|
||||
the source adapter, captured identity, reports, and the pinned upstream
|
||||
contracts.
|
||||
@@ -0,0 +1,49 @@
|
||||
# ADR-0020: Delegate configuration mechanics to ConfigKit
|
||||
|
||||
**Status:** Accepted
|
||||
**Date:** 2026-10-04
|
||||
|
||||
## Context
|
||||
|
||||
[ADR-0018](0018-adopt-configkit.md) establishes ConfigKit as Notarius's
|
||||
configuration loader. This decision clarifies the continuing ownership boundary
|
||||
as ConfigKit's capabilities evolve.
|
||||
|
||||
## Decision
|
||||
|
||||
ConfigKit owns configuration mechanics: source loading, parsing, imports,
|
||||
composition, overlays, inheritance, presence tracking, provenance, and
|
||||
inspection. Notarius adopts those semantics through ConfigKit's public APIs.
|
||||
|
||||
Notarius owns application meaning: its configuration schema, version, defaults,
|
||||
field normalization, validity checks, and pipeline/module compatibility. It
|
||||
supplies application-specific defaults, normalization, and validation through
|
||||
ConfigKit's supported extension points. Source-origin metadata supports
|
||||
application rules such as declaration-relative filesystem paths.
|
||||
|
||||
Notarius selects configuration inputs and options through the public API.
|
||||
ConfigKit resolves those inputs and delivers the final application document
|
||||
with source composition and application-field normalization complete. Notarius
|
||||
converts that document into its runtime configuration and performs
|
||||
catalog-dependent pipeline resolution and runtime preparation afterward.
|
||||
|
||||
Operator-requested effective configuration and setting explanations use
|
||||
ConfigKit's inspection APIs. Application validation evaluates the resolved
|
||||
configuration. Changes to general configuration mechanics are pursued upstream
|
||||
in ConfigKit and adopted through its supported API.
|
||||
|
||||
## Alternatives considered
|
||||
|
||||
- Application-owned configuration loading provides direct control over source
|
||||
semantics and assigns parsing, composition, and provenance maintenance to
|
||||
Notarius.
|
||||
- Shared ownership of configuration mechanics allows local adaptations and
|
||||
requires maintaining coordinated semantics across ConfigKit and Notarius.
|
||||
|
||||
## Consequences
|
||||
|
||||
ConfigKit provides a coherent configuration-resolution and inspection contract.
|
||||
Notarius's configuration work concentrates on application semantics and operator
|
||||
integration. Upstream upgrades are evaluated against this ownership boundary
|
||||
and the application's deliberately selected requirements. This decision
|
||||
clarifies ADR-0018 while retaining its adoption and application-format decisions.
|
||||
70
docs/cli.md
70
docs/cli.md
@@ -6,6 +6,13 @@ Configuration fields, discovery rules, and selectable module keys are defined
|
||||
in [Configuration](config.md); runtime state and recovery procedures are
|
||||
defined in [Operations](operations.md).
|
||||
|
||||
Configuration-consuming commands share the same selection: explicit `--config`,
|
||||
then `NOTARIUS_CONFIG`, then working-directory, user, and system searches. A
|
||||
selected file's loading failure is terminal. Reports use that selected source;
|
||||
the [configuration reference](config.md#configuration-discovery-and-precedence)
|
||||
defines the filename groups and value precedence. Use `--config` for a fixed
|
||||
service or subprocess selection.
|
||||
|
||||
## Command Summary
|
||||
|
||||
~~~
|
||||
@@ -13,6 +20,9 @@ notarius help
|
||||
notarius --version
|
||||
notarius run <pipeline-id> --input path/to/source.json [--json] [flags]
|
||||
notarius config validate [--config path/to/config.yml] [--config-overlay name] [--pipeline pipeline-id] [--only lane-a,lane-b]
|
||||
notarius config effective [--scope notarius|promptkit] [--config path/to/config.yml] [--config-overlay name]
|
||||
notarius config explain [--scope notarius|promptkit] [--config path/to/config.yml] [--config-overlay name] <json-pointer>
|
||||
notarius config profile [--config path/to/config.yml] [--config-overlay name] <profile-id>
|
||||
notarius pipelines list [--config path/to/config.yml] [--config-overlay name] [--json]
|
||||
~~~
|
||||
|
||||
@@ -188,8 +198,16 @@ notarius config effective [--config path/to/config.yml] [--config-overlay name]
|
||||
~~~
|
||||
|
||||
This command writes the selected composed configuration as deterministic,
|
||||
redacted YAML. Standard output contains only that YAML: it does not validate a
|
||||
pipeline, prepare modules, materialize references, or contact a provider.
|
||||
redacted YAML. By default, it reports the Notarius configuration. Set
|
||||
**--scope promptkit** to report PromptKit's captured effective configuration;
|
||||
**--scope notarius** is equivalent to the default. Only the PromptKit scope
|
||||
constructs the PromptKit inspection engine, using the selected configuration
|
||||
and registered application assets. Engine construction validates and captures
|
||||
the definitions. Standard output contains only the selected report: the
|
||||
command does not validate a pipeline, prepare modules, materialize references,
|
||||
resolve credentials, or contact a provider. PromptKit output can include
|
||||
ordinary inline prompt and provider-parameter data; credential-shaped provider
|
||||
parameters are redacted.
|
||||
|
||||
## config explain
|
||||
|
||||
@@ -197,11 +215,49 @@ pipeline, prepare modules, materialize references, or contact a provider.
|
||||
notarius config explain [--config path/to/config.yml] [--config-overlay name] <json-pointer>
|
||||
~~~
|
||||
|
||||
This command writes ConfigKit's redacted, human-readable explanation for one
|
||||
canonical JSON Pointer. Supply `""` explicitly to explain the root. A malformed
|
||||
pointer is a syntax error; a well-formed unknown path is a command failure.
|
||||
The explanation can describe a value displaced by a selected overlay. Standard
|
||||
output contains only the explanation.
|
||||
This command writes a redacted, human-readable explanation for one canonical
|
||||
JSON Pointer. Supply `""` explicitly to explain the root. By default, it
|
||||
explains Notarius configuration; **--scope promptkit** selects PromptKit
|
||||
configuration history, while **--scope notarius** is equivalent to the default.
|
||||
A malformed pointer is a syntax error; a well-formed unknown path is a command
|
||||
failure. The explanation can describe values displaced by source overlays or
|
||||
imports. Standard output contains only the selected explanation. PromptKit
|
||||
scope constructs and validates its captured definitions, and its report can
|
||||
include ordinary inline prompt and provider-parameter data.
|
||||
|
||||
## config profile
|
||||
|
||||
~~~
|
||||
notarius config profile [--config path/to/config.yml] [--config-overlay name] <profile-id>
|
||||
~~~
|
||||
|
||||
This command inspects one exact, nonblank PromptKit profile ID and writes one
|
||||
JSON object followed by a newline:
|
||||
|
||||
The following illustrates the JSON shape; field values vary with the selected
|
||||
profile and its configured backend.
|
||||
|
||||
~~~json
|
||||
{"profile_id":"example","api_key_required":true,"effective_model_params":{"backend_id":"openrouter","endpoint":"https://provider.example/v1","model":"example-model","temperature":0,"max_tokens":0,"top_p":0,"timeout_seconds":600,"service_tier":"","reasoning_effort":"","api_key_env":"PROVIDER_API_KEY","extra_params":null}}
|
||||
~~~
|
||||
|
||||
The object reports the resolved target, including backend-completed endpoint
|
||||
and credential-name settings. An empty **backend_id** identifies an
|
||||
endpoint-only profile. Credential names and **api_key_required** are configured
|
||||
metadata; the command does not look up credential values or report whether an
|
||||
environment variable is set. Credential-shaped keys in **extra_params** are
|
||||
redacted.
|
||||
|
||||
This is the named profile's configured target, not a prediction of provider
|
||||
defaults or the profile selected by an individual request. A request may omit
|
||||
its profile and let its prompt select a default, and a run may supply a profile
|
||||
override; this command applies neither behavior. Zero-valued fields show the
|
||||
effective value but do not distinguish omission from an explicit zero. Use
|
||||
[effective configuration](#config-effective) and
|
||||
[configuration explanation](#config-explain) to inspect source presence and
|
||||
history. Profile/configuration failures use exit status 1; malformed command
|
||||
syntax, a blank ID, or an incorrect argument count use exit status 2. No
|
||||
transcript or model call is involved.
|
||||
|
||||
## pipelines list
|
||||
|
||||
|
||||
299
docs/config.md
299
docs/config.md
@@ -1,7 +1,7 @@
|
||||
# Configuration
|
||||
|
||||
This is the canonical reference for Notarius configuration. Configuration files
|
||||
are YAML and must declare version 5. They select pipelines and their modules;
|
||||
are YAML; the main file must declare version 5. They select pipelines and their modules;
|
||||
the [CLI reference](cli.md) owns invocation syntax, and
|
||||
[Operations](operations.md) owns run-state procedures.
|
||||
|
||||
@@ -11,17 +11,30 @@ Commands that load configuration choose a file in this order:
|
||||
|
||||
1. a non-empty **--config** CLI value;
|
||||
2. a non-empty **NOTARIUS_CONFIG** environment value;
|
||||
3. the installed default file at **/usr/local/etc/notarius/config.yml**, when
|
||||
it exists.
|
||||
3. `notarius.yml`, then `notarius.yaml` in the working directory;
|
||||
4. the user configuration directory; then
|
||||
5. the Unix system configuration directories.
|
||||
|
||||
The command fails if none of these paths provides a configuration file.
|
||||
The user search checks `<user-config>/notarius/` for `notarius.yml`,
|
||||
`notarius.yaml`, `config.yml`, `config.yaml`, `main.yml`, and `main.yaml`, in
|
||||
that order, then `<user-config>/notarius.yml` and `notarius.yaml`. The system
|
||||
search uses the same groups under `/usr/local/etc/notarius/` and `/etc/notarius/`,
|
||||
then `/etc/notarius.yml` and `/etc/notarius.yaml`. Generic filenames in the
|
||||
working directory are not candidates. Home-directory search is not enabled.
|
||||
|
||||
Discovery selects one main file; it does not layer all candidates. A missing
|
||||
candidate advances the search, but an explicitly selected missing file, access
|
||||
error, or invalid selected file fails without fallback. An exhausted search
|
||||
fails. Use an explicit `--config` for services and subprocesses that need a
|
||||
fixed selection independent of their working directory.
|
||||
|
||||
For configuration values, precedence is:
|
||||
|
||||
1. built-in defaults;
|
||||
2. composed YAML sources, including imports;
|
||||
3. the selected ConfigKit overlay; and
|
||||
4. explicitly supplied configuration-valued run flags.
|
||||
3. the selected ConfigKit overlay and inheritance;
|
||||
4. typed decoding and explicitly supplied configuration-valued run flags; and
|
||||
5. Notarius resolution and validation.
|
||||
|
||||
`--config-overlay` selects a ConfigKit overlay from `overlays/<name>.yml`
|
||||
beneath the selected configuration root. Empty and `default` select no overlay.
|
||||
@@ -38,7 +51,23 @@ have no effect; `NOTARIUS_CONFIG` remains a source-discovery input.
|
||||
ordered steps, all implemented D&D lanes, generated references, state
|
||||
settings, bounded LLM concurrency, and the maintained
|
||||
[LLM profile](../examples/llm-profiles/dnd-extraction.yml) and imports its
|
||||
pipeline from [a fragment](../examples/pipelines/dnd-session.yml).
|
||||
pipeline from [a fragment](../examples/pipelines/dnd-session.yml), which imports
|
||||
its [first step](../examples/pipelines/dnd-session.describe-session.yml).
|
||||
|
||||
The complete fragment also defines `dnd-session-development`, which inherits
|
||||
the complete pipeline and profile while setting
|
||||
`structured_output_repair_attempts: 0`. Select it with:
|
||||
|
||||
```sh
|
||||
notarius run dnd-session-development \
|
||||
--config /path/to/dnd-complete.config.yml \
|
||||
--input /path/to/transcript.json
|
||||
```
|
||||
|
||||
See the [CLI reference](cli.md#run) for the full invocation contract. This
|
||||
variant disables PromptKit structural repairs without changing configured producer
|
||||
retry budgets. When using another profile, operators can instead select
|
||||
`dnd-session` and set its `llm_profile` to that profile.
|
||||
|
||||
Use these complete files as starting points rather than combining the
|
||||
illustrative fragments in this reference.
|
||||
@@ -80,16 +109,118 @@ umask, setgid inheritance, and default ACLs; see [Operations](operations.md#cach
|
||||
|
||||
Every nonempty relative configured destination or reference path resolves from
|
||||
the canonical directory of the file that supplies its surviving declaration.
|
||||
This includes imported fragments and selected configuration overlays. Built-in
|
||||
relative destinations use the selected main configuration file's directory;
|
||||
explicit `--output-dir` and `--debug-dir` values use the invocation working
|
||||
directory. Absolute values remain unchanged.
|
||||
This includes imported fragments, selected configuration overlays, and values
|
||||
copied through inheritance: the copied value keeps its original declaring
|
||||
file. A child-supplied replacement resolves from the child's declaring file.
|
||||
Built-in relative destinations use the selected main configuration file's
|
||||
directory; explicit `--output-dir` and `--debug-dir` values use the invocation
|
||||
working directory. Absolute values remain unchanged.
|
||||
|
||||
The root file can use ConfigKit imports to compose disjoint definitions. Imports
|
||||
and selected overlays remain under the configuration root; they do not merge or
|
||||
append a pipeline's `steps` sequence. An overlay replaces sequences as defined
|
||||
by ConfigKit, so changing an individual ordered binding requires supplying the
|
||||
whole replacement sequence.
|
||||
Use source-root `config` controls for imports and collection declarations. For
|
||||
example, an essential fragment should explicitly be required:
|
||||
|
||||
~~~yaml
|
||||
config:
|
||||
imports:
|
||||
- path: pipelines/session.yml
|
||||
required: true
|
||||
~~~
|
||||
|
||||
Imports are optional by default. ConfigKit supports paths, lists, directories,
|
||||
globs, exclusions, recursive selection, destinations, and sequence-item placement;
|
||||
its native input-root rules apply. Imported fragments need not declare a version.
|
||||
The complete example places a mapping fragment at a step-list destination. Nested
|
||||
application data named `config` is ordinary data, not a control block.
|
||||
|
||||
When migrating application files, rename genuine root `configkit` control
|
||||
blocks to `config` in the main file and fragments. This is not a compatibility
|
||||
alias: `configkit` is ordinary data under the new reader. PromptKit profile
|
||||
documents still use their separate older engine and retain `configkit` controls;
|
||||
see [PromptKit Profiles](#promptkit-profiles).
|
||||
|
||||
## Sibling Inheritance
|
||||
|
||||
ConfigKit supports native `extends` composition in mappings and identified
|
||||
sequence records. In a mapping, `extends: production` selects the named immediate
|
||||
sibling in that same mapping. In an identified sequence, the reference is an
|
||||
identity mapping: for steps identified by `id`, `extends: {id: parent}` selects
|
||||
a record in the same final sequence. Neither form is a path or cross-scope
|
||||
lookup. See the pinned [native inheritance examples](https://forge.maximumdirect.net/base-layer/configkit/src/commit/c1c7e87f2cdf77a0cbf3327831ac05578e756b9e/docs/config.md#sibling-inheritance).
|
||||
Imports and the selected overlay are composed before
|
||||
inheritance and typed decoding. Explicit CLI assignments follow decoding and
|
||||
inheritance, before Notarius resolution and validation; they cannot repair an
|
||||
earlier structural or decoding error. Every definition, including a parent and definitions not
|
||||
selected for a run, remains subject to ordinary validation.
|
||||
|
||||
Child mappings recursively overlay parent mappings. Scalars and null replace
|
||||
inherited values. Unidentified sequences replace as whole values; pipeline
|
||||
steps are identified by `id` and use native upsert by default:
|
||||
|
||||
~~~yaml
|
||||
pipelines:
|
||||
production:
|
||||
input: {module: seriatim}
|
||||
llm_profile: dnd-extraction
|
||||
# Other complete pipeline settings and ordered steps...
|
||||
development:
|
||||
extends: production
|
||||
structured_output_repair_attempts: 0
|
||||
~~~
|
||||
|
||||
Pipeline names are siblings within `pipelines`; nested mappings such as named
|
||||
artifact lanes also have their own sibling namespace. Distinct imported step IDs
|
||||
combine in ConfigKit's dependency-ready assembly order, not necessarily import
|
||||
declaration or acquisition order; duplicate peer IDs conflict. Ready contributions
|
||||
are selected by their stable source keys. See the pinned
|
||||
[import assembly contract](https://forge.maximumdirect.net/base-layer/configkit/src/commit/c1c7e87f2cdf77a0cbf3327831ac05578e756b9e/docs/config.md#explicit-imports)
|
||||
for full ordering semantics. In the complete example, the imported describe-session
|
||||
step precedes the importing body's extract-events step because an imported
|
||||
dependency is assembled before its importer. Across an overlay or
|
||||
inheritance, upsert replaces a matching step as a complete record in place and
|
||||
appends new IDs. It does not merge individual fields of a matching step. An
|
||||
identified record can itself use native `extends`. Authors may select `replace`
|
||||
or `append` with a `config.sequences` declaration at the relevant destination;
|
||||
an explicit replacement with `[]` clears the list, while empty append or upsert
|
||||
does not. Notarius then validates the resulting pipeline, including its nonempty
|
||||
step requirement. Validator chains are not automatically identified. Authors
|
||||
may declare identities for other lists when needed. CLI list assignment replaces
|
||||
the complete decoded list.
|
||||
|
||||
Record inheritance does not create template-only steps: every record in the
|
||||
final step list executes and must have artifact lanes. A child that inherits
|
||||
its parent's named lanes also conflicts with the requirement that lane IDs be
|
||||
unique across steps; adding differently named lanes does not remove inherited
|
||||
ones. Use pipeline-level inheritance with step upsert for pipeline variants.
|
||||
Native identified-record inheritance is also available in compatible collections,
|
||||
such as an author-identified validator list, subject to ordinary domain validation.
|
||||
|
||||
For example, an overlay can opt into replacing a pipeline's steps:
|
||||
|
||||
~~~yaml
|
||||
config:
|
||||
sequences:
|
||||
- destination: [pipelines, production, steps]
|
||||
mode: replace
|
||||
pipelines:
|
||||
production:
|
||||
steps:
|
||||
- id: extract
|
||||
artifacts: {spells: {extract: {module: dnd/spells}}}
|
||||
~~~
|
||||
|
||||
Inherited values count as explicit input for Notarius's existing
|
||||
presence-sensitive rules: zero, false, and empty-list overrides stay explicit
|
||||
instead of restoring a default. Null also replaces the inherited value, but
|
||||
each field's ordinary type and validation rules still apply; null is not
|
||||
universally valid. Omitting `extends` preserves a relationship supplied by a
|
||||
lower-priority source, while `extends: null` cancels that relationship. It
|
||||
does not independently clear other inherited fields.
|
||||
|
||||
`extends` is now a reserved key in every input mapping, including dynamic
|
||||
module options and named maps. Existing ordinary data or entry names using that
|
||||
key must be renamed along with any consumer that reads them. Configuration
|
||||
overlays, pipeline inheritance, and PromptKit's native profile `extends`
|
||||
inheritance are separate mechanisms.
|
||||
|
||||
## PromptKit Profiles
|
||||
|
||||
@@ -124,14 +255,16 @@ file that supplies it. The complete example's
|
||||
Use absolute paths for services and containers when their deployment layout is
|
||||
not configuration-relative.
|
||||
|
||||
An operator source is optional. For a requested ID, PromptKit checks the
|
||||
configured operator source first, then Notarius's embedded fallback profiles,
|
||||
then its own built-in catalog. A matching profile is complete: it replaces a
|
||||
lower-precedence definition rather than merging with it. The maintained
|
||||
An operator source is optional. PromptKit's maintained catalog, Notarius's
|
||||
registered prompt and fallback assets, and the selected operator source are
|
||||
assembled in that order; a configured `local_backend` contributes its backend
|
||||
last. Later source groups recursively overlay earlier groups. Within one group,
|
||||
inputs compose strictly; directory filenames do not provide priority. For a
|
||||
requested profile ID, an operator definition can partially override the
|
||||
embedded fallback while retaining unspecified fields. The maintained
|
||||
[`dnd-extraction` LLM profile](../examples/llm-profiles/dnd-extraction.yml)
|
||||
is a secret-free deployment artifact; production, development, and local
|
||||
deployments can each provide a complete definition with that same workload ID.
|
||||
Use workload-oriented IDs for new profiles instead of model names.
|
||||
is a secret-free deployment artifact. It also demonstrates a derived
|
||||
development profile that is not selected by the normal pipeline.
|
||||
[Operations](operations.md#promptkit-profile-deployment) owns the deployment
|
||||
workflow and credential-handling guidance.
|
||||
|
||||
@@ -144,27 +277,125 @@ a positive value limits simultaneous local generations. The application-wide
|
||||
backend field has an environment override. Omitting **local_backend** registers
|
||||
nothing and preserves existing built-in and endpoint-only profile behavior.
|
||||
|
||||
A file-backed PromptKit profile selects the registration by its case-sensitive
|
||||
backend ID:
|
||||
A PromptKit file uses keyed mappings for profiles, backends, and prompts. A
|
||||
profile selects a backend by its exact ID:
|
||||
|
||||
~~~yaml
|
||||
id: local-summary
|
||||
backend: local
|
||||
model: example-model
|
||||
profiles:
|
||||
local-summary:
|
||||
backend: local
|
||||
model: example-model
|
||||
~~~
|
||||
|
||||
Keep credentials out of the local-backend object. A PromptKit profile may name
|
||||
its credential environment variable through `api_key_env`; set that variable
|
||||
only in the run environment. PromptKit owns the
|
||||
[pinned profile-file format](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md),
|
||||
including `base_profile` inheritance. Notarius passes profiles through without
|
||||
merging them. Filesystem profiles cannot express PromptKit's in-memory
|
||||
`APIKeyRequired` setting; an unset `api_key_env` is optional and may reach the
|
||||
provider without authorization.
|
||||
only in the run environment. `api_key_required` is supported in profile files;
|
||||
inspection reports PromptKit's effective metadata but does not check whether
|
||||
the named environment variable is set. PromptKit owns the
|
||||
[pinned v0.12.0 format](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/formats.md),
|
||||
including native `extends` inheritance and field-presence rules. Notarius
|
||||
selects source groups and passes definitions through PromptKit.
|
||||
The [PromptKit upstream boundary](integrations/pkg-promptkit.md) identifies the
|
||||
supported package API, and [Operations](operations.md#operational-limits)
|
||||
describes the effective concurrency layers.
|
||||
|
||||
### Native PromptKit Sources And Migration
|
||||
|
||||
The configured `profile_file` or `profile_dir` is a PromptKit configuration
|
||||
source. It can contain profiles and backends, as well as prompt overlays.
|
||||
PromptKit's catalog is followed by registered application prompt/fallback
|
||||
assets and the selected operator source; Notarius's optional programmatic
|
||||
`local` backend is applied last. Composition, recursion, exact-key inheritance,
|
||||
and final-value validation are owned by PromptKit and ConfigKit. Explicit
|
||||
`null`, zero, false, blank, and omission may have different meanings; an empty
|
||||
map does not clear an inherited map.
|
||||
|
||||
When migrating external files from PromptKit before v0.12, place backend and
|
||||
profile entries beneath their `backends` and `profiles` mappings, and place
|
||||
prompt versions beneath `prompts.<prompt-id>.v1`. Remove embedded prompt
|
||||
`id`/`version` fields. Replace `base_profile` with native `extends`. There is
|
||||
no legacy-format parser or alias layer. Keep Notarius's `version: 5` and
|
||||
`promptkit.profile_file`/`profile_dir` settings unchanged.
|
||||
|
||||
Imports use native ConfigKit declarations relative to the file that declares
|
||||
them. A file-root layout keeps imported fragments outside a recursively
|
||||
selected profile directory, avoiding selection of the same definition both by
|
||||
directory discovery and by import:
|
||||
|
||||
~~~yaml
|
||||
# notarius.yml
|
||||
version: 5
|
||||
promptkit:
|
||||
profile_file: ./promptkit/entry.yml
|
||||
~~~
|
||||
|
||||
~~~yaml
|
||||
# promptkit/entry.yml
|
||||
configkit:
|
||||
imports:
|
||||
- path: fragments/profiles.yml
|
||||
profiles:
|
||||
dnd-extraction-development:
|
||||
extends: dnd-extraction
|
||||
temperature: 0.2
|
||||
~~~
|
||||
|
||||
PromptKit recursively discovers YAML files in a selected directory, and its
|
||||
inputs compose strictly; there is no filename-priority merger. Imports retain
|
||||
their declaring source roots, so relative message/content files resolve from
|
||||
the file that declares them. Use a separate file root for related imports
|
||||
when directory selection would otherwise load those files a second time.
|
||||
|
||||
The same unified document can declare a custom backend or select a catalog
|
||||
backend without redeclaring it:
|
||||
|
||||
~~~yaml
|
||||
backends:
|
||||
private-compatible:
|
||||
endpoint: https://llm.example.invalid/v1
|
||||
api_key_env: PRIVATE_LLM_API_KEY
|
||||
profiles:
|
||||
private-summary:
|
||||
backend: private-compatible
|
||||
model: summary-model
|
||||
catalog-summary:
|
||||
backend: openrouter
|
||||
model: openai/gpt-5.6-luna
|
||||
~~~
|
||||
|
||||
To override the embedded D&D fallback for a configured local backend, supply
|
||||
only the changed fields. Explicitly clear inherited settings that do not apply
|
||||
to the local target:
|
||||
|
||||
~~~yaml
|
||||
profiles:
|
||||
dnd-extraction:
|
||||
backend: local
|
||||
model: local-model
|
||||
service_tier: null
|
||||
~~~
|
||||
|
||||
The backend can be declared as `backends.local` in the operator source or
|
||||
registered through the optional `promptkit.local_backend` convenience object.
|
||||
The [source precedence](#native-promptkit-sources-and-migration) applies when
|
||||
both contribute. Prompt overlays use the same recursive source composition; for
|
||||
example, this changes one output setting while retaining the other embedded
|
||||
definition fields:
|
||||
|
||||
~~~yaml
|
||||
prompts:
|
||||
dnd.spells:
|
||||
v1:
|
||||
output:
|
||||
repair_attempts: 0
|
||||
~~~
|
||||
|
||||
Sequences such as prompt messages are replaced as a whole when overridden.
|
||||
Engine construction validates final effective prompt definitions and captures
|
||||
message files and static templates. Request preparation loads response schemas
|
||||
and resolves request variables and input artifacts. Profile preflight needs no
|
||||
transcript, credential value, or provider call, but can read prompt-content files.
|
||||
|
||||
`notarius config validate --pipeline <id>` resolves the selected pipeline and
|
||||
inspects every explicit effective profile without contacting a provider or
|
||||
requiring credential values. It rejects absent, malformed, or incompatible
|
||||
|
||||
@@ -1,18 +1,21 @@
|
||||
# ConfigKit Integration
|
||||
|
||||
Notarius uses the public root package
|
||||
[`gitea.maximumdirect.net/eric/ConfigKit`](https://gitea.maximumdirect.net/eric/ConfigKit)
|
||||
at v0.4.2 for version 5 configuration acquisition, imports, overlays, explicit
|
||||
overrides, diagnostics, and safe inspection rendering.
|
||||
Notarius application configuration uses
|
||||
[`go.maximumdirect.net/configkit`](https://forge.maximumdirect.net/base-layer/configkit/src/commit/c1c7e87f2cdf77a0cbf3327831ac05578e756b9e)
|
||||
at v1.2.0 for source selection, imports, overlays, identified collections,
|
||||
inheritance, explicit overrides, diagnostics, and frozen inspection. Imports
|
||||
and inheritance precede typed decoding; CLI assignments follow decoding and
|
||||
precede application resolution. Notarius does not repeat source composition.
|
||||
|
||||
Notarius owns the typed document, domain defaults and validation, runtime
|
||||
conversion, catalog-dependent pipeline resolution, source-aware configured
|
||||
paths, and command discovery. It uses only ConfigKit's public path, result,
|
||||
diagnostic, and resolution-source APIs. Upstream documentation owns general
|
||||
ConfigKit semantics; [Configuration](../config.md) owns Notarius's file and
|
||||
operator contract.
|
||||
Notarius supplies the document schema, defaults, normalization, validation,
|
||||
and the step identity registration. It owns runtime conversion and subsequent
|
||||
catalog-dependent pipeline resolution. `LoadSource` receives the CLI's chosen
|
||||
source; retained main-source evidence supplies the selected path. See
|
||||
[Configuration](../config.md) for the operator contract and the pinned
|
||||
upstream [consumer guide](https://forge.maximumdirect.net/base-layer/configkit/src/commit/c1c7e87f2cdf77a0cbf3327831ac05578e756b9e/docs/consumers/README.md)
|
||||
and [configuration contract](https://forge.maximumdirect.net/base-layer/configkit/src/commit/c1c7e87f2cdf77a0cbf3327831ac05578e756b9e/docs/config.md).
|
||||
|
||||
The v0.4.2 patch updates upstream release tooling and policy/licensing
|
||||
documentation without changing configuration behavior or the public API. See
|
||||
the canonical [ConfigKit attribution](../oss/attribution.md#configkit) for its
|
||||
license and notices.
|
||||
PromptKit v0.12.0 separately retains old-path ConfigKit v0.5.0 for its exposed
|
||||
source and inspection types. The two module identities share YAML v4.0.0-rc.6;
|
||||
their named types remain inside their owning boundaries. See the
|
||||
[PromptKit integration](pkg-promptkit.md) and [attribution](../oss/attribution.md#configkit).
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
# PromptKit Integration
|
||||
|
||||
Notarius pins
|
||||
[`gitea.maximumdirect.net/eric/promptkit` v0.11.2](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.2)
|
||||
[`gitea.maximumdirect.net/eric/promptkit` v0.12.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0)
|
||||
as its in-process prompt engine. The upstream
|
||||
[Go package consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.2/docs/consumers/pkg-promptkit.md)
|
||||
[Go package consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/consumers/pkg-promptkit.md)
|
||||
owns the public engine API, and the upstream
|
||||
[format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.2/docs/formats.md)
|
||||
[format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/formats.md)
|
||||
owns prompt, profile, and schema file contracts.
|
||||
|
||||
## Supported Boundary
|
||||
@@ -20,15 +20,20 @@ Notarius relies on the root `promptkit` package to:
|
||||
record credential-redacted details and run that exact execution;
|
||||
- return rendered debug material, validated structured output, selected
|
||||
profile, backend, effective model metadata, and token usage;
|
||||
- register the optional conventional `local` backend through `BackendLocal`,
|
||||
`LocalBackend`, and `WithBackend`;
|
||||
- compose profile and backend mappings through `ConfigurationSources`,
|
||||
`ProfileSource`, and `BackendSource`, retaining filesystem roots with
|
||||
`FilesystemInputs` and `DirectoryInputs`;
|
||||
- register the optional conventional `local` backend with `BackendSource`,
|
||||
`BackendLocal`, and `LocalBackend`;
|
||||
- distinguish structured-output validation failure from execution failure; and
|
||||
- identify a missing explicit profile through `ErrProfileNotFound` and backend
|
||||
admission exhaustion through `ErrCapacityExceeded`.
|
||||
|
||||
The pinned
|
||||
[`BackendLocal`, `LocalBackend`, and `WithBackend` API](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.2/backends.go)
|
||||
owns the registration and backend-capacity contract.
|
||||
[`BackendSource`, `BackendLocal`, and `LocalBackend` API](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/backends.go)
|
||||
owns keyed backend configuration and backend-capacity semantics. The
|
||||
[package consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/consumers/pkg-promptkit.md)
|
||||
describes source-group assembly.
|
||||
|
||||
For one completion, the adapter calls `PrepareExecution`, takes a
|
||||
caller-owned `Details` snapshot, and calls `RunPrepared` for that same opaque
|
||||
@@ -72,7 +77,7 @@ PromptKit capacity rejection into the provider-neutral Notarius
|
||||
`ErrLLMCapacityExceeded` contract. It may include the normalized selected
|
||||
backend ID in safe diagnostic context, without exposing PromptKit's capacity
|
||||
error type. PromptKit's built-in client retries eligible transient HTTP failures
|
||||
with its v0.11.2 default of two additional attempts per generation; Notarius
|
||||
with its v0.12.0 default of two additional attempts per generation; Notarius
|
||||
keeps pipeline-stage retries outside that completion boundary.
|
||||
|
||||
The adapter projects built-in generation failures as the application-owned
|
||||
@@ -84,20 +89,28 @@ exposing PromptKit error values to Notarius callers.
|
||||
## Profile Sources And Compatibility
|
||||
|
||||
Notarius gives PromptKit the configured operator profile source, registered
|
||||
application fallback profile assets, and optional backend registration through
|
||||
the same construction path for inspection and execution. PromptKit owns the
|
||||
resulting source precedence and strict profile parsing: a matching operator
|
||||
profile is a complete replacement for a fallback or built-in profile, while an
|
||||
invalid matching document fails instead of falling through. The operator
|
||||
application fallback profile assets, and optional backend configuration
|
||||
through the same construction path for inspection and execution. PromptKit
|
||||
owns source precedence, strict group composition, and recursive field overlay:
|
||||
omitted operator fields retain lower-layer values, while supplied fields
|
||||
replace them. The application group retains registered filesystem roots; the
|
||||
operator may be one file with native imports or a recursively discovered
|
||||
directory. Imports resolve resources from their declaring file. Prompt and
|
||||
profile definitions are validated after source composition, so a malformed
|
||||
lower-layer value superseded by a valid overlay is not separately audited. A
|
||||
malformed final effective definition fails construction. The operator
|
||||
configuration and deployment workflow are defined in
|
||||
[Configuration](../config.md#promptkit-profiles) and
|
||||
[Operations](../operations.md#promptkit-profile-deployment).
|
||||
|
||||
PromptKit owns `base_profile` resolution under its
|
||||
[pinned format rules](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.2/docs/formats.md).
|
||||
PromptKit and ConfigKit own `extends` inheritance under the
|
||||
[pinned format rules](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/formats.md).
|
||||
Notarius records the selected leaf identity and resolved target without parsing
|
||||
or merging inheritance. An unset filesystem `api_key_env` is optional and may
|
||||
reach the provider without authorization, which can result in a 401 or 403.
|
||||
or merging inheritance. `api_key_env` names the environment variable; inspection
|
||||
does not check whether it is set. PromptKit's effective `api_key_required`
|
||||
metadata is supported in file-backed profiles and reported as metadata, not
|
||||
as an availability check. Whether a usable credential reaches the provider
|
||||
remains an execution concern.
|
||||
|
||||
PromptKit accepts only the `developer`, `system`, `user`, and
|
||||
`assistant` text-chat roles after normalizing case and surrounding whitespace.
|
||||
@@ -124,16 +137,21 @@ PromptKit obtains its maintained OpenRouter and Rakestrawhome backend and
|
||||
profile catalogs from independently versioned transitive modules. Notarius
|
||||
does not import or register either catalog; PromptKit retains catalog source,
|
||||
identity, precedence, credential, and capacity ownership.
|
||||
The selected integration modules are
|
||||
[`promptkit-backend-openrouter/v3`](../oss/attribution.md#promptkit-openrouter-catalog)
|
||||
and
|
||||
[`promptkit-backend-rakestrawhome/v3`](../oss/attribution.md#promptkit-rakestrawhome-catalog),
|
||||
both pinned at `v3.20260930.0` for this compatibility boundary.
|
||||
|
||||
External catalog profile names and settings can change between dependency
|
||||
releases, including removal without aliases. Review directly selected IDs and
|
||||
inherited `base_profile` values when upgrading. An application-owned alias that
|
||||
inherited `extends` values when upgrading. An application-owned alias that
|
||||
inherits an external profile does not freeze that profile's settings.
|
||||
PromptKit's opt-in `promptkit-openrouter-standard` and
|
||||
`promptkit-openrouter-auto` names provide name continuity within its
|
||||
selected-and-tested dependency scope, not a fixed model or provider availability
|
||||
guarantee. See the upstream
|
||||
[catalog compatibility policy](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.2/docs/policy/release.md#catalog-profile-compatibility).
|
||||
[catalog compatibility policy](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/policy/release.md#catalog-profile-compatibility).
|
||||
|
||||
Use a complete operator-owned profile with explicit backend or endpoint, model,
|
||||
and settings when those choices must remain operator-controlled. The embedded
|
||||
@@ -141,13 +159,28 @@ and settings when those choices must remain operator-controlled. The embedded
|
||||
external catalog profile. An upgrade updates Notarius's compiled-in catalog
|
||||
checkpoint identity so prior profile-source checkpoints are not reused.
|
||||
|
||||
Notarius supports this boundary against PromptKit v0.11.2. Its fallback source,
|
||||
Notarius supports this boundary against PromptKit v0.12.0. Its native sources,
|
||||
prepared-execution, inspection, and typed capacity APIs are used as public
|
||||
upstream contracts; other PromptKit APIs or file-format behavior are not
|
||||
implicitly supported. A dependency upgrade requires reviewing the adapter,
|
||||
profile-source construction, and this compatibility statement against the
|
||||
pinned upstream documentation.
|
||||
|
||||
PromptKit v0.12.0 exposes old-path ConfigKit v0.5.0 types and reads its operator
|
||||
documents with that engine. Notarius application configuration uses the separate
|
||||
new-path ConfigKit integration; the adapter does not translate source graphs
|
||||
between the two.
|
||||
|
||||
Captured inspection describes one engine's immutable effective view. Source
|
||||
edits require constructing a new engine to become visible. Effective and
|
||||
explanation reports retain PromptKit's redacted configuration and source
|
||||
history contracts; the profile command is a separate Notarius-owned projection
|
||||
of copied, provider-neutral metadata. None of these views reads credential
|
||||
values or predicts provider defaults. See the public [CLI reference](../cli.md#config-effective),
|
||||
[explanation](../cli.md#config-explain), and
|
||||
[profile](../cli.md#config-profile) entries for their syntax and output
|
||||
contracts.
|
||||
|
||||
## Notarius Ownership
|
||||
|
||||
[LLM Runtime Internals](../internal/llm.md) describes how Notarius mounts
|
||||
@@ -171,8 +204,9 @@ production prompts use PromptKit's bounded structural-repair contract; their
|
||||
current declaration is one additional repair attempt. Notarius retains the
|
||||
transport-neutral boundary and does not expose PromptKit types to modules.
|
||||
|
||||
The v0.11.2 patch updates upstream policy and licensing documentation without
|
||||
changing runtime APIs, profile settings, or dependency/catalog versions.
|
||||
The v0.12.0 upgrade replaces the removed option-based configuration API with
|
||||
native ConfigKit-backed sources and keyed documents; release pins and notices
|
||||
are listed in the [canonical attribution](../oss/attribution.md#promptkit).
|
||||
See the canonical [PromptKit attribution](../oss/attribution.md#promptkit),
|
||||
[OpenRouter catalog notice](../oss/attribution.md#promptkit-openrouter-catalog),
|
||||
and [Rakestrawhome catalog notice](../oss/attribution.md#promptkit-rakestrawhome-catalog).
|
||||
|
||||
@@ -35,23 +35,36 @@ options before dispatch so that a missing production dependency fails as a
|
||||
command error rather than reaching execution.
|
||||
|
||||
Commands that need configuration use one shared ConfigKit-backed loader. The
|
||||
CLI discovers the file, passes the requested overlay and only explicit
|
||||
configuration-valued run overrides, and receives a **LoadedConfig**. The
|
||||
CLI constructs an exclusive file source for an explicit flag or environment
|
||||
selection, or a native required search source, then passes it to `LoadSource`
|
||||
with the requested overlay and only explicit configuration-valued run overrides.
|
||||
It receives a **LoadedConfig** and uses its retained main-source path for
|
||||
invocation provenance without a second search. The
|
||||
configured discovery and precedence contract is in
|
||||
[Configuration](../config.md), while the loading and resolution mechanics are
|
||||
in [Configuration Internals](configuration.md).
|
||||
|
||||
`config effective` and `config explain` only render the loaded ConfigKit view;
|
||||
they do not resolve pipelines or construct providers. Configuration validation without a selected pipeline checks structural
|
||||
`config effective` and `config explain` render the loaded ConfigKit view by
|
||||
default and do not resolve pipelines or construct providers. With explicit
|
||||
`--scope promptkit`, they construct the PromptKit inspection engine from the
|
||||
same captured source groups used by execution. Effective output is PromptKit's
|
||||
redacted YAML; explanation output shows its source history. `config profile`
|
||||
uses that engine to emit one neutral Notarius-owned JSON summary for the exact
|
||||
requested profile ID. These inspection commands load no run pipeline, prepare
|
||||
no request, resolve no credential value, and make no provider call. Ordinary
|
||||
prompt and provider-parameter content is not generally secret-free even though
|
||||
credential-shaped provider parameter keys are redacted.
|
||||
|
||||
Configuration validation without a selected pipeline checks structural
|
||||
configuration only. Validation with a selected pipeline also builds the
|
||||
effective catalog, resolves the pipeline, and verifies every explicit effective
|
||||
PromptKit profile. Selected LLM-backed input, chunk, lane, output, and validator
|
||||
profiles are inspected
|
||||
against the configured PromptKit source and backend registrations without
|
||||
loading a prompt or performing generation, so an unknown or invalid profile
|
||||
fails before pipeline preparation. Credential availability remains an
|
||||
execution-time concern. Pipeline listing validates configuration before
|
||||
returning normalized, sorted identifiers.
|
||||
profiles are inspected against the configured PromptKit source and backend
|
||||
registrations without preparing a request or performing generation. Engine
|
||||
construction validates effective prompt definitions and captures message content;
|
||||
an unknown or invalid profile fails before pipeline preparation.
|
||||
Credential availability remains an execution-time concern. Pipeline listing
|
||||
validates configuration before returning normalized, sorted identifiers.
|
||||
|
||||
## Production Composition
|
||||
|
||||
@@ -62,14 +75,17 @@ catalog used for resolution and the concrete constructors used for preparation.
|
||||
Tests may provide a catalog or registries instead; production code must not
|
||||
silently merge an injected partial catalog with production registrations.
|
||||
|
||||
The production LLM factory builds one PromptKit-backed client from the resolved
|
||||
**promptkit.profile_dir** or **promptkit.profile_file** source, attaches the
|
||||
profile-provenance recorder, creates one scheduler from the effective global
|
||||
LLM limit, and wraps the client before it reaches modules. Registration and LLM
|
||||
construction errors are returned before a pipeline is prepared. Configuration
|
||||
field definitions remain in [Configuration](../config.md#promptkit-profiles);
|
||||
the D&D registrar's fallback profile assets and the adapter mechanics remain in
|
||||
[LLM Runtime](llm.md).
|
||||
The production LLM factory builds one PromptKit-backed client from the native
|
||||
source groups assembled from PromptKit catalogs, registered application prompt
|
||||
and fallback assets, the optional operator file or recursively selected
|
||||
directory, and the optional local backend registration. The same construction
|
||||
path serves runtime, preflight, and configuration inspection. It attaches the
|
||||
profile-provenance recorder, captures one immutable PromptKit configuration
|
||||
identity, creates one scheduler from the effective global LLM limit, and wraps
|
||||
the client before it reaches modules. Registration and LLM construction errors
|
||||
are returned before a pipeline is prepared. Configuration fields remain in
|
||||
[Configuration](../config.md#promptkit-profiles); source ownership, capture,
|
||||
identity, and adapter mechanics remain in [LLM Runtime](llm.md).
|
||||
|
||||
The factory also accepts `LLMRuntimeOverrides`, whose reasoning pointer
|
||||
preserves inherit, replace, and clear states across the composition boundary.
|
||||
|
||||
@@ -15,12 +15,12 @@ runs anything.
|
||||
|
||||
| Boundary | Inputs | Outputs | Does not own |
|
||||
| --- | --- | --- | --- |
|
||||
| Loading | Selected file path, profile, overrides, and invocation directory | **LoadedConfig** with ConfigKit inspection metadata and a populated **Config** | Choosing the file path or reporting a command result. |
|
||||
| Loading | Selected ConfigKit source, overlay, overrides, and invocation directory | **LoadedConfig** with ConfigKit inspection metadata and a populated **Config** | Choosing the source or reporting a command result. |
|
||||
| Validation | **Config** | Structural configuration errors with pipeline, lane, or binding context | Module availability, capabilities, or construction. |
|
||||
| Resolution | Valid **Config**, selected pipeline and lanes, runtime reference changes, LLM override, and module catalog | **EffectiveConfig** with a **ResolvedPipeline** | Materializing reference bytes, preparing modules, execution, or filesystem state. |
|
||||
| Inspection | **LoadedConfig** and an optional JSON Pointer | Redacted effective YAML or explanation text | Pipeline selection, module preparation, or provider work. |
|
||||
|
||||
The CLI discovers a configuration file, invokes this package, and supplies the
|
||||
The CLI describes the selected source, invokes this package, and supplies the
|
||||
result to the framework. Configuration never reads an input file, constructs a
|
||||
module, or creates output, cache, or debug paths. Those responsibilities remain
|
||||
at their respective [CLI](cli.md), [pipeline](pipeline.md), and
|
||||
@@ -28,15 +28,32 @@ at their respective [CLI](cli.md), [pipeline](pipeline.md), and
|
||||
|
||||
## Loading And Validation
|
||||
|
||||
ConfigKit acquires the selected root, imports, optional selected overlay, and
|
||||
explicit CLI overrides in precedence order. Its document definition supplies
|
||||
typed defaults, source-aware path normalization, and structural validation
|
||||
before conversion to the independently owned runtime **Config**. Only
|
||||
`Load` selects an exclusive file; `LoadSource` accepts the CLI's source and
|
||||
enables imports through the same definition and callback path. ConfigKit composes
|
||||
the main input, imports, and optional overlay, then expands native inheritance
|
||||
before typed decoding. Explicit CLI assignments follow decoding and precede
|
||||
application resolution. The document definition supplies typed
|
||||
defaults and structural validation; Notarius's recorded resolution changes
|
||||
normalize configured paths after decode. Reference paths resolve before key
|
||||
normalization moves their source coordinates. Ordinary list leaf edits are
|
||||
recorded at the leaf; a changed step identity requires a covering list record.
|
||||
Actual map-key moves retain structural records. Path normalization follows the source
|
||||
of the surviving value, including a value copied through inheritance, rather
|
||||
than the inheritance directive's location. Inherited input remains explicit
|
||||
for presence-sensitive behavior, including zero, false, and empty values;
|
||||
existing field validation still governs whether the resulting values are valid.
|
||||
ConfigKit's inspection history retains inheritance transfers alongside later
|
||||
normalization, and effective configuration renders the normalized values used
|
||||
by runtime conversion. The private typed document consumes the directive, so
|
||||
no upstream inheritance type or unresolved directive reaches runtime pipeline
|
||||
or module contracts. See [Configuration](../config.md#sibling-inheritance)
|
||||
for the operator syntax and field rules. Only
|
||||
`NOTARIUS_CONFIG` participates in discovery; retired field-level environment
|
||||
variables do not participate in loading. Version 5 decoding rejects scalar
|
||||
module bindings and scalar external references. ConfigKit retains redacted,
|
||||
historical inspection metadata while **LoadedConfig** keeps that implementation
|
||||
detail private behind effective and explanation writers.
|
||||
detail private behind effective and explanation writers. `MainSourcePath` reads
|
||||
the non-imported main file from retained sources, and runtime conversion stays detached.
|
||||
|
||||
**Config.Validate** checks configuration-only invariants before resolution. It
|
||||
rejects incompatible profile sources, invalid state-surface values, unsupported
|
||||
|
||||
@@ -72,8 +72,9 @@ An empty request profile lets the prompt select its configured default. Before a
|
||||
run begins, the CLI asks the adapter to inspect every explicit profile on the
|
||||
resolved selected LLM-backed bindings and validators, including inherited
|
||||
pipeline profiles. Inspection resolves the profile and its selected backend and
|
||||
target without loading a prompt, reading credentials, admitting capacity, or
|
||||
contacting a provider, so a missing or invalid explicit profile fails before
|
||||
target without preparing a request, reading credentials, admitting capacity, or
|
||||
contacting a provider. Engine construction validates effective prompt definitions
|
||||
and captures message files and static templates. A missing or invalid profile fails before
|
||||
stage execution while a valid `api_key_env` may remain unset. Calls record the
|
||||
profile actually selected by PromptKit. The recorder trims and deduplicates
|
||||
non-secret profile identity, provider, model, selected backend ID, and
|
||||
@@ -83,35 +84,43 @@ retain an empty backend ID, which the published JSON omits. Successful
|
||||
completion responses and recorded profile manifests identify the adapter
|
||||
provider as `promptkit`.
|
||||
|
||||
The CLI's profile-inspection engine and the production adapter use the same
|
||||
profile-source construction to apply the configured profile directory or file,
|
||||
the optional registered fallback profile assets, and the optional conventional
|
||||
`local` backend. Preflight therefore resolves the same profile sources and
|
||||
backend membership as runtime without performing generation. Fallback assets
|
||||
are mounted only when at least one source is registered. The production D&D
|
||||
registrar contributes its `dnd-extraction` fallback, and the maintained D&D
|
||||
prompts select that logical ID by default. PromptKit owns source precedence and
|
||||
profile parsing and inheritance: an operator-provided matching profile takes precedence over a
|
||||
fallback profile without Notarius merging either document.
|
||||
When the registration is absent, a profile selecting `backend: local` fails
|
||||
inspection instead of falling back to a built-in or endpoint-only target.
|
||||
The CLI's profile-inspection engine and production adapter use the same source
|
||||
assembly: PromptKit catalogs, one application group containing registered
|
||||
prompts and fallback profiles, the selected operator file or directory, and
|
||||
the optional final `local` backend registration. Execution, preflight, and
|
||||
inspection therefore capture the same effective definitions. PromptKit and
|
||||
ConfigKit compose recursively across groups while retaining each input's file
|
||||
context; inputs within a group compose strictly. Operator imports are enabled
|
||||
for the selected operator input, and imports retain their declaring file's
|
||||
relative resource roots. PromptKit validates final effective definitions at
|
||||
engine construction. It does not run a separate audit that rejects a malformed
|
||||
definition superseded by a later overlay. The D&D registrar contributes its
|
||||
`dnd-extraction` fallback, and maintained D&D prompts select that logical ID
|
||||
by default. A profile selecting `backend: local` can use an operator
|
||||
`backends.local` declaration or the optional Notarius registration; see
|
||||
[native source rules](../config.md#native-promptkit-sources-and-migration).
|
||||
|
||||
Before execution, the adapter also contributes a non-secret checkpoint
|
||||
fingerprint for the effective PromptKit profile source. It combines the
|
||||
identity of PromptKit's compiled-in profile catalog with a deterministic digest
|
||||
of every YAML profile in the configured profile directory, or of the configured
|
||||
profile file, and a deterministic digest of the flattened fallback profile
|
||||
assets. The fingerprint contains neither profile content nor source paths. It
|
||||
covers inherited pipeline profiles, explicit binding profiles, and
|
||||
prompt-selected defaults, so changing a model or other profile setting cannot
|
||||
reuse checkpoints created under the
|
||||
prior profile source. This cache identity is independent of durable
|
||||
profile provenance: run manifests continue to list only profiles actually
|
||||
observed during LLM calls. When the local backend is registered, a second
|
||||
fingerprint hashes its trimmed endpoint behind a stable marker. Changing that
|
||||
semantic execution target invalidates checkpoint reuse. The raw endpoint is not
|
||||
stored in checkpoint identity, and the local concurrency limit is excluded
|
||||
because it changes scheduling rather than execution semantics.
|
||||
This PromptKit integration uses the old-path ConfigKit v0.5.0 types exposed by
|
||||
PromptKit v0.12.0. Application configuration uses the distinct new-path
|
||||
ConfigKit package; those named types do not cross the adapter boundary.
|
||||
|
||||
Each engine captures its definitions at construction. A later source-file edit
|
||||
does not mutate an existing engine; constructing a new engine captures the
|
||||
updated view. Before execution, the adapter computes one immutable,
|
||||
versioned `promptkit_configuration` checkpoint identity from public captured
|
||||
inspection APIs. The private digest covers captured effective configuration,
|
||||
all effective profile targets and credential-policy metadata (including
|
||||
provider parameters), and every effective prompt ID/version and captured
|
||||
content hash. The identity also includes PromptKit and the selected OpenRouter
|
||||
and Rakestrawhome catalog versions. Local-backend concurrency settings remain
|
||||
excluded because they affect scheduling; its endpoint is represented only in
|
||||
the private digest, never as checkpoint metadata. Neither the private digest
|
||||
payload, raw sources, paths, endpoints, provider parameters, nor prompt bodies
|
||||
are published. An existing engine retains its identity after files change; a
|
||||
new engine gets the identity of its newly captured definitions. This replaces
|
||||
the former live-file profile and local-endpoint fingerprints. Run manifests
|
||||
remain independent and list only profile choices actually observed during LLM
|
||||
calls.
|
||||
|
||||
## Shared Provider-Call Limit
|
||||
|
||||
@@ -140,11 +149,16 @@ only by the Notarius scheduler.
|
||||
## Prompt And Schema Assets
|
||||
|
||||
An `AssetRegistry` collects prompt, schema, and optional fallback-profile
|
||||
filesystems from production module families. It flattens registered roots into
|
||||
the corresponding PromptKit filesystems and rejects invalid roots, unreadable
|
||||
assets, duplicate paths, and missing prompt or schema files during preparation.
|
||||
Fallback assets receive a safe content digest for checkpoint identity; raw
|
||||
paths and bytes are never included. The framework’s `promptfs` helper combines
|
||||
filesystems from production module families. It retains each registered root's
|
||||
filesystem context when constructing PromptKit's application source group.
|
||||
Schema files are supplied separately through the engine's schema filesystem;
|
||||
PromptKit reads and validates a selected schema while preparing that request,
|
||||
not while constructing the configuration snapshot. Prompt and fallback
|
||||
definitions are validated after native source composition, and referenced
|
||||
prompt content is captured with the engine. Checkpoint identity uses captured
|
||||
prompt content hashes rather than rescanning live files or hashing only
|
||||
content-file paths. Raw paths and bytes are never included. The framework’s
|
||||
`promptfs` helper combines
|
||||
module-selected prompt files with reusable domain fragments without making the
|
||||
framework depend on D&D content.
|
||||
|
||||
@@ -294,6 +308,10 @@ When debug recording is enabled, the pipeline decorates the shared client. The
|
||||
wrapper records prepared prompt and response material, timing, selected profile
|
||||
and backend, effective model parameters, and call identifiers in the run’s
|
||||
debug bundle, including material available from a failed structured completion.
|
||||
The profile-inspection CLI projects copied values into its Notarius-owned
|
||||
neutral JSON DTO; effective configuration and explanation remain PromptKit's
|
||||
redacted YAML and source-history views. Credential names and `APIKeyRequired`
|
||||
metadata are configuration metadata, not credential-availability checks.
|
||||
Effective parameters use PromptKit's stable lower-case JSON field names and may
|
||||
include `backend_id`. For a successful completion, a debug-write failure is
|
||||
surfaced; when the completion already failed, its call error remains the
|
||||
|
||||
@@ -8,7 +8,7 @@ who need implementation mechanics should read [Run State Internals](internal/sta
|
||||
## Source Deployment
|
||||
|
||||
Linux is the supported deployment platform. Install a pinned source release
|
||||
with the Go version declared in `go.mod` (currently Go 1.25.5):
|
||||
with the Go version declared in `go.mod` (currently Go 1.27.0):
|
||||
|
||||
~~~sh
|
||||
GOWORK=off go install \
|
||||
@@ -99,24 +99,25 @@ Profile deployment has four distinct layers:
|
||||
|
||||
| Layer | Owner | Operational role |
|
||||
| --- | --- | --- |
|
||||
| Prompts and schemas | Notarius module families | Embedded request and structured-output definitions. They are not deployment profile files. |
|
||||
| Prompts and schemas | Notarius module families, with optional operator prompt overlays | Embedded request and structured-output definitions; operator configuration may recursively overlay prompt fields. Schemas remain application assets. |
|
||||
| Fallback profiles | Notarius module families | Embedded application defaults, including D&D's `dnd-extraction` profile. |
|
||||
| Built-in profiles | PromptKit | Upstream catalog entries available when no higher-precedence source defines an ID. |
|
||||
| Operator profiles | Deployment filesystem | Complete environment-specific definitions selected by `promptkit.profile_file` or `promptkit.profile_dir`. |
|
||||
| Operator configuration | Deployment filesystem | Native profile/backend definitions and prompt overlays selected by `promptkit.profile_file` or `promptkit.profile_dir`. |
|
||||
|
||||
The maintained D&D pipeline uses the workload ID `dnd-extraction`. The
|
||||
embedded fallback makes that ID usable without an operator file. Production,
|
||||
development, and local deployments can each install a different complete
|
||||
definition for the same ID, retaining the pipeline while choosing their own
|
||||
model, backend, timeout, or reasoning policy. An operator definition wins over
|
||||
the fallback; it is not merged with it. The configuration field and full
|
||||
precedence rules are owned by [Configuration](config.md#promptkit-profiles).
|
||||
development, and local deployments can each overlay settings for the same ID,
|
||||
retaining inherited values they do not change. Notarius's registered fallback
|
||||
configuration precedes the selected operator source, and groups compose by
|
||||
native recursive overlays; a configured Notarius local backend is applied
|
||||
after the operator source. The configuration fields and source precedence are
|
||||
owned by [Configuration](config.md#promptkit-profiles).
|
||||
|
||||
Use a profile source owned by the service account, keep it readable only by
|
||||
the intended operator, and supply provider credentials through the service
|
||||
environment—not in the Notarius configuration or profile YAML. The maintained
|
||||
[LLM profile](../examples/llm-profiles/dnd-extraction.yml) is secret-free and
|
||||
can be copied as a format starting point. Validate a deployment without a
|
||||
[LLM profiles](../examples/llm-profiles/dnd-extraction.yml) are secret-free
|
||||
and can be copied as a format starting point. Validate a deployment without a
|
||||
provider call or credentials:
|
||||
|
||||
~~~sh
|
||||
@@ -132,6 +133,40 @@ surviving declaration. The complete example's
|
||||
Use absolute paths such as `/etc/notarius/profiles/dnd-extraction.yml` for
|
||||
services and containers when that is clearer for the deployment.
|
||||
|
||||
### Inspect Before Deployment
|
||||
|
||||
Use the three configuration reports together when reviewing an operator
|
||||
source:
|
||||
|
||||
~~~sh
|
||||
notarius config effective --scope promptkit --config /etc/notarius/config.yml
|
||||
notarius config explain --scope promptkit --config /etc/notarius/config.yml \
|
||||
/profiles/dnd-extraction/model
|
||||
notarius config profile --config /etc/notarius/config.yml dnd-extraction
|
||||
~~~
|
||||
|
||||
Each report command constructs an inspection engine from the selected
|
||||
configuration and registered application assets. Construction validates and
|
||||
captures the effective definitions; after editing a source, run the command
|
||||
again to inspect a newly constructed engine. These commands do not resolve
|
||||
credential values, call a provider, or prepare a request. `config profile`
|
||||
reports the named profile's resolved target and effective credential metadata,
|
||||
not whether a credential is available or which profile a particular prompt or
|
||||
run will select.
|
||||
|
||||
Effective and explanation reports can include ordinary inline prompt and
|
||||
provider-parameter data. Credential-shaped provider parameter keys are
|
||||
redacted, but the rest of the content is not treated as secret-free. Reports
|
||||
are written only when requested; Notarius does not automatically persist them
|
||||
in run output or debug bundles. Operators own their destination, access, and
|
||||
retention when redirecting or storing reports.
|
||||
|
||||
Configuration reports are distinct from a prepared execution or a requested
|
||||
debug trace. They describe captured definitions and source history, not
|
||||
rendered request messages, runtime credential values, or an individual
|
||||
provider interaction. The [CLI reference](cli.md#config-effective) documents
|
||||
the report syntax and output contracts.
|
||||
|
||||
## Run Lifecycle
|
||||
|
||||
Use the [run command](cli.md#run) to start a pipeline. A valid invocation loads
|
||||
|
||||
@@ -11,14 +11,14 @@ Copyright (C) 2026 Eric Rakestraw
|
||||
SPDX-License-Identifier: CC-BY-SA-4.0
|
||||
|
||||
License and notices: [CC BY-SA 4.0](licenses/CC-BY-SA-4.0.txt) |
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/COPYRIGHT) |
|
||||
[LICENSE](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/LICENSE).
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/policy/src/commit/593c9b52273c5c879785ae9e995d21026df467b4/COPYRIGHT) |
|
||||
[LICENSE](https://gitea.maximumdirect.net/eric/policy/src/commit/593c9b52273c5c879785ae9e995d21026df467b4/LICENSE).
|
||||
|
||||
[Source](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843).
|
||||
[Source](https://gitea.maximumdirect.net/eric/policy/src/commit/593c9b52273c5c879785ae9e995d21026df467b4).
|
||||
|
||||
## ConfigKit
|
||||
|
||||
Selected module: `gitea.maximumdirect.net/eric/ConfigKit` at `v0.4.2`.
|
||||
Selected module: `go.maximumdirect.net/configkit` at `v1.2.0`.
|
||||
|
||||
Copyright (C) 2026 Eric Rakestraw
|
||||
|
||||
@@ -26,14 +26,29 @@ SPDX-License-Identifier: LGPL-3.0-or-later
|
||||
|
||||
License and notices: [LGPL 3.0](licenses/LGPL-3.0.txt) |
|
||||
[GPL 3.0](licenses/GPL-3.0.txt) |
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/67c03690181c5af8dde6be325abac241dae6732f/COPYRIGHT) |
|
||||
[LICENSE](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/67c03690181c5af8dde6be325abac241dae6732f/LICENSE).
|
||||
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/configkit/src/commit/c1c7e87f2cdf77a0cbf3327831ac05578e756b9e/COPYRIGHT) |
|
||||
[LICENSE](https://forge.maximumdirect.net/base-layer/configkit/src/commit/c1c7e87f2cdf77a0cbf3327831ac05578e756b9e/LICENSE).
|
||||
|
||||
[Source](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/67c03690181c5af8dde6be325abac241dae6732f).
|
||||
[Source](https://forge.maximumdirect.net/base-layer/configkit/src/commit/c1c7e87f2cdf77a0cbf3327831ac05578e756b9e).
|
||||
|
||||
## ConfigKit (old module path)
|
||||
|
||||
Selected module: `gitea.maximumdirect.net/eric/ConfigKit` at `v0.5.0`.
|
||||
|
||||
Copyright (C) 2026 Eric Rakestraw
|
||||
|
||||
SPDX-License-Identifier: LGPL-3.0-or-later
|
||||
|
||||
License and notices: [LGPL 3.0](licenses/LGPL-3.0.txt) |
|
||||
[GPL 3.0](licenses/GPL-3.0.txt) |
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/dc09959cca4e105f49ec316c6d4ba585bd2340bd/COPYRIGHT) |
|
||||
[LICENSE](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/dc09959cca4e105f49ec316c6d4ba585bd2340bd/LICENSE).
|
||||
|
||||
[Source](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/dc09959cca4e105f49ec316c6d4ba585bd2340bd).
|
||||
|
||||
## PromptKit
|
||||
|
||||
Selected module: `gitea.maximumdirect.net/eric/promptkit` at `v0.11.2`.
|
||||
Selected module: `gitea.maximumdirect.net/eric/promptkit` at `v0.12.0`.
|
||||
|
||||
Copyright (C) 2026 Eric Rakestraw
|
||||
|
||||
@@ -41,38 +56,38 @@ SPDX-License-Identifier: LGPL-3.0-or-later
|
||||
|
||||
License and notices: [LGPL 3.0](licenses/LGPL-3.0.txt) |
|
||||
[GPL 3.0](licenses/GPL-3.0.txt) |
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/commit/12d6d80024e80c552d122a076c22429bf6f2f6c2/COPYRIGHT) |
|
||||
[LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/commit/12d6d80024e80c552d122a076c22429bf6f2f6c2/LICENSE).
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/commit/4b8169da286917fde7875425bb32b799b12c6af3/COPYRIGHT) |
|
||||
[LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/commit/4b8169da286917fde7875425bb32b799b12c6af3/LICENSE).
|
||||
|
||||
[Source](https://gitea.maximumdirect.net/eric/promptkit/src/commit/12d6d80024e80c552d122a076c22429bf6f2f6c2).
|
||||
[Source](https://gitea.maximumdirect.net/eric/promptkit/src/commit/4b8169da286917fde7875425bb32b799b12c6af3).
|
||||
|
||||
## PromptKit OpenRouter Catalog
|
||||
|
||||
Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2` at `v2.20260926.0`.
|
||||
Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3` at `v3.20260930.0`.
|
||||
|
||||
Copyright (C) 2026 Eric Rakestraw
|
||||
|
||||
SPDX-License-Identifier: MIT
|
||||
|
||||
License and notices: [MIT](licenses/MIT.txt) |
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/4f731d217e0bc39151c010f50fb5912706e7b6b3/COPYRIGHT) |
|
||||
[LICENSE](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/4f731d217e0bc39151c010f50fb5912706e7b6b3/LICENSE).
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/f17d208cb7ad1fb043cda2266cd3011d11b40aa0/COPYRIGHT) |
|
||||
[LICENSE](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/f17d208cb7ad1fb043cda2266cd3011d11b40aa0/LICENSE).
|
||||
|
||||
[Source](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/4f731d217e0bc39151c010f50fb5912706e7b6b3).
|
||||
[Source](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/f17d208cb7ad1fb043cda2266cd3011d11b40aa0).
|
||||
|
||||
## PromptKit Rakestrawhome Catalog
|
||||
|
||||
Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2` at `v2.20260926.0`.
|
||||
Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3` at `v3.20260930.0`.
|
||||
|
||||
Copyright (C) 2026 Eric Rakestraw
|
||||
|
||||
SPDX-License-Identifier: MIT
|
||||
|
||||
License and notices: [MIT](licenses/MIT.txt) |
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/48918a3ad39fd99843cda94a86133b1b3881d596/COPYRIGHT) |
|
||||
[LICENSE](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/48918a3ad39fd99843cda94a86133b1b3881d596/LICENSE).
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/993443d88c451901f4c917fa00c204fd3b54d28d/COPYRIGHT) |
|
||||
[LICENSE](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/993443d88c451901f4c917fa00c204fd3b54d28d/LICENSE).
|
||||
|
||||
[Source](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/48918a3ad39fd99843cda94a86133b1b3881d596).
|
||||
[Source](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/993443d88c451901f4c917fa00c204fd3b54d28d).
|
||||
|
||||
## jsonschema
|
||||
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
|
||||
**Base Policy Version:** 1.3
|
||||
|
||||
**Last Updated**: 2026-09-28
|
||||
**Last Updated**: 2026-11-17
|
||||
|
||||
**Upstream Source**: https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/agents.md
|
||||
**Upstream Source**: https://gitea.maximumdirect.net/eric/policy/src/commit/593c9b52273c5c879785ae9e995d21026df467b4/docs/policy/agents.md
|
||||
|
||||
## Purpose
|
||||
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
|
||||
**Base Policy Version:** 1.3
|
||||
|
||||
**Last Updated**: 2026-09-28
|
||||
**Last Updated**: 2026-11-17
|
||||
|
||||
**Upstream Source**: https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/architecture.md
|
||||
**Upstream Source**: https://gitea.maximumdirect.net/eric/policy/src/commit/593c9b52273c5c879785ae9e995d21026df467b4/docs/policy/architecture.md
|
||||
|
||||
## Purpose And Ownership
|
||||
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
# Documentation Policy
|
||||
|
||||
**Base Policy Version:** 1.11
|
||||
**Base Policy Version:** 1.12
|
||||
|
||||
**Last Updated**: 2026-09-28
|
||||
**Last Updated**: 2026-11-17
|
||||
|
||||
**Upstream Source**: https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/documentation.md
|
||||
**Upstream Source**: https://gitea.maximumdirect.net/eric/policy/src/commit/593c9b52273c5c879785ae9e995d21026df467b4/docs/policy/documentation.md
|
||||
|
||||
## Purpose
|
||||
|
||||
@@ -86,8 +86,8 @@ secret values.
|
||||
| Integration contracts for external components (if applicable) | `docs/integrations/` | External formats and protocols, upstream and downstream contracts, logical output paths and schemas, media types, and compatibility. | Physical runtime placement and lifecycle, internal transformations, CLI syntax, configuration defaults. |
|
||||
| Inventory of implemented internal packages | `docs/internal/overview.md` | Implemented packages and components, responsibilities, and links to focused internal docs. | Normative architecture, contributor reading policy, external contracts. |
|
||||
| Behavior and architecture of internal packages | Other files under `docs/internal/` | Implementation flow, collaborators, state transitions, package-local guarantees and failures, and relevant tests. | Global architecture invariants, configuration definitions and defaults, external schemas, operator procedures. |
|
||||
| Architectural decision records | `docs/adr/` | Significant decisions, context, alternatives, rationale, consequences, and supersession history. | Current behavior reference, implementation status, task sequencing. |
|
||||
| Future work and implementation status | `docs/roadmap/` | Proposed, accepted, deferred, or rejected work; implementation status, sequencing, and task breakdowns. | Implemented behavior reference and architectural decision rationale. |
|
||||
| Architectural decision records | `docs/adr/` | Records of significant architectural decisions and supporting rationale. | Ordinary design choices, current behavior reference, implementation status, task sequencing. |
|
||||
| Future work and implementation status | `docs/roadmap/` | Proposed, accepted, or future work; implementation status, sequencing, and task breakdowns. | Implemented behavior reference and architectural decision rationale. |
|
||||
| Release-note history | `docs/releases/` | One checked-in summary per source release under the procedure; the note at its immutable tag is the release record. | Current commands, behavior, contracts, and compatibility definitions. |
|
||||
| Complete copyable artifacts | `examples/` | Maintained configuration, inputs, and other copyable or runnable files. | Field-by-field reference, command reference, prose explanation. |
|
||||
|
||||
@@ -153,8 +153,15 @@ Use sequentially numbered ADR filenames such as
|
||||
|
||||
Treat the decision content of an accepted ADR as immutable. When a decision
|
||||
changes, create a new ADR and update the earlier ADR's status to superseded.
|
||||
Rejected architectural alternatives belong in the ADR; rejected product ideas
|
||||
belong in the roadmap.
|
||||
|
||||
ADRs should be reserved for major architectural decisions, such as ownership
|
||||
boundaries, large-scale architectural features, or invariants that rise to
|
||||
the level of architectural significance. Routine choices, implementation
|
||||
decisions, and rejected suggestions do not warrant an ADR unless they
|
||||
otherwise represent a significant architectural decision.
|
||||
|
||||
Roadmaps should preserve plausible future work, not rejected product
|
||||
ideas.
|
||||
|
||||
## Maintenance
|
||||
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
|
||||
**Base Policy Version:** 1.0
|
||||
|
||||
**Last Updated**: 2026-09-28
|
||||
**Last Updated**: 2026-11-17
|
||||
|
||||
**Upstream Source**: https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/meta.md
|
||||
**Upstream Source**: https://gitea.maximumdirect.net/eric/policy/src/commit/593c9b52273c5c879785ae9e995d21026df467b4/docs/policy/meta.md
|
||||
|
||||
## Origin And Local Authority
|
||||
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
# Open Source Licensing And Attribution Policy
|
||||
|
||||
**Base Policy Version:** 1.8
|
||||
**Base Policy Version:** 1.9
|
||||
|
||||
**Last Updated**: 2026-09-28
|
||||
**Last Updated**: 2026-11-17
|
||||
|
||||
**Upstream Source**: https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/oss.md
|
||||
**Upstream Source**: https://gitea.maximumdirect.net/eric/policy/src/commit/593c9b52273c5c879785ae9e995d21026df467b4/docs/policy/oss.md
|
||||
|
||||
## Purpose And Scope
|
||||
|
||||
@@ -67,6 +67,9 @@ individual packages.
|
||||
Use the upstream [application COPYRIGHT example](https://gitea.maximumdirect.net/eric/policy/src/commit/a469273adda336625f96b9d769ea27b33999f146/examples/licensing/application/COPYRIGHT)
|
||||
or [library COPYRIGHT example](https://gitea.maximumdirect.net/eric/policy/src/commit/a469273adda336625f96b9d769ea27b33999f146/examples/licensing/library/COPYRIGHT)
|
||||
to create the project’s root `COPYRIGHT` file.
|
||||
Use the COPYRIGHT examples exactly as provided, except for placeholder
|
||||
substitutions and changes expressly required or authorized by this policy or
|
||||
the author.
|
||||
Use verified project names, contribution years, and copyright holders.
|
||||
For another selected license, adapt the grant, SPDX expression, disclaimer, and
|
||||
text references together to reflect the author's selected grant.
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
# Release Policy
|
||||
|
||||
**Base Policy Version:** 1.4
|
||||
**Base Policy Version:** 1.7
|
||||
|
||||
**Last Updated**: 2026-09-28
|
||||
**Last Updated**: 2026-11-17
|
||||
|
||||
**Upstream Source**: https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/release.md
|
||||
**Upstream Source**: https://gitea.maximumdirect.net/eric/policy/src/commit/593c9b52273c5c879785ae9e995d21026df467b4/docs/policy/release.md
|
||||
|
||||
## Purpose
|
||||
|
||||
@@ -12,6 +12,9 @@ This policy defines a proportionate workflow for validating and publishing an
|
||||
intended source revision under a fresh version. Releases should require little
|
||||
routine intervention, avoid repeated work, and preserve published contents.
|
||||
|
||||
Adopt stronger verification or reproducibility requirements through an explicit
|
||||
project-specific decision justified by the intended audience and support needs.
|
||||
|
||||
The repository-specific commands belong in `docs/release.md`. Historical release
|
||||
summaries belong in `docs/releases/`. The procedure must agree with the adopted
|
||||
policy and link to it rather than introduce additional release requirements.
|
||||
@@ -52,9 +55,14 @@ requires a deliberate update to the policy and procedure where needed.
|
||||
For releases at or after `v1.0.0`:
|
||||
|
||||
- a major release may make intentional incompatible changes;
|
||||
- a minor release may add backward-compatible behavior; and
|
||||
- a minor release may add backward-compatible behavior or raise the Go consumer
|
||||
minimum as described below; and
|
||||
- a patch release may make backward-compatible fixes.
|
||||
|
||||
In this context, "backward-compatible" means only that pre-existing, valid
|
||||
configurations will continue to work following the relase. Behavior need not
|
||||
remain strictly identical.
|
||||
|
||||
Before `v1.0.0`, a minor release may change a documented public, operational,
|
||||
configuration, data, integration, or compatibility contract when the release
|
||||
note explains the impact and required consumer action. A patch release must
|
||||
@@ -64,6 +72,36 @@ Additional compatibility promises and deliberate exceptions belong in the
|
||||
adopted policy or canonical contract documentation. Release notes explain
|
||||
changes; they do not redefine current compatibility contracts.
|
||||
|
||||
## Consumer Minimum And Release Toolchain
|
||||
|
||||
Distinguish the minimum toolchain supported for consumers from the toolchain
|
||||
maintainers use to build and validate a release. Record the consumer minimum in
|
||||
the language's standard metadata and explain its support meaning in canonical
|
||||
documentation. Define the selected release toolchain and its setup commands in
|
||||
`docs/release.md`. Toolchain selection and upgrade cadence are repository-specific
|
||||
choices.
|
||||
|
||||
For Go repositories, normally declare the initial stable release of the selected
|
||||
series as the consumer minimum, such as `go 1.26.0`. Use a higher patch minimum
|
||||
when a dependency or an identified correctness fix requires it, and document
|
||||
the reason.
|
||||
|
||||
For release-policy purposes, successful build and release validation on the
|
||||
selected Go patch release are sufficient to presume support across that Go
|
||||
series, subject to the declared consumer minimum and concrete evidence of
|
||||
incompatibility. Investigate specific compatibility concerns as they arise.
|
||||
|
||||
Use the selected release toolchain for candidate validation and release builds,
|
||||
including CI jobs performing those tasks. A newer validation patch version alone
|
||||
leaves the consumer minimum unchanged.
|
||||
|
||||
A minor release may raise the Go consumer minimum, including from one Go series
|
||||
to the next, both before and after `v1.0.0`. This is an explicit exception to the
|
||||
backward-compatibility requirement for minor releases. Patch releases preserve
|
||||
the consumer minimum. Explain an increase's impact and required consumer action
|
||||
in the release note. Other incompatible changes remain subject to the versioning
|
||||
rules above.
|
||||
|
||||
## Release Documentation And Preparation
|
||||
|
||||
Every new release must include `docs/releases/<version>.md` in the tagged
|
||||
@@ -191,7 +229,8 @@ release without replacing its contents or reusing its version.
|
||||
Maintain `docs/release.md` as a short, executable maintainer runbook. It must
|
||||
identify:
|
||||
|
||||
1. prerequisites, primary branch, and authoritative remote;
|
||||
1. prerequisites, selected release toolchain, primary branch, and authoritative
|
||||
remote;
|
||||
2. version and release-note preparation;
|
||||
3. one publication command with its selected local checks and automated guards;
|
||||
4. source-only delivery, required build targets, and automation responsibilities;
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
|
||||
**Base Policy Version:** 1.3
|
||||
|
||||
**Last Updated**: 2026-09-28
|
||||
**Last Updated**: 2026-11-17
|
||||
|
||||
**Upstream Source**: https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/testing.md
|
||||
**Upstream Source**: https://gitea.maximumdirect.net/eric/policy/src/commit/593c9b52273c5c879785ae9e995d21026df467b4/docs/policy/testing.md
|
||||
|
||||
## Purpose
|
||||
|
||||
|
||||
@@ -7,9 +7,22 @@ for development, and Windows is unsupported. `main` on `origin` is authoritative
|
||||
|
||||
## Prepare
|
||||
|
||||
Use Git, a POSIX shell, and Go 1.25.5 or newer. Configure ordinary Git credentials
|
||||
for `origin` and private-module access without putting secrets in the repository.
|
||||
Prepare dependencies before offline validation where necessary:
|
||||
Use Git, a POSIX shell, and a Go installation supporting toolchain selection.
|
||||
The selected release toolchain is Go 1.27.0. Select it for candidate validation
|
||||
and release builds, including any CI performing those tasks:
|
||||
|
||||
```sh
|
||||
export GOTOOLCHAIN=go1.27.0
|
||||
go version
|
||||
```
|
||||
|
||||
The consumer minimum is declared in [`go.mod`](../go.mod). Its patch-level
|
||||
minimum is required by the selected ConfigKit dependency. Selecting a newer
|
||||
release toolchain is a separate runbook update and leaves that minimum unchanged.
|
||||
|
||||
Configure ordinary Git credentials for `origin` and private-module access
|
||||
without putting secrets in the repository. Prepare the selected toolchain and
|
||||
dependencies before offline validation where necessary:
|
||||
|
||||
```sh
|
||||
GOWORK=off GOPRIVATE=gitea.maximumdirect.net go mod download
|
||||
@@ -39,9 +52,10 @@ Run one command from the repository with the actual selected version:
|
||||
./scripts/release.sh vMAJOR.MINOR.PATCH
|
||||
```
|
||||
|
||||
The command checks version availability on `origin`, branch and worktree state,
|
||||
and the matching note. It runs the complete offline default test suite once and
|
||||
a native CLI build/version check with `GOWORK=off`. The
|
||||
Both release scripts inherit the `GOTOOLCHAIN` selected during preparation.
|
||||
The publication command checks version availability on `origin`, branch and
|
||||
worktree state, and the matching note. It runs the complete offline default test
|
||||
suite once and a native CLI build/version check with `GOWORK=off`. The
|
||||
[gate rationale](policy/release.md#notarius-gate-selection) explains the selected
|
||||
checks. It then rechecks candidate and publication state, pushes the exact
|
||||
candidate to `main` without force, creates a lightweight tag, and pushes only
|
||||
|
||||
244
docs/roadmap/promptkit.md
Normal file
244
docs/roadmap/promptkit.md
Normal file
@@ -0,0 +1,244 @@
|
||||
# PromptKit v0.23.2 Adoption
|
||||
|
||||
## Purpose
|
||||
|
||||
Replace Notarius's PromptKit v0.12 integration with a native v0.23.2
|
||||
integration. Delegate configuration composition, resource inspection, request
|
||||
preparation, and model execution to PromptKit and ConfigKit. Keep application
|
||||
machinery where it implements a distinct Notarius decision, such as typed
|
||||
artifact handling, stage acceptance, cross-backend scheduling, or checkpoint
|
||||
reuse.
|
||||
|
||||
This is a hard cutover for both the Notarius application configuration and
|
||||
PromptKit resource definitions. Provide migration guidance and maintained
|
||||
examples; support only the new formats in the resulting application. This
|
||||
draft describes the target, assuming the recommendations under
|
||||
[Open Decisions](#open-decisions). It does not select a release date or
|
||||
implementation sequence.
|
||||
|
||||
## Reviewed Baseline
|
||||
|
||||
Notarius at `65b8e8f0a2d60041c18a29a3c33afc5c689468df` uses application
|
||||
configuration version 5, Go 1.27, new-path ConfigKit v1.2.0, and old-path
|
||||
PromptKit v0.12.0. The latter brings in a separate old-path ConfigKit v0.5.0
|
||||
and two external provider catalogs. Notarius and PromptKit currently share
|
||||
the YAML v4 release candidate.
|
||||
|
||||
The target is `go.maximumdirect.net/promptkit@v0.23.2`, published at
|
||||
`382ccfb14b9630fd8f48c8364dbe2b6159637d71`. The tagged [release
|
||||
notes][release], [migration guide][migration], [format reference][formats],
|
||||
and [Go consumer guide][consumer] are the primary upstream references.
|
||||
|
||||
The main baseline seams are:
|
||||
|
||||
- `internal/core/config` resolves `promptkit.profile_file` and
|
||||
`profile_dir` and interprets the optional `local_backend` object.
|
||||
`promptkit_profiles.go` builds the operator layers for execution and
|
||||
inspection; `asset_registry.go` supplies the embedded layer.
|
||||
- `assets/` and `examples/llm-profiles/` use keyed resource definitions.
|
||||
Completion requests and diagnostics carry a PromptKit prompt version.
|
||||
- `internal/framework/llm` does split preparation and execution to retain
|
||||
debug material; it projects a backend-completed profile report and walks
|
||||
effective YAML to discover versioned prompts for checkpoint identity.
|
||||
- The pipeline's producer-attempt loop coordinates domain and stage policy.
|
||||
The scheduled LLM client enforces the application-wide limit independently
|
||||
of any PromptKit backend policy.
|
||||
|
||||
## Target End State
|
||||
|
||||
### Dependency And Application Configuration Cutover
|
||||
|
||||
Use the vanity PromptKit import path and shared ConfigKit v1.2.1, stable
|
||||
go-yaml v4, and the resulting compatible validation dependencies. Remove the
|
||||
old-path imports and catalog dependencies once unused. Use tagged modules
|
||||
without a compatibility fork or permanent replacement. The consumer minimum
|
||||
remains Go 1.27. Update attribution for the selected and removed components
|
||||
under the [OSS policy](../policy/oss.md).
|
||||
|
||||
Introduce application configuration version 6 as the only supported application
|
||||
version. Version 6 retains unrelated pipeline structure, discovery, and
|
||||
resolution behavior. ConfigKit continues to own application loading and
|
||||
composition under [ADR-0020](../adr/0020-delegate-configuration-mechanics-to-configkit.md).
|
||||
Retire `profile_file`, `profile_dir`, and `local_backend` in favor of native
|
||||
PromptKit companion files. There is no separate `promptkit.config_file` selector.
|
||||
Preserve the ability to override transport retries through the PromptKit engine
|
||||
option; leave the upstream default in effect when the operator does not
|
||||
override it.
|
||||
|
||||
### Companion Files And Embedded Defaults
|
||||
|
||||
Adopt Audita's companion-file convention. Use the directory of the main
|
||||
Notarius configuration actually selected by ConfigKit, whether explicit or
|
||||
discovered. Load the optional `prompts.yml` and recursive `.yml` and `.yaml`
|
||||
files under `prompts/` beside it as one higher-priority operator underlay
|
||||
using native ConfigKit imports. Do not add a working-directory fallback
|
||||
for companion discovery. Configurations in the same directory share those
|
||||
companions. Missing companions are fine; loading errors follow upstream
|
||||
semantics rather than silently falling back to embedded assets. Native
|
||||
imports and source-relative resource paths govern any files reached from the
|
||||
companions. The two companion locations are strict peers; filenames do not
|
||||
set precedence between conflicting operator records.
|
||||
|
||||
Use one source assembly for runtime, preflight, and inspection: PromptKit's
|
||||
bundled OpenRouter underlay, Notarius's embedded underlay, then the operator
|
||||
underlay when present. Embed all Notarius default prompts and profiles under
|
||||
`collections.notarius` in the application underlay. Operator companion files
|
||||
can replace individual defaults by defining complete records with the same
|
||||
local IDs in that same collection. They can also define new resources and
|
||||
local backends using the native format.
|
||||
|
||||
The bundled OpenRouter inventory differs from the former external catalogs
|
||||
and supplies no backend-local concurrency limit. It does not implicitly
|
||||
select a model for any Notarius request. Migrate deployments that use removed
|
||||
profiles or backends to intentionally selected native records. The
|
||||
application-wide LLM ceiling remains independent of backend-local settings.
|
||||
|
||||
### Native Prompt And Profile Resources
|
||||
|
||||
Convert maintained prompts, profiles, examples, and fixtures to identified
|
||||
lists, with `config:` for source controls and native sibling inheritance
|
||||
(`extends: {id: parent}`). Under native default composition, a later same-ID
|
||||
record replaces the earlier record in full. Explain complete replacements,
|
||||
distinct inherited variants, and the new backend fallback semantics for
|
||||
profile nulls.
|
||||
The optional PromptKit source specification header is version 2; it is not the
|
||||
Notarius application version. Do not translate legacy resource files,
|
||||
synthesize aliases, or reconstruct old recursive same-ID overlays.
|
||||
|
||||
Select prompts by ID alone and remove the obsolete PromptKit version selector
|
||||
and metadata from Notarius's request, debug, and reconciliation identity
|
||||
plumbing. Review affected diagnostic fields and document deliberate changes.
|
||||
Keep the current local resource IDs and use their qualified forms in
|
||||
lookups and references, for example `notarius/dnd.spells` and
|
||||
`notarius/dnd-extraction`. Update prompt profile references, application
|
||||
profile selections, and inspection examples to use the qualified IDs.
|
||||
The `dnd-extraction` profile retains its current model and settings and
|
||||
continues to reference the unqualified bundled `openrouter` backend. A
|
||||
profile that extends it belongs in the same collection and uses its local
|
||||
sibling ID in `extends`. Retain message roles, bytes, order, cache boundaries,
|
||||
and intended output/repair behavior while changing the declaration format.
|
||||
Keep module-owned asset selection and embedded filesystem contexts intact.
|
||||
Verify maintained private response schemas work under upstream's
|
||||
self-contained-schema rule; this is not a redesign of durable D&D artifacts.
|
||||
|
||||
### Inspection And Reuse Identity
|
||||
|
||||
Use PromptKit's captured configuration reports for effective configuration
|
||||
and provenance. Use its resource inventory and inspection APIs for exact
|
||||
identities, presence-aware resource fields, and captured prompt hashes, rather
|
||||
than deriving an inventory from rendered configuration YAML. Change the
|
||||
`config profile` report to show the resolved profile and its selected backend,
|
||||
when present, as separate resources. Use upstream field semantics and retain
|
||||
the existing operator disclosure boundary, including redaction of
|
||||
credential-shaped provider parameters. Do not recreate backend default
|
||||
resolution to produce the former `effective_model_params` view.
|
||||
|
||||
Distinguish configuration and resource preflight from request readiness.
|
||||
Use `InspectRequirements` to explain recognized dependencies when helpful,
|
||||
not as proof that a request is executable. Use `Prepare` or
|
||||
`ValidateRequest` when concrete request inputs are available and their
|
||||
validation is actually needed. A routine config-only check should remain
|
||||
credential-free and offline, without synthetic requests.
|
||||
|
||||
Retain a private, versioned Notarius checkpoint identity based on the engine's
|
||||
captured state. Use native resource enumeration and prompt hashes, keeping
|
||||
the effective execution inputs needed to distinguish reuse safely. A diff of
|
||||
effective values alone cannot detect edits to a referenced message file.
|
||||
Version the fingerprint for the cutover so obsolete identity inputs cannot
|
||||
cause a false hit; do not reread mutable sources or expose the digest inputs.
|
||||
Keep run manifests based on the profiles actually used, not the inventory.
|
||||
|
||||
### Native Execution And Retained Pipeline Policy
|
||||
|
||||
Use ordinary `Engine.Run` for structured completion and use `OnPrepared`
|
||||
where opt-in debug capture needs the actual prepared request. The callback
|
||||
runs after admission, before generation, and is not delivered for failed
|
||||
preparation or admission. Keep the callback lightweight and use native
|
||||
observations for safe terminal execution facts, including attempts, repairs,
|
||||
and available token usage. Where a diagnostic needs totals over selected
|
||||
executions, use `ExecutionGroup` rather than inventing a second accumulator.
|
||||
Use `OperationTimeout` where a completion calls for that scope of budget;
|
||||
do not add a new default deadline or replace application deadlines that also
|
||||
cover work outside the PromptKit call.
|
||||
|
||||
PromptKit owns provider admission, transport retries, and structural repair.
|
||||
Keep the Notarius cross-backend scheduler because it serves a different
|
||||
budget from backend or pipeline worker limits. Keep the adapter's typed
|
||||
decode, actual-run provenance, and provider-neutral error and diagnostic
|
||||
mapping. PromptKit execution types stay within that integration boundary.
|
||||
In particular, preserve the distinction between invalid structured
|
||||
output, provider failure, capacity rejection, and cancellation. A configured
|
||||
repair override still selects a complete upstream output contract; it must not
|
||||
accidentally drop the selected schema or validation mode.
|
||||
|
||||
Notarius owns stage attempts, validation and fallback decisions, and
|
||||
publication of accepted artifacts. Keep its latest-response-only correction
|
||||
policy using native appended messages, as defined by
|
||||
[ADR-0014](../adr/0014-feedback-aware-validation-retries.md) and
|
||||
[ADR-0016](../adr/0016-feedback-aware-module-requested-retries.md).
|
||||
`Exchange` and `Coordinate` are for workflows that benefit from retained
|
||||
conversation history. Adopting them for stage retries would change what the
|
||||
model sees and which work consumes an attempt. They are not needed to execute
|
||||
the existing retry policy.
|
||||
|
||||
## Migration And Verification
|
||||
|
||||
Update the maintained minimal and complete D&D examples to distinguish
|
||||
embedded-only operation from companion configuration. Use separate example
|
||||
directories if necessary to demonstrate those deployments independently.
|
||||
Document how to move existing PromptKit sources into the companion layout
|
||||
or import them from it, declare a local backend, and replace or extend an
|
||||
embedded resource. Explain qualified IDs, retired catalog IDs, the changed
|
||||
profile report, and any new checkpoint cold misses. Update the canonical
|
||||
configuration, CLI, operations, and integration guides when behavior lands.
|
||||
Record the changed decisions about source precedence and inspection in a
|
||||
successor to
|
||||
[ADR-0019](../adr/0019-adopt-native-promptkit-configuration.md) rather than
|
||||
rewriting its accepted decision text. Release preparation is separate.
|
||||
|
||||
Verify the dependency graph and offline examples after the cutover. Use
|
||||
focused behavior tests for the new format and intentional rejection of the
|
||||
old one, main-relative companion discovery, and native relative imports.
|
||||
Verify that a companion can replace a qualified embedded prompt or profile
|
||||
without changing unrelated defaults, and that preflight, inspection, and
|
||||
runtime use the same effective definitions. Exercise preparation,
|
||||
generation, structural repair, and semantic retries through the maintained
|
||||
fake-provider integration. Check that real message or target changes affect
|
||||
checkpoint reuse, and that session routing, cache controls, diagnostics, and
|
||||
durable outputs keep their existing meaning. Live provider checks are useful
|
||||
for deployment confidence, not a requirement of the offline suite.
|
||||
|
||||
[release]: https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.23.2/docs/releases/v0.23.2.md
|
||||
[migration]: https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.23.2/docs/consumers/configuration-migration.md
|
||||
[formats]: https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.23.2/docs/formats.md
|
||||
[consumer]: https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.23.2/docs/consumers/pkg-promptkit.md
|
||||
|
||||
## Open Decisions
|
||||
|
||||
### How Far Offline Preflight Goes
|
||||
|
||||
**Recommendation:** Check that the captured configuration is valid and that
|
||||
selected resource IDs resolve. Make clear that a valid profile can be partial
|
||||
and that concrete request readiness is checked during preparation. This
|
||||
preserves a useful offline preflight without duplicating target resolution or
|
||||
requiring credentials or source data.
|
||||
|
||||
**Alternative:** Preserve offline checking of completed targets by pursuing a
|
||||
dedicated upstream API that resolves targets without artifacts or credentials.
|
||||
That would catch more missing endpoint/model settings before a run and could
|
||||
serve other consumers, but it would add an upstream dependency to this work or
|
||||
require a subsequent release. Reimplementing completion rules in Notarius
|
||||
would risk a discrepancy with actual execution.
|
||||
|
||||
### Operator Diagnostics In This Upgrade
|
||||
|
||||
**Recommendation:** Migrate existing inspection commands and use native
|
||||
inventory and observation APIs where they replace local work. Evaluate new
|
||||
commands for provider smoke checks or effective-config diffs separately,
|
||||
after identifying the operator workflows and output contracts they need.
|
||||
|
||||
**Alternative:** Include an explicitly invoked smoke command in the cutover.
|
||||
This would help operators verify credentials and structured inference after
|
||||
migrating a deployment, and it can delegate to `Engine.SmokeTest`. It adds a
|
||||
network-capable CLI surface, with cost and output handling to define. Keep it
|
||||
separate from ordinary offline validation in either case.
|
||||
@@ -1,7 +1,8 @@
|
||||
version: 5
|
||||
configkit:
|
||||
config:
|
||||
imports:
|
||||
- path: pipelines/dnd-session.yml
|
||||
required: true
|
||||
promptkit:
|
||||
profile_file: ./llm-profiles/dnd-extraction.yml
|
||||
transient_retries: 2
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
id: dnd-extraction
|
||||
backend: openrouter
|
||||
model: openai/gpt-5.6-luna
|
||||
timeout_seconds: 240
|
||||
service_tier: flex
|
||||
profiles:
|
||||
dnd-extraction:
|
||||
backend: openrouter
|
||||
model: openai/gpt-5.6-luna
|
||||
timeout_seconds: 240
|
||||
service_tier: flex
|
||||
dnd-extraction-development:
|
||||
extends: dnd-extraction
|
||||
temperature: 0.2
|
||||
|
||||
37
examples/pipelines/dnd-session.describe-session.yml
Normal file
37
examples/pipelines/dnd-session.describe-session.yml
Normal file
@@ -0,0 +1,37 @@
|
||||
# Establish session-wide reference artifacts before their consumers.
|
||||
id: describe-session
|
||||
artifacts:
|
||||
item-registry:
|
||||
extract:
|
||||
module: dnd/item-registry
|
||||
retries: 2
|
||||
merge:
|
||||
module: appendorder
|
||||
normalize:
|
||||
module: dnd/item-registry
|
||||
npc-registry:
|
||||
extract:
|
||||
module: dnd/npc-registry
|
||||
retries: 2
|
||||
merge:
|
||||
module: appendorder
|
||||
normalize:
|
||||
module: dnd/npc-registry
|
||||
retries: 2
|
||||
location-registry:
|
||||
extract:
|
||||
module: dnd/location-registry
|
||||
retries: 2
|
||||
merge:
|
||||
module: appendorder
|
||||
normalize:
|
||||
module: dnd/location-registry
|
||||
retries: 2
|
||||
scene-descriptions:
|
||||
extract:
|
||||
module: dnd/scene-descriptions
|
||||
retries: 2
|
||||
merge:
|
||||
module: appendorder
|
||||
normalize:
|
||||
module: dnd/scene-descriptions
|
||||
@@ -1,3 +1,9 @@
|
||||
config:
|
||||
imports:
|
||||
- path: dnd-session.describe-session.yml
|
||||
required: true
|
||||
destination: [pipelines, dnd-session, steps]
|
||||
placement: sequence-item
|
||||
pipelines:
|
||||
dnd-session:
|
||||
llm_profile: dnd-extraction
|
||||
@@ -30,43 +36,6 @@ pipelines:
|
||||
- npc-occurrences
|
||||
- enemy-events
|
||||
steps:
|
||||
# Establish session-wide reference artifacts before their consumers.
|
||||
- id: describe-session
|
||||
artifacts:
|
||||
item-registry:
|
||||
extract:
|
||||
module: dnd/item-registry
|
||||
retries: 2
|
||||
merge:
|
||||
module: appendorder
|
||||
normalize:
|
||||
module: dnd/item-registry
|
||||
npc-registry:
|
||||
extract:
|
||||
module: dnd/npc-registry
|
||||
retries: 2
|
||||
merge:
|
||||
module: appendorder
|
||||
normalize:
|
||||
module: dnd/npc-registry
|
||||
retries: 2
|
||||
location-registry:
|
||||
extract:
|
||||
module: dnd/location-registry
|
||||
retries: 2
|
||||
merge:
|
||||
module: appendorder
|
||||
normalize:
|
||||
module: dnd/location-registry
|
||||
retries: 2
|
||||
scene-descriptions:
|
||||
extract:
|
||||
module: dnd/scene-descriptions
|
||||
retries: 2
|
||||
merge:
|
||||
module: appendorder
|
||||
normalize:
|
||||
module: dnd/scene-descriptions
|
||||
- id: extract-events
|
||||
# Accepted registry artifacts and scene-description eligibility artifacts
|
||||
# are supplied in memory to their compatible consumers in this step.
|
||||
@@ -162,3 +131,9 @@ pipelines:
|
||||
module: appendorder
|
||||
normalize:
|
||||
module: dnd/enemy-events
|
||||
|
||||
# Development variant: disable PromptKit structural repairs while retaining
|
||||
# the inherited steps and configured producer retry budgets.
|
||||
dnd-session-development:
|
||||
extends: dnd-session
|
||||
structured_output_repair_attempts: 0
|
||||
|
||||
17
go.mod
17
go.mod
@@ -1,18 +1,17 @@
|
||||
module gitea.maximumdirect.net/eric/notarius
|
||||
|
||||
go 1.25.5
|
||||
go 1.27.0
|
||||
|
||||
require (
|
||||
gitea.maximumdirect.net/eric/ConfigKit v0.4.2
|
||||
gitea.maximumdirect.net/eric/promptkit v0.11.2
|
||||
gitea.maximumdirect.net/eric/ConfigKit v0.5.0
|
||||
gitea.maximumdirect.net/eric/promptkit v0.12.0
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
go.maximumdirect.net/configkit v1.2.0
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.6
|
||||
golang.org/x/text v0.40.0
|
||||
)
|
||||
|
||||
require golang.org/x/text v0.40.0
|
||||
|
||||
require (
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2 v2.20260926.0 // indirect
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2 v2.20260926.0 // indirect
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0 // indirect
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0 // indirect
|
||||
)
|
||||
|
||||
20
go.sum
20
go.sum
@@ -1,20 +1,20 @@
|
||||
gitea.maximumdirect.net/eric/ConfigKit v0.4.2 h1:ZMxmv8pOnLtAx2hFTf8Z+pHKUbR89xErkBhwjr4yK24=
|
||||
gitea.maximumdirect.net/eric/ConfigKit v0.4.2/go.mod h1:tYWpsrbLyu05r8+1MapWp5fN6ZrX9I8hU2xEPedYmYk=
|
||||
gitea.maximumdirect.net/eric/promptkit v0.11.2 h1:2RqI+KtflvSh7vLQaTAQCmMaFQscEA7EPwPazyvXRoY=
|
||||
gitea.maximumdirect.net/eric/promptkit v0.11.2/go.mod h1:onm/FBPUKQ8/NSPwTEk26Ut+dVX2iXcq8vEseB+g/0E=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2 v2.20260926.0 h1:wdab+R+lgMZxzYwhbAuCrW7zdSQ1cW9ovBR6UzLmZR4=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2 v2.20260926.0/go.mod h1:a3VBbh9oaGXD6ruxhtwHj+AWW6DY/SSyby1QtMmYSUo=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2 v2.20260926.0 h1:baCCIB6Ee5t0S4Zc++NTBIPcJEv5UPS6DlSrpZfykwg=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2 v2.20260926.0/go.mod h1:mtgD1E0SGm0e/I8Ffhz+AYX7GN7RKUWJXehNFIKHs4Y=
|
||||
gitea.maximumdirect.net/eric/ConfigKit v0.5.0 h1:2KwG10+kApWM3avOoYA+p768xcbHUp7tQ2rqizBNpyg=
|
||||
gitea.maximumdirect.net/eric/ConfigKit v0.5.0/go.mod h1:tYWpsrbLyu05r8+1MapWp5fN6ZrX9I8hU2xEPedYmYk=
|
||||
gitea.maximumdirect.net/eric/promptkit v0.12.0 h1:oIFWEg6GPxtmX3Plqv0euP+FnK1LCk/t4xDy6oKCpP4=
|
||||
gitea.maximumdirect.net/eric/promptkit v0.12.0/go.mod h1:pFOOy5v1zH17stEi9rmHYmfbuLjnfVMcUhfmk+4VtJY=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0 h1:lzfCDbzJQRGzIlRPRprnAV9hVVL5zUQy80VMQ0kbYa0=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0/go.mod h1:HjS0f+3q0IdHCTEz+VVAEgIwIWJFAZUjuKdAtvtqKdg=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0 h1:h12ZTRldAmpLAomj0oQRGteFxHLN1LPE6XDl9zJNbm8=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0/go.mod h1:OEFBeKdjTsLzqJwLmXI2O5gG3r64FktcVJVb3mBrRfg=
|
||||
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
||||
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
||||
go.maximumdirect.net/configkit v1.2.0 h1:c4tDJ6Xeh/Jg90UZplLxsnf500lvrZBVj0E3i1ysir8=
|
||||
go.maximumdirect.net/configkit v1.2.0/go.mod h1:s1zLJ9GGB/1Q4a+aOHMLISonJxWFgNWpA6SpPlOafeA=
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
|
||||
golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
|
||||
golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
@@ -104,27 +104,6 @@ func TestConfigDiscoveryPrefersExplicitPathThenEnvironment(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigDiscoveryUsesCompiledDefaultOnlyWhenAvailable(t *testing.T) {
|
||||
info, statErr := os.Stat(defaultConfigPath)
|
||||
if statErr != nil && !os.IsNotExist(statErr) {
|
||||
t.Fatalf("stat compiled default config: %v", statErr)
|
||||
}
|
||||
if statErr == nil && !info.Mode().IsRegular() {
|
||||
t.Skipf("compiled default config has unexpected host state: %s", info.Mode())
|
||||
}
|
||||
|
||||
path, err := discoverConfigPath("", commandContractOptions(t))
|
||||
if statErr == nil {
|
||||
if err != nil || path != defaultConfigPath {
|
||||
t.Fatalf("discoverConfigPath() = %q, %v; want compiled default", path, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err == nil || !strings.Contains(err.Error(), "config file not found") {
|
||||
t.Fatalf("discoverConfigPath() error = %v, want documented not-found context", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigLoadingFailuresReturnOneWithPathContext(t *testing.T) {
|
||||
missing := filepath.Join(t.TempDir(), "missing.yml")
|
||||
var stdout, stderr bytes.Buffer
|
||||
|
||||
@@ -3,13 +3,15 @@ package cli
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
frameworkllm "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
configkit "go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
func runConfigEffective(args []string, stdout, stderr io.Writer, opts Options) int {
|
||||
@@ -17,6 +19,7 @@ func runConfigEffective(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
fs.SetOutput(io.Discard)
|
||||
configPath := fs.String("config", "", "config file path")
|
||||
overlay := fs.String("config-overlay", "", "configuration overlay")
|
||||
scope := fs.String("scope", "notarius", "configuration scope: notarius or promptkit")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||
return 2
|
||||
@@ -25,12 +28,28 @@ func runConfigEffective(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
fmt.Fprintf(stderr, "notarius: unexpected argument %q\n", fs.Arg(0))
|
||||
return 2
|
||||
}
|
||||
if *scope != "notarius" && *scope != "promptkit" {
|
||||
fmt.Fprintf(stderr, "notarius: invalid configuration scope %q (want notarius or promptkit)\n", *scope)
|
||||
return 2
|
||||
}
|
||||
loaded, _, err := loadConfigKit(*configPath, *overlay, nil, opts)
|
||||
if err != nil {
|
||||
writeConfigLoadError(stderr, err)
|
||||
return 1
|
||||
}
|
||||
if err := loaded.WriteEffective(stdout); err != nil {
|
||||
if *scope == "notarius" {
|
||||
if err := loaded.WriteEffective(stdout); err != nil {
|
||||
fmt.Fprintf(stderr, "notarius: write effective configuration: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
inspector, err := newPromptKitInspection(loaded.Config, opts)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "notarius: inspect PromptKit configuration: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
if err := inspector.WriteEffectiveConfiguration(stdout); err != nil {
|
||||
fmt.Fprintf(stderr, "notarius: write effective configuration: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
@@ -42,6 +61,7 @@ func runConfigExplain(args []string, stdout, stderr io.Writer, opts Options) int
|
||||
fs.SetOutput(io.Discard)
|
||||
configPath := fs.String("config", "", "config file path")
|
||||
overlay := fs.String("config-overlay", "", "configuration overlay")
|
||||
scope := fs.String("scope", "notarius", "configuration scope: notarius or promptkit")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||
return 2
|
||||
@@ -50,12 +70,42 @@ func runConfigExplain(args []string, stdout, stderr io.Writer, opts Options) int
|
||||
fmt.Fprintln(stderr, "notarius: config explain requires exactly one JSON Pointer")
|
||||
return 2
|
||||
}
|
||||
if *scope != "notarius" && *scope != "promptkit" {
|
||||
fmt.Fprintf(stderr, "notarius: invalid configuration scope %q (want notarius or promptkit)\n", *scope)
|
||||
return 2
|
||||
}
|
||||
pointer := fs.Arg(0)
|
||||
parsed, err := configkit.ParsePath(pointer)
|
||||
if err != nil || parsed.String() != pointer {
|
||||
fmt.Fprintf(stderr, "notarius: %q is not a canonical JSON Pointer\n", pointer)
|
||||
return 2
|
||||
}
|
||||
if *scope == "promptkit" {
|
||||
loaded, _, err := loadConfigKit(*configPath, *overlay, nil, opts)
|
||||
if err != nil {
|
||||
writeConfigLoadError(stderr, err)
|
||||
return 1
|
||||
}
|
||||
inspector, err := newPromptKitInspection(loaded.Config, opts)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "notarius: inspect PromptKit configuration: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
found, err := inspector.WriteConfigurationExplanation(stdout, pointer)
|
||||
if err != nil {
|
||||
if errors.Is(err, frameworkllm.ErrInvalidPromptKitConfigurationPointer) {
|
||||
fmt.Fprintf(stderr, "notarius: %q is not a canonical JSON Pointer\n", pointer)
|
||||
return 2
|
||||
}
|
||||
fmt.Fprintf(stderr, "notarius: write PromptKit configuration explanation: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
if !found {
|
||||
fmt.Fprintf(stderr, "notarius: PromptKit configuration pointer %q is unknown\n", pointer)
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
loaded, _, err := loadConfigKit(*configPath, *overlay, nil, opts)
|
||||
if err != nil {
|
||||
writeConfigLoadError(stderr, err)
|
||||
@@ -68,6 +118,58 @@ func runConfigExplain(args []string, stdout, stderr io.Writer, opts Options) int
|
||||
return 0
|
||||
}
|
||||
|
||||
func runConfigProfile(args []string, stdout, stderr io.Writer, opts Options) int {
|
||||
fs := flag.NewFlagSet("config profile", flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
configPath := fs.String("config", "", "config file path")
|
||||
overlay := fs.String("config-overlay", "", "configuration overlay")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||
return 2
|
||||
}
|
||||
if fs.NArg() != 1 || strings.TrimSpace(fs.Arg(0)) == "" {
|
||||
fmt.Fprintln(stderr, "notarius: config profile requires exactly one nonblank profile ID")
|
||||
return 2
|
||||
}
|
||||
profileID := fs.Arg(0)
|
||||
loaded, _, err := loadConfigKit(*configPath, *overlay, nil, opts)
|
||||
if err != nil {
|
||||
writeConfigLoadError(stderr, err)
|
||||
return 1
|
||||
}
|
||||
inspector, err := newPromptKitInspection(loaded.Config, opts)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "notarius: inspect PromptKit configuration: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
profile, err := inspector.InspectProfile(context.Background(), profileID)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
result := struct {
|
||||
ProfileID string `json:"profile_id"`
|
||||
APIKeyRequired bool `json:"api_key_required"`
|
||||
EffectiveModelParams frameworkllm.PromptKitEffectiveModelParams `json:"effective_model_params"`
|
||||
}{
|
||||
ProfileID: profile.ProfileID,
|
||||
APIKeyRequired: profile.CredentialRequired,
|
||||
EffectiveModelParams: profile.EffectiveModelParams,
|
||||
}
|
||||
if err := json.NewEncoder(stdout).Encode(result); err != nil {
|
||||
fmt.Fprintf(stderr, "notarius: write PromptKit profile inspection: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func newPromptKitInspection(cfg config.Config, opts Options) (*frameworkllm.PromptKitProfileInspector, error) {
|
||||
return frameworkllm.NewPromptKitProfileInspector(frameworkllm.PromptKitProfileInspectorConfig{
|
||||
Source: promptKitProfileSourceConfig(cfg),
|
||||
Assets: opts.promptKitAssets,
|
||||
})
|
||||
}
|
||||
|
||||
func runConfigValidate(args []string, stdout, stderr io.Writer, opts Options) int {
|
||||
fs := flag.NewFlagSet("config validate", flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
|
||||
@@ -3,6 +3,7 @@ package cli
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"path/filepath"
|
||||
@@ -14,13 +15,30 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
func TestExplainRejectsMalformedPointerBeforeLoadingEitherScope(t *testing.T) {
|
||||
for _, scope := range []string{"notarius", "promptkit"} {
|
||||
t.Run(scope, func(t *testing.T) {
|
||||
var stdout, stderr strings.Builder
|
||||
options := commandContractOptions(t)
|
||||
options.LookupEnv = func(string) (string, bool) {
|
||||
t.Fatal("configuration discovery ran for invalid syntax")
|
||||
return "", false
|
||||
}
|
||||
code := RunWithOptions([]string{"config", "explain", "--scope", scope, "--config", filepath.Join(t.TempDir(), "missing.yml"), "/profiles/~2"}, &stdout, &stderr, options)
|
||||
if code != 2 || stdout.Len() != 0 || !strings.Contains(stderr.String(), "canonical JSON Pointer") {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigKitReadOnlyCommandsLoadImportsAndOverlays(t *testing.T) {
|
||||
t.Parallel()
|
||||
root := t.TempDir()
|
||||
path := filepath.Join(root, "notarius.yml")
|
||||
writeConfigKitFile(t, filepath.Join(root, "fragment.yml"), "pipelines:\n imported:\n input: {module: input}\n")
|
||||
writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "pipelines:\n overlaid:\n input: {module: input}\n")
|
||||
writeConfigKitFile(t, path, "version: 5\nconfigkit:\n imports:\n - path: fragment.yml\npipelines:\n base:\n input: {module: input}\n")
|
||||
writeConfigKitFile(t, path, "version: 5\nconfig:\n imports:\n - path: fragment.yml\n required: true\npipelines:\n base:\n input: {module: input}\n")
|
||||
var stdout, stderr bytes.Buffer
|
||||
if code := RunWithOptions([]string{"pipelines", "list", "--config", path, "--config-overlay", "development"}, &stdout, &stderr, commandContractOptions(t)); code != 0 || stdout.String() != "base\nimported\noverlaid\n" {
|
||||
t.Fatalf("list = %d, %q, %q", code, stdout.String(), stderr.String())
|
||||
@@ -40,6 +58,130 @@ func TestConfigKitReadOnlyCommandsLoadImportsAndOverlays(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPipelineInheritanceCommandsSelectAndInspectDerivedPipelines(t *testing.T) {
|
||||
t.Parallel()
|
||||
path := writeCommandConfigContent(t, `version: 5
|
||||
pipelines:
|
||||
" parent ":
|
||||
input:
|
||||
module: seriatim
|
||||
artifacts:
|
||||
spells:
|
||||
extract:
|
||||
module: dnd/spells
|
||||
structured_output_repair_attempts: 2
|
||||
child:
|
||||
extends: " parent "
|
||||
structured_output_repair_attempts: 0
|
||||
`)
|
||||
options := commandContractOptions(t)
|
||||
|
||||
var stdout, stderr bytes.Buffer
|
||||
if code := RunWithOptions([]string{"pipelines", "list", "--config", path}, &stdout, &stderr, options); code != 0 || stdout.String() != "child\nparent\n" || stderr.Len() != 0 {
|
||||
t.Fatalf("list = %d, %q, %q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
if code := RunWithOptions([]string{"config", "validate", "--config", path, "--pipeline", "child"}, &stdout, &stderr, options); code != 0 || !strings.Contains(stdout.String(), `is valid for pipeline "child"`) || stderr.Len() != 0 {
|
||||
t.Fatalf("validate child = %d, %q, %q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
if code := RunWithOptions([]string{"config", "effective", "--config", path}, &stdout, &stderr, options); code != 0 || !strings.Contains(stdout.String(), "module: seriatim") || !strings.Contains(stdout.String(), "structured_output_repair_attempts: 0") || strings.Contains(stdout.String(), "extends:") || stderr.Len() != 0 {
|
||||
t.Fatalf("effective = %d, %q, %q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
if code := RunWithOptions([]string{"config", "explain", "--config", path, "/pipelines/child/input/module"}, &stdout, &stderr, options); code != 0 || !strings.Contains(stdout.String(), `path: "/pipelines/child/input/module"`) || !strings.Contains(stdout.String(), "parent") || stderr.Len() != 0 {
|
||||
t.Fatalf("explain inherited input = %d, %q, %q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestPipelineInheritanceLoadFailuresUseRuntimeDiagnostics(t *testing.T) {
|
||||
t.Parallel()
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
content string
|
||||
args []string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "missing parent",
|
||||
content: `version: 5
|
||||
pipelines:
|
||||
child:
|
||||
extends: absent
|
||||
`,
|
||||
args: []string{"config", "validate", "--pipeline", "child"},
|
||||
want: "inheritance.missing",
|
||||
},
|
||||
{
|
||||
name: "cycle",
|
||||
content: `version: 5
|
||||
pipelines:
|
||||
first:
|
||||
extends: second
|
||||
second:
|
||||
extends: first
|
||||
`,
|
||||
args: []string{"config", "validate", "--pipeline", "first"},
|
||||
want: "cycle",
|
||||
},
|
||||
{
|
||||
name: "invalid directive type",
|
||||
content: `version: 5
|
||||
pipelines:
|
||||
child:
|
||||
extends: 7
|
||||
`,
|
||||
args: []string{"config", "validate", "--pipeline", "child"},
|
||||
want: "extends",
|
||||
},
|
||||
{
|
||||
name: "unselected missing parent",
|
||||
content: `version: 5
|
||||
pipelines:
|
||||
parent:
|
||||
input:
|
||||
module: seriatim
|
||||
child:
|
||||
extends: parent
|
||||
unrelated:
|
||||
extends: absent
|
||||
`,
|
||||
args: []string{"config", "validate", "--pipeline", "child"},
|
||||
want: "inheritance.missing",
|
||||
},
|
||||
{
|
||||
name: "exact sibling mismatch",
|
||||
content: `version: 5
|
||||
pipelines:
|
||||
parent:
|
||||
input:
|
||||
module: seriatim
|
||||
child:
|
||||
extends: "parent "
|
||||
`,
|
||||
args: []string{"pipelines", "list"},
|
||||
want: "inheritance.missing",
|
||||
},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
path := writeCommandConfigContent(t, test.content)
|
||||
args := append([]string{}, test.args...)
|
||||
args = append(args, "--config", path)
|
||||
var stdout, stderr bytes.Buffer
|
||||
if code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t)); code != 1 || stdout.Len() != 0 || stderr.Len() == 0 || !strings.Contains(stderr.String(), test.want) {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetiredConfigProfileFlagUsesNormalUnknownFlagBehavior(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -152,6 +294,229 @@ func TestConfigInspectionCommandsReportWriterFailures(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptKitScopedInspectionCommandsUseSelectedSources(t *testing.T) {
|
||||
t.Parallel()
|
||||
root := t.TempDir()
|
||||
configPath := filepath.Join(root, "notarius.yml")
|
||||
writeConfigKitFile(t, configPath, "version: 5\npromptkit:\n profile_file: profiles/base.yml\n")
|
||||
writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "promptkit:\n profile_file: "+filepath.Join(root, "profiles", "imported.yml")+"\n")
|
||||
writeConfigKitFile(t, filepath.Join(root, "profiles", "base.yml"), `profiles:
|
||||
base:
|
||||
endpoint: https://example.invalid/v1
|
||||
model: base-model
|
||||
extra_params:
|
||||
Authorization: base-auth-secret
|
||||
ordinary: visible-value
|
||||
`)
|
||||
writeConfigKitFile(t, filepath.Join(root, "profiles", "imported.yml"), `configkit:
|
||||
imports:
|
||||
- path: base.yml
|
||||
profiles:
|
||||
"escaped/id":
|
||||
model: imported-model
|
||||
dnd-extraction:
|
||||
model: imported-override
|
||||
`)
|
||||
assets, err := productionPromptAssets()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
factoryCalls := 0
|
||||
opts := commandContractOptions(t)
|
||||
opts.promptKitAssets = assets
|
||||
opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
||||
factoryCalls++
|
||||
return nil, nil, errors.New("inspection must not construct an LLM client")
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
if code := RunWithOptions([]string{"config", "effective", "--config", configPath}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "pipelines:") || strings.Contains(stdout.String(), "profiles:") || stderr.Len() != 0 {
|
||||
t.Fatalf("default effective report = %d, %q, %q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
defaultReport := stdout.String()
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
if code := RunWithOptions([]string{"config", "effective", "--scope", "notarius", "--config", configPath}, &stdout, &stderr, opts); code != 0 || stdout.String() != defaultReport || stderr.Len() != 0 {
|
||||
t.Fatalf("explicit Notarius effective report = %d, %q, %q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
if code := RunWithOptions([]string{"config", "effective", "--scope", "promptkit", "--config", configPath, "--config-overlay", "development"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "imported-model") || strings.Contains(stdout.String(), "base-auth-secret") || !strings.Contains(stdout.String(), "visible-value") || stderr.Len() != 0 {
|
||||
t.Fatalf("PromptKit effective report = %d, %q, %q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
if code := RunWithOptions([]string{"config", "explain", "--scope", "promptkit", "--config", configPath, "--config-overlay", "development", "/profiles/escaped~1id/model"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "imported-model") || !strings.Contains(stdout.String(), "imported.yml") || stderr.Len() != 0 {
|
||||
t.Fatalf("PromptKit imported explanation = %d, %q, %q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
if code := RunWithOptions([]string{"config", "explain", "--scope", "promptkit", "--config", configPath, "--config-overlay", "development", "/profiles/base/model"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "base-model") || !strings.Contains(stdout.String(), "base.yml") || stderr.Len() != 0 {
|
||||
t.Fatalf("PromptKit imported base explanation = %d, %q, %q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
if code := RunWithOptions([]string{"config", "explain", "--scope", "promptkit", "--config", configPath, "--config-overlay", "development", "/profiles/dnd-extraction/model"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "imported-override") || !strings.Contains(stdout.String(), "dnd-extraction.yaml") || stderr.Len() != 0 {
|
||||
t.Fatalf("PromptKit historical fallback explanation = %d, %q, %q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
if factoryCalls != 0 {
|
||||
t.Fatalf("inspection invoked LLM client factory %d times", factoryCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptKitScopedInspectionClassifiesSelectorsAndWriterErrors(t *testing.T) {
|
||||
t.Parallel()
|
||||
path := writeCommandConfig(t, "session", "other")
|
||||
opts := commandContractOptions(t)
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
args []string
|
||||
want int
|
||||
}{
|
||||
{name: "blank scope", args: []string{"config", "effective", "--scope="}, want: 2},
|
||||
{name: "unknown scope", args: []string{"config", "effective", "--scope", "unknown"}, want: 2},
|
||||
{name: "malformed pointer", args: []string{"config", "explain", "--scope", "promptkit", "--config", path, "/profiles/~2"}, want: 2},
|
||||
{name: "unknown pointer", args: []string{"config", "explain", "--scope", "promptkit", "--config", path, "/not-configured"}, want: 1},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
var stdout, stderr bytes.Buffer
|
||||
if code := RunWithOptions(test.args, &stdout, &stderr, opts); code != test.want || stderr.Len() == 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q, want exit %d and diagnostic", code, stdout.String(), stderr.String(), test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
var stderr bytes.Buffer
|
||||
if code := RunWithOptions([]string{"config", "effective", "--scope", "promptkit", "--config", path}, failingInspectionWriter{}, &stderr, opts); code != 1 || !strings.Contains(stderr.String(), "write effective configuration") {
|
||||
t.Fatalf("PromptKit writer failure = %d, %q", code, stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigProfileReportsResolvedRedactedSettingsWithoutCredentialLookup(t *testing.T) {
|
||||
t.Parallel()
|
||||
root := t.TempDir()
|
||||
profilePath := filepath.Join(root, "profiles.yml")
|
||||
writeConfigKitFile(t, profilePath, `backends:
|
||||
managed:
|
||||
endpoint: https://provider.example.invalid/v1
|
||||
api_key_env: NOT_SET_SYNTHETIC_PROFILE_KEY
|
||||
profiles:
|
||||
inherited:
|
||||
backend: managed
|
||||
model: inherited-model
|
||||
temperature: 0.4
|
||||
api_key_env: NOT_SET_SYNTHETIC_PROFILE_KEY
|
||||
api_key_required: true
|
||||
extra_params:
|
||||
Authorization: private-authorization
|
||||
ordinary: visible-value
|
||||
selected:
|
||||
extends: inherited
|
||||
temperature: 0
|
||||
extra_params: null
|
||||
endpoint-only:
|
||||
endpoint: https://endpoint.example.invalid/v1
|
||||
model: endpoint-model
|
||||
api_key_required: false
|
||||
" literal profile ":
|
||||
endpoint: https://literal.example.invalid/v1
|
||||
model: literal-model
|
||||
`)
|
||||
configPath := filepath.Join(root, "notarius.yml")
|
||||
writeConfigKitFile(t, configPath, "version: 5\npromptkit:\n profile_file: "+profilePath+"\n")
|
||||
writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "promptkit:\n profile_file: "+profilePath+"\n")
|
||||
assets, err := productionPromptAssets()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lookupCalls := 0
|
||||
factoryCalls := 0
|
||||
opts := commandContractOptions(t)
|
||||
opts.promptKitAssets = assets
|
||||
opts.LookupEnv = func(string) (string, bool) {
|
||||
lookupCalls++
|
||||
return "", false
|
||||
}
|
||||
opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
||||
factoryCalls++
|
||||
return nil, nil, errors.New("profile inspection must not construct an LLM client")
|
||||
}
|
||||
|
||||
type profileReport struct {
|
||||
ProfileID string `json:"profile_id"`
|
||||
APIKeyRequired bool `json:"api_key_required"`
|
||||
Target struct {
|
||||
BackendID string `json:"backend_id"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
Model string `json:"model"`
|
||||
Temperature float64 `json:"temperature"`
|
||||
APIKeyEnv string `json:"api_key_env"`
|
||||
ExtraParams map[string]any `json:"extra_params"`
|
||||
} `json:"effective_model_params"`
|
||||
}
|
||||
run := func(profileID string) (int, string, string, profileReport) {
|
||||
t.Helper()
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"config", "profile", "--config", configPath, "--config-overlay", "development", profileID}, &stdout, &stderr, opts)
|
||||
var report profileReport
|
||||
if code == 0 {
|
||||
if bytes.Count(stdout.Bytes(), []byte("\n")) != 1 {
|
||||
t.Fatalf("profile JSON output = %q, want one object and newline", stdout.String())
|
||||
}
|
||||
if err := json.Unmarshal(stdout.Bytes(), &report); err != nil {
|
||||
t.Fatalf("decode profile JSON %q: %v", stdout.String(), err)
|
||||
}
|
||||
}
|
||||
return code, stdout.String(), stderr.String(), report
|
||||
}
|
||||
code, output, diagnostic, selected := run("selected")
|
||||
if code != 0 || diagnostic != "" || selected.ProfileID != "selected" || !selected.APIKeyRequired || selected.Target.BackendID != "managed" || selected.Target.Endpoint != "https://provider.example.invalid/v1" || selected.Target.Model != "inherited-model" || selected.Target.Temperature != 0 || selected.Target.APIKeyEnv != "NOT_SET_SYNTHETIC_PROFILE_KEY" || selected.Target.ExtraParams != nil {
|
||||
t.Fatalf("resolved inherited profile = %d %q %q %#v", code, output, diagnostic, selected)
|
||||
}
|
||||
code, output, diagnostic, inherited := run("inherited")
|
||||
if code != 0 || diagnostic != "" || inherited.Target.ExtraParams["Authorization"] != "[REDACTED]" || inherited.Target.ExtraParams["ordinary"] != "visible-value" || strings.Contains(output, "private-authorization") {
|
||||
t.Fatalf("redacted profile = %d %q %q %#v", code, output, diagnostic, inherited)
|
||||
}
|
||||
code, output, diagnostic, endpointOnly := run("endpoint-only")
|
||||
if code != 0 || diagnostic != "" || endpointOnly.ProfileID != "endpoint-only" || endpointOnly.APIKeyRequired || endpointOnly.Target.BackendID != "" || endpointOnly.Target.Endpoint != "https://endpoint.example.invalid/v1" {
|
||||
t.Fatalf("endpoint-only profile = %d %q %q %#v", code, output, diagnostic, endpointOnly)
|
||||
}
|
||||
code, output, diagnostic, literal := run(" literal profile ")
|
||||
if code != 0 || diagnostic != "" || literal.ProfileID != " literal profile " || literal.Target.Model != "literal-model" {
|
||||
t.Fatalf("literal profile ID = %d %q %q %#v", code, output, diagnostic, literal)
|
||||
}
|
||||
if lookupCalls != 0 || factoryCalls != 0 {
|
||||
t.Fatalf("profile inspection called credential lookup %d times and LLM factory %d times", lookupCalls, factoryCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigProfileClassifiesSyntaxAndRuntimeFailures(t *testing.T) {
|
||||
t.Parallel()
|
||||
configPath := writeCommandConfig(t, "session", "other")
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
args []string
|
||||
want int
|
||||
}{
|
||||
{name: "missing ID", args: []string{"config", "profile", "--config", configPath}, want: 2},
|
||||
{name: "blank ID", args: []string{"config", "profile", "--config", configPath, " "}, want: 2},
|
||||
{name: "multiple IDs", args: []string{"config", "profile", "--config", configPath, "one", "two"}, want: 2},
|
||||
{name: "unknown profile", args: []string{"config", "profile", "--config", configPath, "unknown-profile"}, want: 1},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
var stdout, stderr bytes.Buffer
|
||||
if code := RunWithOptions(test.args, &stdout, &stderr, commandContractOptions(t)); code != test.want || stderr.Len() == 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q, want exit %d and diagnostic", code, stdout.String(), stderr.String(), test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
profilePath := filepath.Join(t.TempDir(), "profiles.yml")
|
||||
writeConfigKitFile(t, profilePath, "profiles:\n writer:\n endpoint: https://example.invalid/v1\n model: writer-model\n")
|
||||
configPath = writeCommandConfigContent(t, "version: 5\npromptkit:\n profile_file: "+profilePath+"\n")
|
||||
var stderr bytes.Buffer
|
||||
if code := RunWithOptions([]string{"config", "profile", "--config", configPath, "writer"}, failingInspectionWriter{}, &stderr, commandContractOptions(t)); code != 1 || !strings.Contains(stderr.String(), "write PromptKit profile inspection") {
|
||||
t.Fatalf("profile writer failure = %d, %q", code, stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
type failingInspectionWriter struct{}
|
||||
|
||||
func (failingInspectionWriter) Write([]byte) (int, error) {
|
||||
|
||||
@@ -5,25 +5,32 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
configkit "go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
func loadConfigKit(configPath, overlay string, overrides []configkit.Override, opts Options) (*config.LoadedConfig, string, error) {
|
||||
path, err := discoverConfigPath(configPath, opts)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
var source configkit.Source
|
||||
if path := strings.TrimSpace(configPath); path != "" {
|
||||
source = configkit.File(path)
|
||||
} else if path, ok := opts.LookupEnv("NOTARIUS_CONFIG"); ok && strings.TrimSpace(path) != "" {
|
||||
source = configkit.File(strings.TrimSpace(path))
|
||||
} else {
|
||||
source = configkit.SearchConfig("notarius", configkit.Searches(
|
||||
configkit.WorkingDirectory(), configkit.UserConfigDirectory(), configkit.UnixSystemDirectories(),
|
||||
), configkit.SearchOptions{Required: true})
|
||||
}
|
||||
workingDirectory, err := os.Getwd()
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("determine invocation directory: %w", err)
|
||||
}
|
||||
loaded, err := config.Load(path, config.LoadOptions{Overlay: overlay, Overrides: overrides, InvocationDirectory: workingDirectory})
|
||||
loaded, err := config.LoadSource(source, config.LoadOptions{Overlay: overlay, Overrides: overrides, InvocationDirectory: workingDirectory})
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
return loaded, path, nil
|
||||
return loaded, loaded.MainSourcePath(), nil
|
||||
}
|
||||
|
||||
func writeConfigLoadError(output io.Writer, err error) {
|
||||
|
||||
@@ -4,11 +4,12 @@ import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
configkit "go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
func TestConfigLoadDiagnosticsIdentifyComposedSources(t *testing.T) {
|
||||
@@ -24,7 +25,7 @@ func TestConfigLoadDiagnosticsIdentifyComposedSources(t *testing.T) {
|
||||
writeConfigKitFile(t, main, "version: 5\n")
|
||||
args = append(args, "--config-overlay", "development")
|
||||
} else {
|
||||
writeConfigKitFile(t, main, "version: 5\nconfigkit:\n imports:\n - path: "+strconv.Quote(filepath.Base(fragment))+"\n")
|
||||
writeConfigKitFile(t, main, "version: 5\nconfig:\n imports:\n - path: "+strconv.Quote(filepath.Base(fragment))+"\n required: true\n")
|
||||
}
|
||||
writeConfigKitFile(t, fragment, "concurrency: {total_llm: invalid-private-value}\n")
|
||||
var stdout, stderr bytes.Buffer
|
||||
@@ -59,6 +60,79 @@ func TestLoadConfigKitUsesDiscoveryOverlayAndOverrides(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeDiscoverySelectionAndTerminalFailures(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
work := filepath.Join(root, "work")
|
||||
user := filepath.Join(root, "user")
|
||||
if err := os.MkdirAll(work, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
switch runtime.GOOS {
|
||||
case "darwin":
|
||||
t.Setenv("HOME", user)
|
||||
user = filepath.Join(user, "Library", "Application Support")
|
||||
case "windows":
|
||||
t.Setenv("AppData", user)
|
||||
default:
|
||||
t.Setenv("XDG_CONFIG_HOME", user)
|
||||
}
|
||||
previous, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Chdir(work); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := os.Chdir(previous); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
})
|
||||
userFile := filepath.Join(user, "notarius", "config.yml")
|
||||
writeConfigKitFile(t, userFile, "version: 5\npipelines: {user: {input: {module: input}}}\n")
|
||||
writeConfigKitFile(t, filepath.Join(work, "config.yml"), "version: 5\npipelines: {generic: {input: {module: input}}}\n")
|
||||
opts := commandContractOptions(t)
|
||||
loaded, path, err := loadConfigKit("", "", nil, opts)
|
||||
if err != nil || path != userFile || loaded.Config.Pipelines["user"].Input.Module != "input" {
|
||||
t.Fatalf("user discovery = %q, %v", path, err)
|
||||
}
|
||||
workingYAML := filepath.Join(work, "notarius.yaml")
|
||||
workingYML := filepath.Join(work, "notarius.yml")
|
||||
writeConfigKitFile(t, workingYAML, "version: 5\npipelines: {yaml: {input: {module: input}}}\n")
|
||||
writeConfigKitFile(t, workingYML, "version: 5\npipelines: {yml: {input: {module: input}}}\n")
|
||||
loaded, path, err = loadConfigKit("", "", nil, opts)
|
||||
if err != nil || path != workingYML || loaded.Config.Pipelines["yml"].Input.Module != "input" || loaded.Config.Output.Directory != filepath.Join(work, "notarius-output") {
|
||||
t.Fatalf("working discovery = %q, %v", path, err)
|
||||
}
|
||||
for _, args := range [][]string{{"pipelines", "list"}, {"config", "validate"}, {"config", "effective"}, {"config", "explain", "/version"}} {
|
||||
var stdout, stderr bytes.Buffer
|
||||
if code := RunWithOptions(args, &stdout, &stderr, opts); code != 0 || stderr.Len() != 0 || stdout.Len() == 0 {
|
||||
t.Fatalf("%v: code=%d stdout=%q stderr=%q", args, code, stdout.String(), stderr.String())
|
||||
}
|
||||
}
|
||||
missingEnv := opts
|
||||
missingEnv.LookupEnv = func(name string) (string, bool) { return filepath.Join(root, "missing.yml"), name == "NOTARIUS_CONFIG" }
|
||||
if _, _, err := loadConfigKit("", "", nil, missingEnv); err == nil {
|
||||
t.Fatal("missing environment selection fell back")
|
||||
}
|
||||
if err := os.WriteFile(workingYML, []byte("version: [\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := loadConfigKit("", "", nil, opts); err == nil {
|
||||
t.Fatal("invalid selected file fell back")
|
||||
}
|
||||
if _, _, err := loadConfigKit(filepath.Join(root, "missing.yml"), "", nil, opts); err == nil {
|
||||
t.Fatal("missing explicit file fell back")
|
||||
}
|
||||
if err := os.Remove(workingYML); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
loaded, path, err = loadConfigKit("", "", nil, opts)
|
||||
if err != nil || path != workingYAML || loaded.Config.Pipelines["yaml"].Input.Module != "input" {
|
||||
t.Fatalf("alternate named file = %q, %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteConfigLoadErrorUsesSafeDiagnostics(t *testing.T) {
|
||||
t.Parallel()
|
||||
path := filepath.Join(t.TempDir(), "invalid.yml")
|
||||
|
||||
@@ -286,6 +286,13 @@ func completeExampleConfigWithTemporaryCache(t *testing.T) string {
|
||||
if err := os.WriteFile(filepath.Join(root, "pipelines", "dnd-session.yml"), fragment, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
description, err := os.ReadFile(repositoryPath("examples", "pipelines", "dnd-session.describe-session.yml"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, "pipelines", "dnd-session.describe-session.yml"), description, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
path := filepath.Join(root, "dnd-complete.config.yml")
|
||||
if err := os.WriteFile(path, []byte(updated), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -3,6 +3,7 @@ package cli
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
@@ -25,6 +26,7 @@ import (
|
||||
|
||||
func TestMaintainedExamplesLoadResolveAndList(t *testing.T) {
|
||||
components := productionTestComponents(t)
|
||||
completePipelines := make(map[string]pipeline.ResolvedPipeline)
|
||||
for _, example := range maintainedExampleFiles(t) {
|
||||
t.Run(example.name, func(t *testing.T) {
|
||||
cfg := loadMaintainedExample(t, example.path)
|
||||
@@ -58,9 +60,45 @@ func TestMaintainedExamplesLoadResolveAndList(t *testing.T) {
|
||||
t.Fatalf("materialize maintained example references for %q: %v", pipelineID, err)
|
||||
}
|
||||
if example.name == "complete" {
|
||||
if pipelineID == "dnd-session-development" {
|
||||
parent, found := completePipelines["dnd-session"]
|
||||
if !found {
|
||||
t.Fatal("complete parent pipeline was not resolved before its development variant")
|
||||
}
|
||||
if strings.Join(exampleStepLaneIDs(materialized), "|") != strings.Join(exampleStepLaneIDs(parent), "|") {
|
||||
t.Fatalf("development variant steps and lanes = %v, want parent composition %v", exampleStepLaneIDs(materialized), exampleStepLaneIDs(parent))
|
||||
}
|
||||
if got, want := strings.Join(generatedReferenceBindingDescriptions(materialized), "|"), strings.Join(generatedReferenceBindingDescriptions(parent), "|"); got != want {
|
||||
t.Fatalf("development generated reference bindings = %q, want parent bindings %q", got, want)
|
||||
}
|
||||
eligibleBindings := 0
|
||||
for _, step := range materialized.Steps {
|
||||
for _, lane := range step.ArtifactLanes {
|
||||
for _, binding := range []struct {
|
||||
module pipeline.ModuleBinding
|
||||
executionClass contracts.ExecutionClass
|
||||
}{
|
||||
{module: lane.Extract, executionClass: lane.ExtractExecutionClass},
|
||||
{module: lane.Normalize, executionClass: lane.NormalizeExecutionClass},
|
||||
} {
|
||||
if binding.executionClass != "llm_backed" {
|
||||
continue
|
||||
}
|
||||
eligibleBindings++
|
||||
if got := binding.module.StructuredOutputRepairAttempts; got == nil || *got != 0 {
|
||||
t.Fatalf("development %s repair attempts = %v, want explicit zero", binding.module.Module, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if eligibleBindings == 0 {
|
||||
t.Fatal("development variant had no eligible LLM bindings for repair policy")
|
||||
}
|
||||
}
|
||||
if got := exampleStepLaneIDs(materialized); strings.Join(got, "|") != "describe-session:item-registry,location-registry,npc-registry,scene-descriptions|extract-events:combat-turns,item-occurrences,location-occurrences,npc-occurrences,spells|track-enemies:enemy-events" {
|
||||
t.Fatalf("complete example steps and lanes = %v, want the documented D&D extractor composition", got)
|
||||
}
|
||||
completePipelines[pipelineID] = materialized
|
||||
locationLane := referenceContractLane(t, materialized, "location-registry")
|
||||
if locationLane.ArtifactKind != dnd.LocationRegistryKind || locationLane.Extract.Module != locationextract.Key || locationLane.Extract.Retries != 2 || locationLane.Merge.Module != pipeline.DefaultMergeModule || locationLane.Normalize.Module != locationnormalize.Key || locationLane.Normalize.Retries != 2 {
|
||||
t.Fatalf("location lane = %#v, want typed registry composition", locationLane)
|
||||
@@ -155,7 +193,7 @@ func TestMaintainedConfigurationExampleSet(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(llmProfiles) != 1 || llmProfiles[0].Name() != "dnd-extraction.yml" {
|
||||
t.Fatalf("maintained LLM profiles = %#v, want dnd-extraction.yml", llmProfiles)
|
||||
t.Fatalf("maintained LLM profile files = %#v, want the single combined profile file", llmProfiles)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -164,15 +202,38 @@ func TestMaintainedExamplesValidateEffectiveProfilesOffline(t *testing.T) {
|
||||
t.Setenv("OPENROUTER_API_KEY", "")
|
||||
for _, example := range maintainedExampleFiles(t) {
|
||||
t.Run(example.name, func(t *testing.T) {
|
||||
var stdout, stderr strings.Builder
|
||||
code := RunWithOptions([]string{
|
||||
"config", "validate", "--config", example.path, "--pipeline", "dnd-session",
|
||||
}, &stdout, &stderr, Options{})
|
||||
if code != 0 || stderr.Len() != 0 || !strings.Contains(stdout.String(), `valid for pipeline "dnd-session"`) {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
for _, pipelineID := range example.pipelineIDs {
|
||||
t.Run(pipelineID, func(t *testing.T) {
|
||||
var stdout, stderr strings.Builder
|
||||
code := RunWithOptions([]string{
|
||||
"config", "validate", "--config", example.path, "--pipeline", pipelineID,
|
||||
}, &stdout, &stderr, Options{})
|
||||
if code != 0 || stderr.Len() != 0 || !strings.Contains(stdout.String(), `valid for pipeline "`+pipelineID+`"`) {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
var stdout, stderr strings.Builder
|
||||
code := RunWithOptions([]string{
|
||||
"config", "profile", "--config", repositoryPath("examples", "dnd-complete.config.yml"), "dnd-extraction-development",
|
||||
}, &stdout, &stderr, Options{})
|
||||
var profile struct {
|
||||
ProfileID string `json:"profile_id"`
|
||||
Target struct {
|
||||
BackendID string `json:"backend_id"`
|
||||
Model string `json:"model"`
|
||||
Temperature float64 `json:"temperature"`
|
||||
} `json:"effective_model_params"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(stdout.String()), &profile); err != nil {
|
||||
t.Fatalf("development profile JSON = %q: %v", stdout.String(), err)
|
||||
}
|
||||
if code != 0 || stderr.Len() != 0 || profile.ProfileID != "dnd-extraction-development" || profile.Target.BackendID != "openrouter" || profile.Target.Model != "openai/gpt-5.6-luna" || profile.Target.Temperature != 0.2 {
|
||||
t.Fatalf("development profile inspection = code %d, %q, %q, %#v", code, stdout.String(), stderr.String(), profile)
|
||||
}
|
||||
}
|
||||
|
||||
func exampleStepLaneIDs(resolved pipeline.ResolvedPipeline) []string {
|
||||
@@ -188,6 +249,31 @@ func exampleStepLaneIDs(resolved pipeline.ResolvedPipeline) []string {
|
||||
return result
|
||||
}
|
||||
|
||||
func generatedReferenceBindingDescriptions(resolved pipeline.ResolvedPipeline) []string {
|
||||
var descriptions []string
|
||||
for _, step := range resolved.Steps {
|
||||
for _, lane := range step.ArtifactLanes {
|
||||
for _, target := range []struct {
|
||||
name string
|
||||
target pipeline.ResolvedReferenceTarget
|
||||
}{
|
||||
{name: "extract", target: lane.ExtractReferences},
|
||||
{name: "merge", target: lane.MergeReferences},
|
||||
{name: "normalize", target: lane.NormalizeReferences},
|
||||
} {
|
||||
for _, binding := range target.target.Bindings {
|
||||
if binding.Artifact == nil {
|
||||
continue
|
||||
}
|
||||
descriptions = append(descriptions, fmt.Sprintf("%s/%s/%s/%s=%s/%s", step.ID, lane.ID, target.name, binding.SlotName, binding.Artifact.Step, binding.Artifact.Lane))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(descriptions)
|
||||
return descriptions
|
||||
}
|
||||
|
||||
func TestMaintainedMinimalInvocationProducesJSONBundle(t *testing.T) {
|
||||
outputRoot := filepath.Join(t.TempDir(), "output")
|
||||
fake := &productionFakeLLMClient{}
|
||||
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
"testing/fstest"
|
||||
"time"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkmap"
|
||||
@@ -59,6 +60,7 @@ import (
|
||||
scenenormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/scenedescriptions"
|
||||
spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
@@ -228,14 +230,16 @@ func TestProductionAssetsResolveDNDExtractionProfile(t *testing.T) {
|
||||
components := productionTestComponents(t)
|
||||
newEngine := func(profileFile string) (*promptkit.Engine, error) {
|
||||
t.Helper()
|
||||
options, err := components.assets.PromptKitOptions()
|
||||
sources, options, err := components.assets.PromptKitConfigurationSources()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if profileFile != "" {
|
||||
options = append(options, promptkit.WithProfileFile(profileFile))
|
||||
sources.Underlays = append(sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{
|
||||
Source: configkit.File(profileFile), Imports: true,
|
||||
}}})
|
||||
}
|
||||
return promptkit.NewEngine(promptkit.Config{}, options...)
|
||||
return promptkit.NewEngine(promptkit.Config{Sources: sources}, options...)
|
||||
}
|
||||
|
||||
t.Run("fallback", func(t *testing.T) {
|
||||
@@ -258,10 +262,11 @@ func TestProductionAssetsResolveDNDExtractionProfile(t *testing.T) {
|
||||
|
||||
t.Run("valid operator profile wins", func(t *testing.T) {
|
||||
profilePath := filepath.Join(t.TempDir(), "profiles.yaml")
|
||||
if err := os.WriteFile(profilePath, []byte(`id: dnd-extraction
|
||||
endpoint: http://operator.example.test/v1
|
||||
model: operator-model
|
||||
timeout_seconds: 75
|
||||
if err := os.WriteFile(profilePath, []byte(`profiles:
|
||||
dnd-extraction:
|
||||
endpoint: http://operator.example.test/v1
|
||||
model: operator-model
|
||||
timeout_seconds: 75
|
||||
`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -274,14 +279,14 @@ timeout_seconds: 75
|
||||
t.Fatalf("InspectProfile() error = %v, want operator profile", err)
|
||||
}
|
||||
params := inspection.EffectiveModelParams
|
||||
if params.BackendID != "" || params.Endpoint != "http://operator.example.test/v1" || params.Model != "operator-model" || params.TimeoutSeconds != 75 || params.ServiceTier != "" {
|
||||
t.Fatalf("operator profile parameters = %#v, want complete replacement", params)
|
||||
if params.Endpoint != "http://operator.example.test/v1" || params.Model != "operator-model" || params.TimeoutSeconds != 75 || params.ServiceTier != "flex" {
|
||||
t.Fatalf("operator profile parameters = %#v, want recursive overlay over fallback profile", params)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("invalid operator profile does not fall through", func(t *testing.T) {
|
||||
profilePath := filepath.Join(t.TempDir(), "profiles.yaml")
|
||||
if err := os.WriteFile(profilePath, []byte("id: dnd-extraction\nendpoint: http://operator.example.test/v1\nmodel: operator-model\nunknown: value\n"), 0o600); err != nil {
|
||||
if err := os.WriteFile(profilePath, []byte("profiles:\n dnd-extraction:\n endpoint: http://operator.example.test/v1\n model: operator-model\n unknown: value\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine, err := newEngine(profilePath)
|
||||
@@ -320,17 +325,9 @@ func TestProductionPromptAssetsPrepareWithoutProviderCredentials(t *testing.T) {
|
||||
if _, err := fs.ReadFile(schemaFS, filepath.Base(semanticreconcile.SchemaAssetPath)); err != nil {
|
||||
t.Fatalf("generic reconciliation schema asset: %v", err)
|
||||
}
|
||||
options, err := components.assets.PromptKitOptions()
|
||||
if err != nil {
|
||||
t.Fatalf("production PromptKit options: %v", err)
|
||||
}
|
||||
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
||||
ID: "assembled-prompt-test", Endpoint: "http://127.0.0.1:1/v1", Model: "test",
|
||||
})))
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{}, options...)
|
||||
if err != nil {
|
||||
t.Fatalf("production prompt engine: %v", err)
|
||||
}
|
||||
engine := promptkitfixture.NewEngine(t, components.assets, promptkit.Config{}, map[string]promptkit.ProfileConfig{
|
||||
"assembled-prompt-test": promptkitfixture.Profile("http://127.0.0.1:1/v1", "test"),
|
||||
})
|
||||
if engine == nil {
|
||||
t.Fatal("production PromptKit engine is nil")
|
||||
}
|
||||
@@ -504,11 +501,11 @@ func TestProductionLLMClientFactoryBuildsOfflineRuntime(t *testing.T) {
|
||||
}
|
||||
fingerprintProvider, ok := client.(llm.CheckpointFingerprintProvider)
|
||||
if !ok {
|
||||
t.Fatalf("production LLM client %T does not provide one profile-source checkpoint fingerprint", client)
|
||||
t.Fatalf("production LLM client %T does not provide one captured-configuration checkpoint fingerprint", client)
|
||||
}
|
||||
fingerprints, err := fingerprintProvider.LLMCheckpointFingerprints()
|
||||
if err != nil || len(fingerprints) != 1 {
|
||||
t.Fatalf("production LLM checkpoint fingerprints = %#v, error = %v, want one profile-source identity", fingerprints, err)
|
||||
t.Fatalf("production LLM checkpoint fingerprints = %#v, error = %v, want one captured-configuration identity", fingerprints, err)
|
||||
}
|
||||
if _, ok := client.(contracts.LLMProfileManifestProvider); !ok {
|
||||
t.Fatalf("production LLM client %T does not provide profile manifests", client)
|
||||
@@ -550,7 +547,7 @@ func TestNormalizeOptionsSharesProductionProfileAssetsWithDefaultRuntime(t *test
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(runtimeFingerprints, inspectionFingerprints) {
|
||||
t.Fatalf("runtime profile fingerprints = %#v, inspection profile fingerprints = %#v", runtimeFingerprints, inspectionFingerprints)
|
||||
t.Fatalf("runtime configuration fingerprints = %#v, inspection configuration fingerprints = %#v", runtimeFingerprints, inspectionFingerprints)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -575,25 +572,27 @@ func TestProductionLLMClientFactoryUsesConfiguredLocalBackend(t *testing.T) {
|
||||
defer server.Close()
|
||||
|
||||
profilePath := filepath.Join(t.TempDir(), "profiles.yml")
|
||||
if err := os.WriteFile(profilePath, []byte(`id: local-profile
|
||||
backend: local
|
||||
model: local-model
|
||||
if err := os.WriteFile(profilePath, []byte(`profiles:
|
||||
local-profile:
|
||||
backend: local
|
||||
model: local-model
|
||||
`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assets := llm.NewAssetRegistry()
|
||||
if err := assets.RegisterPromptFS(fstest.MapFS{
|
||||
"production.local.yaml": {Data: []byte(`id: production.local
|
||||
version: "v1"
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
messages:
|
||||
- role: user
|
||||
content: '{{ input "transcript" }}'
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json
|
||||
"production.local.yaml": {Data: []byte(`prompts:
|
||||
production.local:
|
||||
v1:
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
messages:
|
||||
- role: user
|
||||
content: '{{ input "transcript" }}'
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json
|
||||
`)},
|
||||
}, "."); err != nil {
|
||||
t.Fatalf("register prompt assets: %v", err)
|
||||
@@ -824,7 +823,7 @@ func maintainedExampleFiles(t *testing.T) []maintainedExample {
|
||||
t.Helper()
|
||||
return []maintainedExample{
|
||||
{name: "minimal", path: repositoryPath("examples", "dnd-minimal.config.yml"), transcriptPath: repositoryPath("examples", "seriatim-minimal-transcript.json"), pipelineIDs: []string{"dnd-session"}},
|
||||
{name: "complete", path: repositoryPath("examples", "dnd-complete.config.yml"), transcriptPath: repositoryPath("examples", "dnd-complete-transcript.json"), pipelineIDs: []string{"dnd-session"}},
|
||||
{name: "complete", path: repositoryPath("examples", "dnd-complete.config.yml"), transcriptPath: repositoryPath("examples", "dnd-complete-transcript.json"), pipelineIDs: []string{"dnd-session", "dnd-session-development"}},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -31,11 +31,12 @@ func TestExplicitPromptKitProfileValidationInspectsProfilesWithoutGeneration(t *
|
||||
}
|
||||
return profilePath
|
||||
}
|
||||
localProfile := "id: local-profile\nbackend: local\nmodel: local-model\n"
|
||||
credentialProfile := `id: credential-profile
|
||||
endpoint: ` + server.URL + `/v1
|
||||
model: credential-model
|
||||
api_key_env: NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY
|
||||
localProfile := "profiles:\n local-profile:\n backend: local\n model: local-model\n"
|
||||
credentialProfile := `profiles:
|
||||
credential-profile:
|
||||
endpoint: ` + server.URL + `/v1
|
||||
model: credential-model
|
||||
api_key_env: NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY
|
||||
`
|
||||
t.Setenv("NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY", "")
|
||||
|
||||
@@ -60,7 +61,7 @@ api_key_env: NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY
|
||||
name: "missing local backend registration",
|
||||
profilePath: writeProfile(t, "local-profile", localProfile),
|
||||
profileID: "local-profile",
|
||||
wantErr: []string{`PromptKit profile "local-profile" is invalid or unreadable`},
|
||||
wantErr: []string{"load PromptKit profiles", "configuration is invalid or unreadable"},
|
||||
},
|
||||
{
|
||||
name: "absent profile",
|
||||
@@ -71,9 +72,9 @@ api_key_env: NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY
|
||||
},
|
||||
{
|
||||
name: "malformed profile",
|
||||
profilePath: writeProfile(t, "malformed-profile", "id: malformed-profile\nendpoint: https://provider.example/v1?credential=forbidden\nmodel: malformed-model\n"),
|
||||
profilePath: writeProfile(t, "malformed-profile", "profiles:\n malformed-profile:\n endpoint: https://provider.example/v1?credential=forbidden\n model: malformed-model\n"),
|
||||
profileID: "malformed-profile",
|
||||
wantErr: []string{`PromptKit profile "malformed-profile" is invalid or unreadable`},
|
||||
wantErr: []string{"load PromptKit profiles", "configuration is invalid or unreadable"},
|
||||
rejectErr: []string{"malformed-profile.yaml", "credential=forbidden"},
|
||||
},
|
||||
{
|
||||
@@ -149,7 +150,7 @@ api_key_env: NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY
|
||||
func TestExplicitPromptKitProfileValidationUsesFallbackAssets(t *testing.T) {
|
||||
assets := llm.NewAssetRegistry()
|
||||
if err := assets.RegisterFallbackProfileFS(fstest.MapFS{
|
||||
"profiles/fallback.yaml": {Data: []byte("id: fallback-profile\nendpoint: http://promptkit.test/v1\nmodel: fallback-model\n")},
|
||||
"profiles/fallback.yaml": {Data: []byte("profiles:\n fallback-profile:\n endpoint: http://promptkit.test/v1\n model: fallback-model\n")},
|
||||
}, "profiles"); err != nil {
|
||||
t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err)
|
||||
}
|
||||
@@ -160,8 +161,8 @@ func TestExplicitPromptKitProfileValidationUsesFallbackAssets(t *testing.T) {
|
||||
|
||||
func TestExplicitPromptKitProfileValidationRejectsInvalidInheritanceBeforeGeneration(t *testing.T) {
|
||||
for _, profiles := range []string{
|
||||
"id: child\nbase_profile: missing\n",
|
||||
"id: first\nbase_profile: second\n\n---\nid: second\nbase_profile: first\n",
|
||||
"profiles:\n child:\n extends: missing\n",
|
||||
"profiles:\n first:\n extends: second\n second:\n extends: first\n",
|
||||
} {
|
||||
t.Run("invalid inheritance", func(t *testing.T) {
|
||||
profilePath := filepath.Join(t.TempDir(), "profiles.yaml")
|
||||
@@ -169,7 +170,7 @@ func TestExplicitPromptKitProfileValidationRejectsInvalidInheritanceBeforeGenera
|
||||
t.Fatal(err)
|
||||
}
|
||||
profileID := "child"
|
||||
if strings.Contains(profiles, "id: first") {
|
||||
if strings.Contains(profiles, "first:") {
|
||||
profileID = "first"
|
||||
}
|
||||
err := validateExplicitPromptKitProfiles(context.Background(), config.Config{PromptKit: config.PromptKitConfig{ProfileFile: profilePath}}, []string{profileID}, nil)
|
||||
|
||||
@@ -15,7 +15,6 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/buildinfo"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
@@ -27,16 +26,17 @@ import (
|
||||
frameworkdebug "gitea.maximumdirect.net/eric/notarius/internal/framework/debug"
|
||||
frameworkllm "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
configkit "go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
const defaultConfigPath = "/usr/local/etc/notarius/config.yml"
|
||||
const usage = `Usage:
|
||||
notarius help
|
||||
notarius --version
|
||||
notarius run <pipeline-id> --input path/to/source.json [--json] [flags]
|
||||
notarius config validate --config path/to/config.yml [--config-overlay name] [--pipeline pipeline-id] [--only lane-a,lane-b]
|
||||
notarius config effective --config path/to/config.yml [--config-overlay name]
|
||||
notarius config explain --config path/to/config.yml [--config-overlay name] <json-pointer>
|
||||
notarius config effective [--scope notarius|promptkit] --config path/to/config.yml [--config-overlay name]
|
||||
notarius config explain [--scope notarius|promptkit] --config path/to/config.yml [--config-overlay name] <json-pointer>
|
||||
notarius config profile [--config path/to/config.yml] [--config-overlay name] <profile-id>
|
||||
notarius pipelines list --config path/to/config.yml [--config-overlay name] [--json]
|
||||
`
|
||||
|
||||
@@ -1059,6 +1059,8 @@ func runConfig(args []string, stdout, stderr io.Writer, opts Options) int {
|
||||
return runConfigEffective(args[1:], stdout, stderr, opts)
|
||||
case "explain":
|
||||
return runConfigExplain(args[1:], stdout, stderr, opts)
|
||||
case "profile":
|
||||
return runConfigProfile(args[1:], stdout, stderr, opts)
|
||||
default:
|
||||
fmt.Fprintf(stderr, "notarius: unknown config subcommand %q\n", args[0])
|
||||
writeUsage(stderr)
|
||||
@@ -1082,39 +1084,6 @@ func runPipelines(args []string, stdout, stderr io.Writer, opts Options) int {
|
||||
}
|
||||
}
|
||||
|
||||
func discoverConfigPath(configPath string, opts Options) (string, error) {
|
||||
if path := strings.TrimSpace(configPath); path != "" {
|
||||
if err := requireConfigFile(path); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return path, nil
|
||||
}
|
||||
if path, ok := opts.LookupEnv("NOTARIUS_CONFIG"); ok && strings.TrimSpace(path) != "" {
|
||||
path = strings.TrimSpace(path)
|
||||
if err := requireConfigFile(path); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return path, nil
|
||||
}
|
||||
if _, err := os.Stat(defaultConfigPath); err == nil {
|
||||
return defaultConfigPath, nil
|
||||
} else if err != nil && !os.IsNotExist(err) {
|
||||
return "", fmt.Errorf("check default config %q: %w", defaultConfigPath, err)
|
||||
}
|
||||
return "", fmt.Errorf("config file not found; pass --config or set NOTARIUS_CONFIG")
|
||||
}
|
||||
|
||||
func requireConfigFile(path string) error {
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("config file %q is not available: %w", path, err)
|
||||
}
|
||||
if info.IsDir() {
|
||||
return fmt.Errorf("config file %q is a directory", path)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseOnly(raw string) ([]string, error) {
|
||||
if strings.TrimSpace(raw) == "" {
|
||||
return nil, nil
|
||||
|
||||
@@ -271,6 +271,41 @@ func TestRunAppliesConfigurationOverlayAndRecordsSelection(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRecordsDiscoveredConfigurationPath(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
work := t.TempDir()
|
||||
selected := filepath.Join(work, "notarius.yml")
|
||||
data, err := os.ReadFile(roots.config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(selected, data, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
previous, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Chdir(work); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := os.Chdir(previous); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
})
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "sample", "--input", roots.input, "--chunk_cache", "bypass", "--debug"}, &stdout, &stderr, newStateTestHarness().options())
|
||||
if code != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
var invocation debugbundle.Invocation
|
||||
readStateTestSummaryJSON(t, onlyChildDir(t, roots.debug), "invocation.json", &invocation)
|
||||
if invocation.ConfigPath != selected {
|
||||
t.Fatalf("recorded config path = %q, want %q", invocation.ConfigPath, selected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEffectiveConfigOverlayOmitsBaseSelection(t *testing.T) {
|
||||
for _, overlay := range []string{"", "default"} {
|
||||
if got := effectiveConfigOverlay(overlay); got != "" {
|
||||
@@ -679,7 +714,7 @@ func writeRunContractProfiles(t *testing.T, ids ...string) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
for _, id := range ids {
|
||||
profile := fmt.Sprintf("id: %s\nendpoint: http://127.0.0.1:1/v1\nmodel: %s-model\n", id, id)
|
||||
profile := fmt.Sprintf("profiles:\n %s:\n endpoint: http://127.0.0.1:1/v1\n model: %s-model\n", id, id)
|
||||
if err := os.WriteFile(filepath.Join(dir, id+".yaml"), []byte(profile), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -214,7 +214,7 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t
|
||||
t.Fatal(err)
|
||||
}
|
||||
fingerprints := prepared.CheckpointFingerprints()
|
||||
llmFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_profile_source", Value: "sha256:profile-source-one"}}
|
||||
llmFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_configuration", Value: "sha256:configuration-one"}}
|
||||
settings := config.CheckpointCacheConfig{Enabled: true, Directory: t.TempDir()}
|
||||
recorder, _, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, false)
|
||||
if err != nil {
|
||||
@@ -274,16 +274,16 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t
|
||||
t.Fatalf("changed mapping policy decision = %#v, want cold miss", decision)
|
||||
}
|
||||
|
||||
changedLLMFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_profile_source", Value: "sha256:profile-source-two"}}
|
||||
changedLLMFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_configuration", Value: "sha256:configuration-two"}}
|
||||
_, profileLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, changedLLMFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, decision := profileLoader.Source(materialized.Input.Module); decision.Reused {
|
||||
t.Fatalf("changed PromptKit profile source decision = %#v, want cold miss", decision)
|
||||
t.Fatalf("changed PromptKit configuration decision = %#v, want cold miss", decision)
|
||||
}
|
||||
if _, decision := profileLoader.Normalize("spells", spellnormalize.Key, normalizeDependencies); decision.Reused {
|
||||
t.Fatalf("changed PromptKit profile normalize decision = %#v, want cold miss", decision)
|
||||
t.Fatalf("changed PromptKit configuration normalize decision = %#v, want cold miss", decision)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -154,7 +154,7 @@ func TestRunRetainsDebugBundlesAcrossFailures(t *testing.T) {
|
||||
root := filepath.Join(t.TempDir(), "debug")
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "sample", "--config", filepath.Join(t.TempDir(), "missing.yml"), "--input", "missing", "--debug", "--debug-dir", root}, &stdout, &stderr, newStateTestHarness().options())
|
||||
if code != 1 || !strings.Contains(stderr.String(), "config file") {
|
||||
if code != 1 || !strings.Contains(stderr.String(), "source.missing") {
|
||||
t.Fatalf("code=%d stderr=%q", code, stderr.String())
|
||||
}
|
||||
assertAbsent(t, root)
|
||||
|
||||
305
internal/core/config/collection_composition_test.go
Normal file
305
internal/core/config/collection_composition_test.go
Normal file
@@ -0,0 +1,305 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
configkit "go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
func TestStepCompositionAcrossSourcesAndOverlays(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
write := func(name, data string) {
|
||||
t.Helper()
|
||||
path := filepath.Join(root, name)
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(data), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
write("notarius.yml", `version: 5
|
||||
config:
|
||||
imports:
|
||||
- path: one/pipeline.yml
|
||||
required: true
|
||||
- path: two/pipeline.yml
|
||||
required: true
|
||||
`)
|
||||
write("one/pipeline.yml", `pipelines:
|
||||
session:
|
||||
input: {module: input}
|
||||
steps:
|
||||
- id: first
|
||||
references: {source: {path: ./first.txt}}
|
||||
artifacts: {first: {extract: {module: extract, retries: 4}}}
|
||||
`)
|
||||
write("two/pipeline.yml", `pipelines:
|
||||
session:
|
||||
steps:
|
||||
- id: second
|
||||
references: {source: {path: ./second.txt}}
|
||||
artifacts: {second: {extract: {module: extract}}}
|
||||
`)
|
||||
write("overlays/upsert.yml", `pipelines:
|
||||
session:
|
||||
steps:
|
||||
- id: first
|
||||
artifacts: {replacement: {extract: {module: extract}}}
|
||||
`)
|
||||
write("overlays/replace.yml", `config:
|
||||
sequences:
|
||||
- destination: [pipelines, session, steps]
|
||||
mode: replace
|
||||
pipelines:
|
||||
session:
|
||||
steps:
|
||||
- id: third
|
||||
artifacts: {third: {extract: {module: extract}}}
|
||||
`)
|
||||
write("overlays/append.yml", `config:
|
||||
sequences:
|
||||
- destination: [pipelines, session, steps]
|
||||
mode: append
|
||||
pipelines:
|
||||
session:
|
||||
steps:
|
||||
- id: third
|
||||
artifacts: {third: {extract: {module: extract}}}
|
||||
`)
|
||||
write("overlays/clear.yml", `config:
|
||||
sequences:
|
||||
- destination: [pipelines, session, steps]
|
||||
mode: replace
|
||||
pipelines:
|
||||
session: {steps: []}
|
||||
`)
|
||||
main := filepath.Join(root, "notarius.yml")
|
||||
loaded, err := Load(main, LoadOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
steps := loaded.Config.Pipelines["session"].Steps
|
||||
if len(steps) != 2 || steps[0].ID != "first" || steps[1].ID != "second" || steps[0].References["source"].Path != filepath.Join(root, "one", "first.txt") || steps[1].References["source"].Path != filepath.Join(root, "two", "second.txt") {
|
||||
t.Fatalf("composed steps = %#v", steps)
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
overlay string
|
||||
ids []string
|
||||
}{{"upsert", []string{"first", "second"}}, {"replace", []string{"third"}}, {"append", []string{"first", "second", "third"}}} {
|
||||
t.Run(tc.overlay, func(t *testing.T) {
|
||||
got, err := Load(main, LoadOptions{Overlay: tc.overlay})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
steps := got.Config.Pipelines["session"].Steps
|
||||
if len(steps) != len(tc.ids) {
|
||||
t.Fatalf("steps = %#v", steps)
|
||||
}
|
||||
for i, id := range tc.ids {
|
||||
if steps[i].ID != id {
|
||||
t.Fatalf("steps = %#v", steps)
|
||||
}
|
||||
}
|
||||
if tc.overlay == "upsert" && (len(steps[0].References) != 0 || steps[0].Artifacts["replacement"].Extract.Module != "extract") {
|
||||
t.Fatalf("upsert did not replace record: %#v", steps[0])
|
||||
}
|
||||
})
|
||||
}
|
||||
if _, err := Load(main, LoadOptions{Overlay: "clear"}); err == nil {
|
||||
t.Fatal("domain validation accepted empty steps")
|
||||
}
|
||||
overrides := []configkit.Override{{Path: configkit.Keys("pipelines", "session", "steps"), Value: []any{map[string]any{"id": "override", "artifacts": map[string]any{"result": map[string]any{"extract": map[string]any{"module": "extract"}}}}}, Flag: "--steps"}}
|
||||
replaced, err := Load(main, LoadOptions{Overrides: overrides})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := replaced.Config.Pipelines["session"].Steps; len(got) != 1 || got[0].ID != "override" {
|
||||
t.Fatalf("override steps = %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStepIdentityRejectsDuplicatePeersWithoutRegisteringOtherLists(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, first, second string
|
||||
}{
|
||||
{"duplicate steps", "steps: [{id: same, artifacts: {a: {extract: {module: extract}}}}]", "steps: [{id: same, artifacts: {b: {extract: {module: extract}}}}]"},
|
||||
{"unregistered validators", "input: {module: input, validators: [{module: one}]}", "input: {validators: [{module: two}]}"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
for name, data := range map[string]string{
|
||||
"main.yml": "version: 5\nconfig:\n imports:\n - path: one.yml\n - path: two.yml\n",
|
||||
"one.yml": "pipelines:\n session:\n " + tc.first + "\n",
|
||||
"two.yml": "pipelines:\n session:\n " + tc.second + "\n",
|
||||
} {
|
||||
if err := os.WriteFile(filepath.Join(root, name), []byte(data), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := Load(filepath.Join(root, "main.yml"), LoadOptions{}); err == nil {
|
||||
t.Fatal("conflicting peer lists were accepted")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStepRecordInheritanceReplacesMatchingRecord(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
main := filepath.Join(root, "main.yml")
|
||||
if err := os.WriteFile(main, []byte(`version: 5
|
||||
pipelines:
|
||||
parent:
|
||||
input: {module: input}
|
||||
steps:
|
||||
- id: first
|
||||
artifacts: {old: {extract: {module: extract, retries: 4}}}
|
||||
- id: second
|
||||
artifacts: {second: {extract: {module: extract}}}
|
||||
child:
|
||||
extends: parent
|
||||
steps:
|
||||
- id: first
|
||||
artifacts: {new: {extract: {module: extract}}}
|
||||
`), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
loaded, err := Load(main, LoadOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
steps := loaded.Config.Pipelines["child"].Steps
|
||||
if len(steps) != 2 || steps[0].ID != "first" || steps[1].ID != "second" || len(steps[0].Artifacts) != 1 || steps[0].Artifacts["new"].Extract.Module != "extract" {
|
||||
t.Fatalf("inherited steps = %#v", steps)
|
||||
}
|
||||
if _, err := loaded.Config.Resolve(ResolveInput{PipelineID: "child", Catalog: effectiveCatalog(t)}); err != nil {
|
||||
t.Fatalf("resolve composed pipeline: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIdentifiedStepInheritanceDoesNotCreateTemplateSteps(t *testing.T) {
|
||||
main := filepath.Join(t.TempDir(), "main.yml")
|
||||
if err := os.WriteFile(main, []byte(`version: 5
|
||||
pipelines:
|
||||
session:
|
||||
input: {module: input}
|
||||
steps:
|
||||
- id: parent
|
||||
references: {source: {path: ./shared.txt}}
|
||||
- id: child
|
||||
extends: {id: parent}
|
||||
artifacts: {result: {extract: {module: extract}}}
|
||||
`), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
loaded, err := Load(main, LoadOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
steps := loaded.Config.Pipelines["session"].Steps
|
||||
if len(steps) != 2 || steps[1].ID != "child" || steps[1].References["source"].Path != filepath.Join(filepath.Dir(main), "shared.txt") {
|
||||
t.Fatalf("inherited step = %#v", steps)
|
||||
}
|
||||
if err := loaded.Config.Validate(); err != nil {
|
||||
t.Fatalf("validate inherited steps: %v", err)
|
||||
}
|
||||
if _, err := loaded.Config.Resolve(ResolveInput{PipelineID: "session", Catalog: effectiveCatalog(t)}); err == nil || !strings.Contains(err.Error(), `step "parent" must declare at least one artifact lane`) {
|
||||
t.Fatalf("expected non-executable parent rejection, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIdentifiedValidatorRecordInheritanceResolves(t *testing.T) {
|
||||
main := filepath.Join(t.TempDir(), "main.yml")
|
||||
if err := os.WriteFile(main, []byte(`version: 5
|
||||
config:
|
||||
sequences:
|
||||
- destination: [pipelines, session, artifacts, result, extract, validators]
|
||||
key: module
|
||||
pipelines:
|
||||
session:
|
||||
input: {module: input}
|
||||
artifacts:
|
||||
result:
|
||||
extract:
|
||||
module: extract
|
||||
validators:
|
||||
- module: configured-a
|
||||
options: {nested: {inherited: true}}
|
||||
- module: configured-b
|
||||
extends: {module: configured-a}
|
||||
`), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
loaded, err := Load(main, LoadOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
effective, err := loaded.Config.Resolve(ResolveInput{PipelineID: "session", Catalog: effectiveCatalog(t)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, chain := range effective.ResolvedPipeline.ValidatorChains {
|
||||
for _, validator := range chain.Validators {
|
||||
if validator.Binding.Module == "configured-b" {
|
||||
if nested, ok := validator.Binding.Options["nested"].(map[string]any); !ok || nested["inherited"] != true {
|
||||
t.Fatalf("inherited options = %#v", validator.Binding.Options)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
t.Fatal("inheriting validator missing from resolved pipeline")
|
||||
}
|
||||
|
||||
func TestEmptyStepOperationsAndOptionalImports(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
main := filepath.Join(root, "main.yml")
|
||||
if err := os.WriteFile(main, []byte(`version: 5
|
||||
config:
|
||||
imports:
|
||||
- path: absent.yml
|
||||
pipelines:
|
||||
session:
|
||||
input: {module: input}
|
||||
steps:
|
||||
- id: first
|
||||
artifacts: {result: {extract: {module: extract}}}
|
||||
`), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
mode, value string
|
||||
wantError bool
|
||||
}{
|
||||
{"append", "[]", false}, {"upsert", "[]", false}, {"replace", "[]", true}, {"replace", "null", true},
|
||||
} {
|
||||
t.Run(tc.mode+tc.value, func(t *testing.T) {
|
||||
name := tc.mode + map[string]string{"[]": "empty", "null": "null"}[tc.value]
|
||||
path := filepath.Join(root, "overlays", name+".yml")
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data := "config:\n sequences:\n - destination: [pipelines, session, steps]\n mode: " + tc.mode + "\npipelines:\n session: {steps: " + tc.value + "}\n"
|
||||
if err := os.WriteFile(path, []byte(data), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
loaded, err := Load(main, LoadOptions{Overlay: name})
|
||||
if tc.wantError {
|
||||
if err == nil {
|
||||
t.Fatal("invalid steps were accepted")
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
steps := loaded.Config.Pipelines["session"].Steps
|
||||
if len(steps) != 1 || steps[0].ID != "first" {
|
||||
t.Fatalf("steps = %#v", steps)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,26 +1,16 @@
|
||||
package config
|
||||
|
||||
import configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
import configkit "go.maximumdirect.net/configkit"
|
||||
|
||||
// documentNormalizer records leaf provenance and, while visiting a sequence,
|
||||
// covers its atomic mutation boundary. ConfigKit audits sequences as whole
|
||||
// values, so a record for only a descendant is insufficient. The outermost
|
||||
// sequence covers nested sequences as well; unchanged visits record nothing.
|
||||
// A step whose identity is normalized no longer corresponds to its original
|
||||
// record. Its ancestor record covers its edits instead of invalid leaf records.
|
||||
type documentNormalizer struct {
|
||||
*configkit.Resolution
|
||||
sequence *configkit.Path
|
||||
covered bool
|
||||
}
|
||||
|
||||
func (n documentNormalizer) inSequence(path configkit.Path) documentNormalizer {
|
||||
if n.sequence == nil {
|
||||
n.sequence = &path
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func (n documentNormalizer) Record(path configkit.Path, kind configkit.ChangeKind, reason string, dependencies ...configkit.Path) {
|
||||
n.Resolution.Record(path, kind, reason, dependencies...)
|
||||
if n.sequence != nil {
|
||||
n.Resolution.Record(*n.sequence, kind, reason, append([]configkit.Path{path}, dependencies...)...)
|
||||
func (n documentNormalizer) RecordNormalization(path configkit.Path, reason string, dependencies ...configkit.Path) {
|
||||
if !n.covered {
|
||||
n.Resolution.RecordNormalization(path, reason, dependencies...)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,12 +2,13 @@ package config
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
configkit "go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
func normalizeDocumentPaths(document *Document, context *configkit.Resolution, state *documentLoadState) {
|
||||
resolution := documentNormalizer{Resolution: context}
|
||||
func normalizeDocumentPaths(document *Document, resolution *configkit.Resolution, state *documentLoadState) {
|
||||
normalizer := documentNormalizer{Resolution: resolution}
|
||||
if document == nil {
|
||||
return
|
||||
}
|
||||
@@ -33,25 +34,29 @@ func normalizeDocumentPaths(document *Document, context *configkit.Resolution, s
|
||||
for _, id := range sortedPipelineDocumentKeys(document.Pipelines) {
|
||||
profile := document.Pipelines[id]
|
||||
path := configkit.Keys("pipelines").Key(id)
|
||||
normalizeDocumentReferencePaths(profile.References, path.Key("references"), resolution, state)
|
||||
normalizeDocumentBindingReferencePaths(&profile.Input, path.Key("input"), resolution, state)
|
||||
normalizeDocumentReferencePaths(profile.References, path.Key("references"), normalizer, state)
|
||||
normalizeDocumentBindingReferencePaths(&profile.Input, path.Key("input"), normalizer, state)
|
||||
if profile.Chunk != nil {
|
||||
normalizeDocumentBindingReferencePaths(profile.Chunk, path.Key("chunk"), resolution, state)
|
||||
normalizeDocumentBindingReferencePaths(profile.Chunk, path.Key("chunk"), normalizer, state)
|
||||
}
|
||||
if profile.Output != nil {
|
||||
normalizeDocumentBindingReferencePaths(profile.Output, path.Key("output"), resolution, state)
|
||||
normalizeDocumentBindingReferencePaths(profile.Output, path.Key("output"), normalizer, state)
|
||||
}
|
||||
for laneID, lane := range profile.Artifacts {
|
||||
normalizeDocumentLaneReferencePaths(&lane, path.Key("artifacts").Key(laneID), resolution, state)
|
||||
normalizeDocumentLaneReferencePaths(&lane, path.Key("artifacts").Key(laneID), normalizer, state)
|
||||
profile.Artifacts[laneID] = lane
|
||||
}
|
||||
stepResolution := resolution.inSequence(path.Key("steps"))
|
||||
for index := range profile.Steps {
|
||||
step := &profile.Steps[index]
|
||||
stepPath := path.Key("steps").Index(index)
|
||||
normalizeDocumentReferencePaths(step.References, stepPath.Key("references"), stepResolution, state)
|
||||
stepNormalizer := normalizer
|
||||
if step.ID != strings.TrimSpace(step.ID) {
|
||||
resolution.RecordNormalization(path.Key("steps"), "normalized step identity")
|
||||
stepNormalizer.covered = true
|
||||
}
|
||||
normalizeDocumentReferencePaths(step.References, stepPath.Key("references"), stepNormalizer, state)
|
||||
for laneID, lane := range step.Artifacts {
|
||||
normalizeDocumentLaneReferencePaths(&lane, stepPath.Key("artifacts").Key(laneID), stepResolution, state)
|
||||
normalizeDocumentLaneReferencePaths(&lane, stepPath.Key("artifacts").Key(laneID), stepNormalizer, state)
|
||||
step.Artifacts[laneID] = lane
|
||||
}
|
||||
}
|
||||
@@ -59,14 +64,14 @@ func normalizeDocumentPaths(document *Document, context *configkit.Resolution, s
|
||||
}
|
||||
}
|
||||
|
||||
func initializeDocumentMainDirectory(resolution documentNormalizer, state *documentLoadState) {
|
||||
func initializeDocumentMainDirectory(resolution *configkit.Resolution, state *documentLoadState) {
|
||||
if state == nil || state.mainDirectory != "" {
|
||||
return
|
||||
}
|
||||
info, ok := resolution.InputInfo(configkit.Keys("version"))
|
||||
info, ok := resolution.EntryInfo(configkit.Keys("version"))
|
||||
if ok && len(info.Origin.Locations) == 1 {
|
||||
if source, sourceOK := resolution.Source(info.Origin.Locations[0].SourceID); sourceOK && source.Path != "" {
|
||||
state.mainDirectory = filepath.Dir(source.Path)
|
||||
if directory := state.sourceDirectory(info.Origin.Locations[0].SourceID, resolution.Source); directory != "" {
|
||||
state.mainDirectory = directory
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -95,23 +100,30 @@ func normalizeDocumentReferencePaths(references map[string]ReferenceDocument, pa
|
||||
for _, name := range sortedReferenceDocumentKeys(references) {
|
||||
source := references[name]
|
||||
sourcePath := path.Key(name).Key("path")
|
||||
if source.Path != nil && resolution.IsExplicit(sourcePath) {
|
||||
value := strings.TrimSpace(*source.Path)
|
||||
if value != *source.Path {
|
||||
resolution.RecordNormalization(sourcePath, "trimmed reference path")
|
||||
source.Path = documentValue(value)
|
||||
}
|
||||
}
|
||||
if source.Path != nil && *source.Path != "" && !filepath.IsAbs(*source.Path) {
|
||||
base, ok := documentPathBase(resolution, state, sourcePath)
|
||||
base, ok := documentPathBase(resolution.Resolution, state, sourcePath)
|
||||
if !ok {
|
||||
resolution.Error(sourcePath, "relative path has no unambiguous source base")
|
||||
} else {
|
||||
value := filepath.Clean(filepath.Join(base, *source.Path))
|
||||
if value != *source.Path {
|
||||
resolution.Record(sourcePath, configkit.ChangeNormalization, "resolved external reference path")
|
||||
resolution.RecordNormalization(sourcePath, "resolved external reference path")
|
||||
source.Path = documentValue(value)
|
||||
}
|
||||
}
|
||||
references[name] = source
|
||||
}
|
||||
references[name] = source
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeDocumentPath(resolution documentNormalizer, state *documentLoadState, path configkit.Path, target **string, optional bool, reason string) {
|
||||
func normalizeDocumentPath(resolution *configkit.Resolution, state *documentLoadState, path configkit.Path, target **string, optional bool, reason string) {
|
||||
if target == nil || *target == nil || **target == "" || filepath.IsAbs(**target) {
|
||||
return
|
||||
}
|
||||
@@ -125,16 +137,16 @@ func normalizeDocumentPath(resolution documentNormalizer, state *documentLoadSta
|
||||
return
|
||||
}
|
||||
if *target != nil && **target != value {
|
||||
configkit.Normalize(resolution.Resolution, path, target, documentValue(value), reason)
|
||||
configkit.Normalize(resolution, path, target, documentValue(value), reason)
|
||||
}
|
||||
}
|
||||
|
||||
func documentPathBase(resolution documentNormalizer, state *documentLoadState, path configkit.Path) (string, bool) {
|
||||
func documentPathBase(resolution *configkit.Resolution, state *documentLoadState, path configkit.Path) (string, bool) {
|
||||
original := path
|
||||
if state != nil {
|
||||
original = state.originalPath(path)
|
||||
}
|
||||
info, ok := resolution.InputInfo(original)
|
||||
info, ok := resolution.EntryInfo(original)
|
||||
if !ok {
|
||||
if state != nil && state.mainDirectory != "" {
|
||||
return state.mainDirectory, true
|
||||
@@ -154,14 +166,35 @@ func documentPathBase(resolution documentNormalizer, state *documentLoadState, p
|
||||
if len(info.Origin.Locations) != 1 {
|
||||
return "", false
|
||||
}
|
||||
source, ok := resolution.Source(info.Origin.Locations[0].SourceID)
|
||||
if !ok || source.Path == "" {
|
||||
directory := state.sourceDirectory(info.Origin.Locations[0].SourceID, resolution.Source)
|
||||
if directory == "" {
|
||||
if state != nil && state.mainDirectory != "" {
|
||||
return state.mainDirectory, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
return filepath.Dir(source.Path), true
|
||||
return directory, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// Source evidence is fixed during resolution. Cache only the directory needed
|
||||
// by paths, avoiding repeated copies of unrelated import-selection evidence.
|
||||
func (state *documentLoadState) sourceDirectory(id string, lookup func(string) (configkit.SourceInfo, bool)) string {
|
||||
if state != nil {
|
||||
if directory, ok := state.sourceDirectories[id]; ok {
|
||||
return directory
|
||||
}
|
||||
}
|
||||
directory := ""
|
||||
if source, ok := lookup(id); ok && source.Path != "" {
|
||||
directory = filepath.Dir(source.Path)
|
||||
}
|
||||
if state != nil {
|
||||
if state.sourceDirectories == nil {
|
||||
state.sourceDirectories = make(map[string]string)
|
||||
}
|
||||
state.sourceDirectories[id] = directory
|
||||
}
|
||||
return directory
|
||||
}
|
||||
|
||||
@@ -5,9 +5,47 @@ import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
configkit "go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
func TestSourceDirectoryLookupIsMemoizedPerLoad(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
state := newDocumentLoadState(dir)
|
||||
calls := make(map[string]int)
|
||||
lookup := func(id string) (configkit.SourceInfo, bool) {
|
||||
calls[id]++
|
||||
if id == "missing" {
|
||||
return configkit.SourceInfo{}, false
|
||||
}
|
||||
if id == "memory" {
|
||||
return configkit.SourceInfo{}, true
|
||||
}
|
||||
return configkit.SourceInfo{Path: filepath.Join(dir, id, "config.yml")}, true
|
||||
}
|
||||
for range 2 {
|
||||
for _, id := range []string{"first", "second", "missing", "memory"} {
|
||||
want := ""
|
||||
if id == "first" || id == "second" {
|
||||
want = filepath.Join(dir, id)
|
||||
}
|
||||
if got := state.sourceDirectory(id, lookup); got != want {
|
||||
t.Fatalf("directory(%s) = %q, want %q", id, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
for id, count := range calls {
|
||||
if count != 1 {
|
||||
t.Fatalf("source %s queried %d times", id, count)
|
||||
}
|
||||
}
|
||||
other := newDocumentLoadState(dir)
|
||||
if got := other.sourceDirectory("first", func(string) (configkit.SourceInfo, bool) {
|
||||
return configkit.SourceInfo{Path: filepath.Join(dir, "changed", "config.yml")}, true
|
||||
}); got != filepath.Join(dir, "changed") {
|
||||
t.Fatalf("another load reused directory %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDocumentPathsUseDeclaringFilesAcrossImports(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -27,7 +65,7 @@ func TestDocumentPathsUseDeclaringFilesAcrossImports(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(main, []byte(`version: 5
|
||||
configkit:
|
||||
config:
|
||||
imports:
|
||||
- path: fragments/pipelines.yml
|
||||
output: {directory: ./output}
|
||||
@@ -56,6 +94,51 @@ pipelines:
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscoveredMainAndStepReferenceNormalization(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
fragmentDir := filepath.Join(root, "fragments")
|
||||
if err := os.Mkdir(fragmentDir, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
main := filepath.Join(root, "notarius.yml")
|
||||
fragment := filepath.Join(fragmentDir, "pipeline.yml")
|
||||
if err := os.WriteFile(main, []byte("version: 5\nconfig:\n imports:\n - path: fragments/pipeline.yml\n required: true\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(fragment, []byte(`pipelines:
|
||||
session:
|
||||
input: {module: input}
|
||||
steps:
|
||||
- id: " first "
|
||||
references:
|
||||
" source ": {path: " ./source.txt "}
|
||||
artifacts:
|
||||
result:
|
||||
extract: {module: " extract "}
|
||||
- id: second
|
||||
artifacts:
|
||||
next:
|
||||
extract: {module: " extract "}
|
||||
`), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
loaded, err := LoadSource(configkit.SearchConfig("", configkit.SearchPaths(main), configkit.SearchOptions{Required: true}), LoadOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if loaded.MainSourcePath() != main || loaded.Config.Output.Directory != filepath.Join(root, "notarius-output") {
|
||||
t.Fatalf("discovered main or default directory: %q, %q", loaded.MainSourcePath(), loaded.Config.Output.Directory)
|
||||
}
|
||||
step := loaded.Config.Pipelines["session"].Steps[0]
|
||||
if step.ID != "first" || step.Artifacts["result"].Extract.Module != "extract" || step.References["source"].Path != filepath.Join(fragmentDir, "source.txt") {
|
||||
t.Fatalf("normalized step = %#v", step)
|
||||
}
|
||||
history := loaded.result.Metadata().History(configkit.Keys("pipelines", "session", "steps").Index(1).Key("artifacts").Key("next").Key("extract").Key("module"))
|
||||
if len(history) < 2 || history[len(history)-1].Origin.Operation != configkit.ChangeNormalization {
|
||||
t.Fatalf("module leaf history = %#v", history)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDocumentPathsUseInvocationDirectoryForOverrides(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -77,6 +160,37 @@ func TestDocumentPathsUseInvocationDirectoryForOverrides(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscoveredSymlinkUsesCanonicalMainDirectory(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
realDir := filepath.Join(root, "real")
|
||||
if err := os.Mkdir(realDir, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
main := filepath.Join(realDir, "main.yml")
|
||||
if err := os.WriteFile(main, []byte("version: 5\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
link := filepath.Join(root, "notarius.yml")
|
||||
if err := os.Symlink(main, link); err != nil {
|
||||
t.Skipf("symlink unavailable: %v", err)
|
||||
}
|
||||
loaded, err := LoadSource(configkit.SearchConfig("", configkit.SearchPaths(link), configkit.SearchOptions{Required: true}), LoadOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if loaded.MainSourcePath() != main || loaded.Config.Output.Directory != filepath.Join(realDir, "notarius-output") {
|
||||
t.Fatalf("canonical source = %q, output = %q", loaded.MainSourcePath(), loaded.Config.Output.Directory)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequiredSearchExhaustion(t *testing.T) {
|
||||
missing := filepath.Join(t.TempDir(), "missing.yml")
|
||||
_, err := LoadSource(configkit.SearchConfig("", configkit.SearchPaths(missing), configkit.SearchOptions{Required: true}), LoadOptions{})
|
||||
if err == nil {
|
||||
t.Fatal("missing required search succeeded")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDocumentPathsUseOverlayDeclarationDirectory(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
configkit "go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
func TestDocumentLoadAcceptsVersion5AndCanonicalShapes(t *testing.T) {
|
||||
|
||||
622
internal/core/config/inheritance_test.go
Normal file
622
internal/core/config/inheritance_test.go
Normal file
@@ -0,0 +1,622 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
configkit "go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
func TestPipelineInheritancePreservesAndUpsertsSteps(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
childSteps string
|
||||
wantStepIDs []string
|
||||
}{
|
||||
{
|
||||
name: "inherits ordered steps",
|
||||
wantStepIDs: []string{"prepare", "summarize"},
|
||||
},
|
||||
{
|
||||
name: "appends distinct ordered steps",
|
||||
childSteps: `
|
||||
steps:
|
||||
- id: replacement
|
||||
artifacts:
|
||||
summary:
|
||||
extract: {module: extract}`,
|
||||
wantStepIDs: []string{"prepare", "summarize", "replacement"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
source := `version: 5
|
||||
pipelines:
|
||||
parent:
|
||||
input: {module: input}
|
||||
structured_output_repair_attempts: 2
|
||||
steps:
|
||||
- id: prepare
|
||||
artifacts:
|
||||
score:
|
||||
extract: {module: extract, retries: 2}
|
||||
- id: summarize
|
||||
artifacts:
|
||||
notes:
|
||||
extract: {module: extract}
|
||||
child:
|
||||
extends: parent
|
||||
llm_profile: local
|
||||
structured_output_repair_attempts: 0` + test.childSteps + "\n"
|
||||
loaded, err := loadConfigSource(configkit.Bytes("inheritance.yml", []byte(source)), LoadOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("loadConfigSource() error = %v", err)
|
||||
}
|
||||
|
||||
parent := loaded.Config.Pipelines["parent"]
|
||||
child := loaded.Config.Pipelines["child"]
|
||||
if parent.Input.Module != "input" || child.Input.Module != "input" {
|
||||
t.Fatalf("parent/child inputs = %#v / %#v", parent.Input, child.Input)
|
||||
}
|
||||
if child.LLMProfile != "local" ||
|
||||
child.StructuredOutputRepairAttempts == nil || *child.StructuredOutputRepairAttempts != 0 {
|
||||
t.Fatalf("child overrides = %#v", child)
|
||||
}
|
||||
if parent.StructuredOutputRepairAttempts == nil || *parent.StructuredOutputRepairAttempts != 2 {
|
||||
t.Fatalf("parent settings changed: %#v", parent)
|
||||
}
|
||||
if len(child.Steps) != len(test.wantStepIDs) {
|
||||
t.Fatalf("child steps = %#v, want %v", child.Steps, test.wantStepIDs)
|
||||
}
|
||||
for i, want := range test.wantStepIDs {
|
||||
if child.Steps[i].ID != want {
|
||||
t.Fatalf("child step %d = %q, want %q", i, child.Steps[i].ID, want)
|
||||
}
|
||||
}
|
||||
if test.childSteps == "" && child.Steps[0].Artifacts["score"].Extract.Module != "extract" {
|
||||
t.Fatalf("inherited first artifact = %#v", child.Steps[0].Artifacts["score"])
|
||||
}
|
||||
|
||||
for _, id := range []string{"parent", "child"} {
|
||||
resolved, err := loaded.Config.Resolve(ResolveInput{PipelineID: id, Catalog: effectiveCatalog(t)})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve(%q) error = %v", id, err)
|
||||
}
|
||||
if resolved.ResolvedPipeline.ID != id {
|
||||
t.Fatalf("resolved pipeline ID = %q, want %q", resolved.ResolvedPipeline.ID, id)
|
||||
}
|
||||
wantResolvedSteps := len(test.wantStepIDs)
|
||||
if id == "parent" {
|
||||
wantResolvedSteps = 2
|
||||
}
|
||||
if len(resolved.ResolvedPipeline.Steps) != wantResolvedSteps {
|
||||
t.Fatalf("resolved %q steps = %#v", id, resolved.ResolvedPipeline.Steps)
|
||||
}
|
||||
if id == "child" && resolved.ResolvedPipeline.Steps[0].ArtifactLanes[0].Extract.LLMProfile != "local" {
|
||||
t.Fatalf("resolved child profile = %q", resolved.ResolvedPipeline.Steps[0].ArtifactLanes[0].Extract.LLMProfile)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestArtifactLaneInheritanceUsesSiblingMappingComposition(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
loaded, err := loadConfigSource(configkit.Bytes("lanes.yml", []byte(`version: 5
|
||||
pipelines:
|
||||
session:
|
||||
input: {module: input}
|
||||
artifacts:
|
||||
base:
|
||||
extract: {module: extract, retries: 2}
|
||||
derived:
|
||||
extends: base
|
||||
extract: {retries: 4}
|
||||
`)), LoadOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("loadConfigSource() error = %v", err)
|
||||
}
|
||||
|
||||
lanes := loaded.Config.Pipelines["session"].Artifacts
|
||||
if lanes["base"].Extract.Module != "extract" || lanes["base"].Extract.Retries != 2 {
|
||||
t.Fatalf("base lane = %#v", lanes["base"])
|
||||
}
|
||||
if lanes["derived"].Extract.Module != "extract" || lanes["derived"].Extract.Retries != 4 {
|
||||
t.Fatalf("derived lane = %#v", lanes["derived"])
|
||||
}
|
||||
if _, err := loaded.Config.Resolve(ResolveInput{PipelineID: "session", Catalog: effectiveCatalog(t)}); err != nil {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInheritedPresenceOverridesRemainExplicit(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
loaded, err := loadConfigSource(configkit.Bytes("presence.yml", []byte(`version: 5
|
||||
pipelines:
|
||||
parent:
|
||||
input: {module: input}
|
||||
structured_output_repair_attempts: 3
|
||||
chunk:
|
||||
module: generic
|
||||
validators: []
|
||||
artifacts:
|
||||
score:
|
||||
extract:
|
||||
module: extract
|
||||
validators: [{module: configured-a}]
|
||||
child:
|
||||
extends: parent
|
||||
structured_output_repair_attempts: 0
|
||||
artifacts:
|
||||
score:
|
||||
extract:
|
||||
validators: []
|
||||
`)), LoadOptions{})
|
||||
if err != nil {
|
||||
var loadError *configkit.LoadError
|
||||
if errors.As(err, &loadError) {
|
||||
t.Fatalf("loadConfigSource() diagnostics = %#v", loadError.Diagnostics())
|
||||
}
|
||||
t.Fatalf("loadConfigSource() error = %v", err)
|
||||
}
|
||||
|
||||
parent := loaded.Config.Pipelines["parent"]
|
||||
child := loaded.Config.Pipelines["child"]
|
||||
if parent.Chunk.Module != "generic" || !parent.Chunk.Validators.Set || parent.Chunk.Validators.Validators == nil || len(parent.Chunk.Validators.Validators) != 0 {
|
||||
t.Fatalf("parent empty chunk validators = %#v", parent.Chunk)
|
||||
}
|
||||
if !parent.Artifacts["score"].Extract.Validators.Set || len(parent.Artifacts["score"].Extract.Validators.Validators) != 1 ||
|
||||
parent.Artifacts["score"].Extract.Validators.Validators[0].Module != "configured-a" {
|
||||
t.Fatalf("parent extraction validators = %#v", parent.Artifacts["score"].Extract.Validators)
|
||||
}
|
||||
if parent.StructuredOutputRepairAttempts == nil || *parent.StructuredOutputRepairAttempts != 3 {
|
||||
t.Fatalf("parent repair budget = %#v", parent.StructuredOutputRepairAttempts)
|
||||
}
|
||||
if child.StructuredOutputRepairAttempts == nil || *child.StructuredOutputRepairAttempts != 0 {
|
||||
t.Fatalf("child zero repair budget = %#v", child.StructuredOutputRepairAttempts)
|
||||
}
|
||||
extract := child.Artifacts["score"].Extract
|
||||
if !extract.Validators.Set || extract.Validators.Validators == nil || len(extract.Validators.Validators) != 0 {
|
||||
t.Fatalf("child empty extraction validators = %#v", extract.Validators)
|
||||
}
|
||||
if !child.Chunk.Validators.Set || child.Chunk.Validators.Validators == nil || len(child.Chunk.Validators.Validators) != 0 {
|
||||
t.Fatalf("inherited empty chunk validators = %#v", child.Chunk.Validators)
|
||||
}
|
||||
|
||||
resolved, err := loaded.Config.Resolve(ResolveInput{PipelineID: "child", Catalog: effectiveCatalog(t)})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve(child) error = %v", err)
|
||||
}
|
||||
lanes := resolved.ResolvedPipeline.AllArtifactLanes()
|
||||
if len(lanes) != 1 || lanes[0].Extract.StructuredOutputRepairAttempts == nil || *lanes[0].Extract.StructuredOutputRepairAttempts != 0 {
|
||||
t.Fatalf("resolved child repair budget = %#v", lanes)
|
||||
}
|
||||
if !resolved.ResolvedPipeline.Chunk.Validators.Set || len(resolved.ResolvedPipeline.Chunk.Validators.Validators) != 0 ||
|
||||
!lanes[0].Extract.Validators.Set || len(lanes[0].Extract.Validators.Validators) != 0 {
|
||||
t.Fatalf("resolved empty validator overrides = chunk %#v, extraction %#v", resolved.ResolvedPipeline.Chunk.Validators, lanes[0].Extract.Validators)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInheritedFalseOptionRetainsExplicitMetadata(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
loaded, err := loadConfigSource(configkit.Bytes("false-option.yml", []byte(`version: 5
|
||||
pipelines:
|
||||
parent:
|
||||
input: {module: input}
|
||||
output:
|
||||
module: json
|
||||
options:
|
||||
evidence_context: {enabled: false}
|
||||
child:
|
||||
extends: parent
|
||||
`)), LoadOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("loadConfigSource() error = %v", err)
|
||||
}
|
||||
options := loaded.Config.Pipelines["child"].Output.Options["evidence_context"].(map[string]any)
|
||||
if enabled, ok := options["enabled"].(bool); !ok || enabled {
|
||||
t.Fatalf("inherited evidence_context.enabled = %#v", options["enabled"])
|
||||
}
|
||||
path := configkit.Keys("pipelines", "child", "output", "options", "evidence_context", "enabled")
|
||||
info, ok := loaded.result.Metadata().Lookup(path)
|
||||
if !ok || !info.Explicit || info.Null || len(info.Origin.Inheritance) == 0 {
|
||||
t.Fatalf("inherited false metadata = %#v, found=%v", info, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInheritedAndDirectNullRepairBudgetsRemainInvalid(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
source string
|
||||
path string
|
||||
}{
|
||||
{
|
||||
name: "child null",
|
||||
source: `version: 5
|
||||
pipelines:
|
||||
parent: {input: {module: input}, structured_output_repair_attempts: 2}
|
||||
child:
|
||||
extends: parent
|
||||
structured_output_repair_attempts: null
|
||||
`,
|
||||
path: "/pipelines/child/structured_output_repair_attempts",
|
||||
},
|
||||
{
|
||||
name: "inherited null",
|
||||
source: `version: 5
|
||||
pipelines:
|
||||
parent: {input: {module: input}, structured_output_repair_attempts: null}
|
||||
child: {extends: parent}
|
||||
`,
|
||||
path: "/pipelines/parent/structured_output_repair_attempts",
|
||||
},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
_, err := loadConfigSource(configkit.Bytes("null.yml", []byte(test.source)), LoadOptions{})
|
||||
assertDocumentDiagnosticAt(t, err, test.path)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestOverlayInheritancePrecedesOverrides(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
root := t.TempDir()
|
||||
overlays := filepath.Join(root, "overlays")
|
||||
if err := os.MkdirAll(overlays, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
main := filepath.Join(root, "config.yml")
|
||||
if err := os.WriteFile(main, []byte(`version: 5
|
||||
pipelines:
|
||||
parent:
|
||||
input: {module: input}
|
||||
structured_output_repair_attempts: 1
|
||||
inherited:
|
||||
extends: parent
|
||||
overridden:
|
||||
extends: parent
|
||||
structured_output_repair_attempts: 0
|
||||
cancelled:
|
||||
extends: parent
|
||||
input: {module: input}
|
||||
`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(overlays, "development.yml"), []byte(`pipelines:
|
||||
parent:
|
||||
structured_output_repair_attempts: 2
|
||||
cancelled:
|
||||
extends: null
|
||||
`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
loaded, err := Load(main, LoadOptions{Overlay: "development"})
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
if got := *loaded.Config.Pipelines["inherited"].StructuredOutputRepairAttempts; got != 2 {
|
||||
t.Fatalf("overlay-inherited repair budget = %d, want 2", got)
|
||||
}
|
||||
if got := *loaded.Config.Pipelines["overridden"].StructuredOutputRepairAttempts; got != 0 {
|
||||
t.Fatalf("child replacement repair budget = %d, want 0", got)
|
||||
}
|
||||
if loaded.Config.Pipelines["cancelled"].StructuredOutputRepairAttempts != nil {
|
||||
t.Fatalf("cancelled child inherited parent settings: %#v", loaded.Config.Pipelines["cancelled"])
|
||||
}
|
||||
|
||||
withOverride, err := Load(main, LoadOptions{
|
||||
Overlay: "development",
|
||||
Overrides: []configkit.Override{{
|
||||
Path: configkit.Keys("pipelines", "parent", "structured_output_repair_attempts"),
|
||||
Value: 3,
|
||||
Flag: "--repair-budget",
|
||||
}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Load() with explicit override error = %v", err)
|
||||
}
|
||||
if got := *withOverride.Config.Pipelines["inherited"].StructuredOutputRepairAttempts; got != 2 {
|
||||
t.Fatalf("inherited repair budget after parent override = %d, want 2", got)
|
||||
}
|
||||
if got := *withOverride.Config.Pipelines["parent"].StructuredOutputRepairAttempts; got != 3 {
|
||||
t.Fatalf("parent override repair budget = %d, want 3", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInheritedReferencePathsKeepDeclaringSource(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
root := t.TempDir()
|
||||
parentDirectory := filepath.Join(root, "parents")
|
||||
childDirectory := filepath.Join(root, "children")
|
||||
overlayDirectory := filepath.Join(root, "overlays")
|
||||
fixtureDirectory := filepath.Join(root, "fixtures")
|
||||
for _, directory := range []string{parentDirectory, childDirectory, overlayDirectory, fixtureDirectory} {
|
||||
if err := os.MkdirAll(directory, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, fixture := range []struct{ directory, name string }{
|
||||
{parentDirectory, "shared.txt"},
|
||||
{parentDirectory, "parent.txt"},
|
||||
{parentDirectory, "step.txt"},
|
||||
{childDirectory, "child.txt"},
|
||||
{overlayDirectory, "overlay.txt"},
|
||||
{fixtureDirectory, "outside.txt"},
|
||||
} {
|
||||
if err := os.WriteFile(filepath.Join(fixture.directory, fixture.name), []byte("fixture\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
parent := filepath.Join(parentDirectory, "pipelines.yml")
|
||||
if err := os.WriteFile(parent, []byte(`pipelines:
|
||||
" parent ":
|
||||
input: {module: input}
|
||||
references:
|
||||
" shared ": {path: ./shared.txt}
|
||||
nested:
|
||||
extends: " shared "
|
||||
source: {path: ./parent.txt}
|
||||
outside: {path: ../fixtures/outside.txt}
|
||||
steps:
|
||||
- id: first
|
||||
references:
|
||||
step-source: {path: ./step.txt}
|
||||
artifacts:
|
||||
summary:
|
||||
extract: {module: extract}
|
||||
`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
child := filepath.Join(childDirectory, "pipelines.yml")
|
||||
if err := os.WriteFile(child, []byte(`pipelines:
|
||||
child:
|
||||
extends: " parent "
|
||||
references:
|
||||
source: {path: ./child.txt}
|
||||
`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
main := filepath.Join(root, "notarius.yml")
|
||||
if err := os.WriteFile(main, []byte(`version: 5
|
||||
config:
|
||||
imports:
|
||||
- path: parents/pipelines.yml
|
||||
- path: children/pipelines.yml
|
||||
`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(overlayDirectory, "development.yml"), []byte(`pipelines:
|
||||
" parent ":
|
||||
references:
|
||||
" shared ": {path: ./overlay.txt}
|
||||
`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
loaded, err := loadConfig(main, LoadOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("loadConfig() error = %v", err)
|
||||
}
|
||||
assertReferencePath := func(pipelineID, referenceID, want string) {
|
||||
t.Helper()
|
||||
if got := loaded.Config.Pipelines[pipelineID].References[referenceID].Path; got != want {
|
||||
t.Errorf("%s reference %s path = %q, want %q", pipelineID, referenceID, got, want)
|
||||
}
|
||||
}
|
||||
assertReferencePath("parent", "shared", filepath.Join(parentDirectory, "shared.txt"))
|
||||
assertReferencePath("child", "shared", filepath.Join(parentDirectory, "shared.txt"))
|
||||
assertReferencePath("child", "nested", filepath.Join(parentDirectory, "shared.txt"))
|
||||
assertReferencePath("parent", "source", filepath.Join(parentDirectory, "parent.txt"))
|
||||
assertReferencePath("child", "source", filepath.Join(childDirectory, "child.txt"))
|
||||
// Keep ../ coverage separate from paths that distinguish the declaring file.
|
||||
assertReferencePath("child", "outside", filepath.Join(fixtureDirectory, "outside.txt"))
|
||||
if got := loaded.Config.Pipelines["child"].Steps[0].References["step-source"].Path; got != filepath.Join(parentDirectory, "step.txt") {
|
||||
t.Errorf("inherited step reference path = %q", got)
|
||||
}
|
||||
|
||||
overlaid, err := loadConfig(main, LoadOptions{Overlay: "development"})
|
||||
if err != nil {
|
||||
t.Fatalf("loadConfig() with overlay error = %v", err)
|
||||
}
|
||||
for _, pipelineID := range []string{"parent", "child"} {
|
||||
for _, referenceID := range []string{"shared", "nested"} {
|
||||
if got, want := overlaid.Config.Pipelines[pipelineID].References[referenceID].Path, filepath.Join(overlayDirectory, "overlay.txt"); got != want {
|
||||
t.Errorf("overlaid %s reference %s path = %q, want %q", pipelineID, referenceID, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
if got, want := overlaid.Config.Pipelines["child"].References["source"].Path, filepath.Join(childDirectory, "child.txt"); got != want {
|
||||
t.Errorf("child replacement path = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := overlaid.Config.Pipelines["child"].Steps[0].References["step-source"].Path, filepath.Join(parentDirectory, "step.txt"); got != want {
|
||||
t.Errorf("overlaid inherited step reference path = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInheritedInspectionRetainsOriginsRedactionAndOwnership(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
root := t.TempDir()
|
||||
parentDirectory := filepath.Join(root, "parents")
|
||||
childDirectory := filepath.Join(root, "children")
|
||||
overlayDirectory := filepath.Join(root, "overlays")
|
||||
for _, directory := range []string{parentDirectory, childDirectory, overlayDirectory} {
|
||||
if err := os.MkdirAll(directory, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
parentFile := filepath.Join(parentDirectory, "pipelines.yml")
|
||||
if err := os.WriteFile(parentFile, []byte(`pipelines:
|
||||
parent:
|
||||
input: {module: input}
|
||||
structured_output_repair_attempts: 2
|
||||
references:
|
||||
source: {path: ./shared.txt}
|
||||
artifacts:
|
||||
summary:
|
||||
extract:
|
||||
module: extract
|
||||
options:
|
||||
nested: {value: original}
|
||||
credentials: {api_key: fake-parent-secret}
|
||||
`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(parentDirectory, "shared.txt"), []byte("source\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
childFile := filepath.Join(childDirectory, "pipelines.yml")
|
||||
if err := os.WriteFile(childFile, []byte(`pipelines:
|
||||
child:
|
||||
extends: parent
|
||||
structured_output_repair_attempts: 0
|
||||
`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
main := filepath.Join(root, "notarius.yml")
|
||||
if err := os.WriteFile(main, []byte(`version: 5
|
||||
config:
|
||||
imports:
|
||||
- path: parents/pipelines.yml
|
||||
- path: children/pipelines.yml
|
||||
`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(overlayDirectory, "development.yml"), []byte(`pipelines:
|
||||
parent:
|
||||
artifacts:
|
||||
summary:
|
||||
extract:
|
||||
options:
|
||||
credentials: {api_key: fake-overlay-secret}
|
||||
`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
loaded, err := Load(main, LoadOptions{Overlay: "development"})
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
child := loaded.Config.Pipelines["child"]
|
||||
wantPath := filepath.Join(parentDirectory, "shared.txt")
|
||||
if got := child.References["source"].Path; got != wantPath {
|
||||
t.Fatalf("runtime inherited path = %q, want %q", got, wantPath)
|
||||
}
|
||||
if child.StructuredOutputRepairAttempts == nil || *child.StructuredOutputRepairAttempts != 0 {
|
||||
t.Fatalf("runtime inherited repair setting = %#v", child.StructuredOutputRepairAttempts)
|
||||
}
|
||||
|
||||
var effective bytes.Buffer
|
||||
if err := loaded.WriteEffective(&effective); err != nil {
|
||||
t.Fatalf("WriteEffective() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(effective.String(), wantPath) || strings.Contains(effective.String(), "extends:") ||
|
||||
!strings.Contains(effective.String(), "structured_output_repair_attempts: 0") {
|
||||
t.Fatalf("effective configuration is inconsistent with runtime conversion: %s", effective.String())
|
||||
}
|
||||
for _, secret := range []string{"fake-parent-secret", "fake-overlay-secret"} {
|
||||
if strings.Contains(effective.String(), secret) {
|
||||
t.Fatalf("effective configuration exposed %q", secret)
|
||||
}
|
||||
}
|
||||
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
path configkit.Path
|
||||
}{
|
||||
{name: "repair setting", path: configkit.Keys("pipelines", "child", "structured_output_repair_attempts")},
|
||||
{name: "normalized reference", path: configkit.Keys("pipelines", "child", "references", "source", "path")},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
explanation, ok := loaded.result.Explain(test.path)
|
||||
if !ok || len(explanation.History) == 0 {
|
||||
t.Fatalf("inherited explanation = %#v, found=%v", explanation, ok)
|
||||
}
|
||||
inheritanceFromParent := false
|
||||
for _, change := range explanation.History {
|
||||
if len(change.Origin.Inheritance) == 0 {
|
||||
continue
|
||||
}
|
||||
for _, location := range change.Origin.Locations {
|
||||
for _, source := range loaded.result.Sources() {
|
||||
if source.ID == location.SourceID && source.Path == parentFile {
|
||||
inheritanceFromParent = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if !inheritanceFromParent {
|
||||
t.Fatalf("inheritance history lacks parent source evidence: %#v", explanation.History)
|
||||
}
|
||||
var rendered bytes.Buffer
|
||||
if err := loaded.WriteExplanation(&rendered, test.path.String()); err != nil {
|
||||
t.Fatalf("WriteExplanation() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(rendered.String(), filepath.ToSlash(filepath.Join("parents", "pipelines.yml"))) {
|
||||
t.Fatalf("explanation lacks parent source evidence: %s", rendered.String())
|
||||
}
|
||||
if test.name == "normalized reference" {
|
||||
foundNormalization := false
|
||||
for _, change := range explanation.History {
|
||||
if change.Origin.Operation == configkit.ChangeNormalization {
|
||||
foundNormalization = true
|
||||
}
|
||||
}
|
||||
if !foundNormalization || !strings.Contains(rendered.String(), wantPath) {
|
||||
t.Fatalf("normalized path evidence = %#v; rendered %s", explanation.History, rendered.String())
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
secretPath := configkit.Keys("pipelines", "child", "artifacts", "summary", "extract", "options", "credentials", "api_key")
|
||||
secretExplanation, ok := loaded.result.Explain(secretPath)
|
||||
if !ok || !secretExplanation.Effective.Redacted {
|
||||
t.Fatalf("inherited secret explanation = %#v, found=%v", secretExplanation, ok)
|
||||
}
|
||||
redactedHistory := false
|
||||
for _, change := range secretExplanation.History {
|
||||
redactedHistory = redactedHistory || change.Before.Redacted || change.After.Redacted
|
||||
}
|
||||
if !redactedHistory {
|
||||
t.Fatalf("secret history was not redacted: %#v", secretExplanation.History)
|
||||
}
|
||||
var secretOutput bytes.Buffer
|
||||
if err := loaded.WriteExplanation(&secretOutput, secretPath.String()); err != nil {
|
||||
t.Fatalf("WriteExplanation(secret) error = %v", err)
|
||||
}
|
||||
if strings.Contains(secretOutput.String(), "fake-parent-secret") || strings.Contains(secretOutput.String(), "fake-overlay-secret") ||
|
||||
!strings.Contains(secretOutput.String(), "[REDACTED]") {
|
||||
t.Fatalf("secret explanation did not redact values and history: %s", secretOutput.String())
|
||||
}
|
||||
|
||||
childOptions := loaded.Config.Pipelines["child"].Artifacts["summary"].Extract.Options
|
||||
childOptions["nested"].(map[string]any)["value"] = "changed"
|
||||
parentValue := loaded.Config.Pipelines["parent"].Artifacts["summary"].Extract.Options["nested"].(map[string]any)["value"]
|
||||
if parentValue != "original" {
|
||||
t.Fatalf("child runtime mutation changed parent value: %#v", parentValue)
|
||||
}
|
||||
var frozen bytes.Buffer
|
||||
if err := loaded.WriteEffective(&frozen); err != nil {
|
||||
t.Fatalf("WriteEffective() after runtime mutation error = %v", err)
|
||||
}
|
||||
if !strings.Contains(frozen.String(), "original") || strings.Contains(frozen.String(), "changed") {
|
||||
t.Fatalf("runtime mutation changed frozen inspection: %s", frozen.String())
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,7 @@ package config
|
||||
import (
|
||||
"path/filepath"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
configkit "go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
// LoadOptions contains configuration selections supplied by an application
|
||||
@@ -57,7 +57,14 @@ func documentDefinition(state *documentLoadState) configkit.Definition[Document]
|
||||
Path: configkit.Keys("version"),
|
||||
Check: validateDocumentVersion,
|
||||
},
|
||||
Secret: sensitiveDocumentPath,
|
||||
Sensitive: sensitiveDocumentPath,
|
||||
SequenceIdentities: []configkit.SequenceIdentity{
|
||||
configkit.SequenceIdentityMatching(configkit.SequenceSelector{
|
||||
configkit.SequenceLiteralKey("pipelines"),
|
||||
configkit.SequenceAnyMappingKey(),
|
||||
configkit.SequenceLiteralKey("steps"),
|
||||
}, "id"),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -4,8 +4,8 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
configkit "go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
// LoadedConfig keeps the finalized runtime configuration together with the
|
||||
@@ -59,6 +59,28 @@ func Load(path string, options LoadOptions) (*LoadedConfig, error) {
|
||||
return loadConfig(path, options)
|
||||
}
|
||||
|
||||
// LoadSource loads a selected source, including its imports, into an owned configuration.
|
||||
func LoadSource(source configkit.Source, options LoadOptions) (*LoadedConfig, error) {
|
||||
result, err := loadDocumentWithSource(source, options, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return newLoadedConfig(result)
|
||||
}
|
||||
|
||||
// MainSourcePath returns the selected main file's recorded path, if any.
|
||||
func (loaded *LoadedConfig) MainSourcePath() string {
|
||||
if loaded == nil || loaded.result == nil {
|
||||
return ""
|
||||
}
|
||||
for _, source := range loaded.result.Sources() {
|
||||
if source.Layer == configkit.LayerMain && source.Kind == configkit.SourceKindFile && source.ImportedFrom == nil {
|
||||
return source.Path
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func loadConfigSource(source configkit.Source, options LoadOptions) (*LoadedConfig, error) {
|
||||
result, err := loadDocumentSource(source, options)
|
||||
if err != nil {
|
||||
|
||||
@@ -2,16 +2,47 @@ package config
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
units "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/chunk/units"
|
||||
jsonoutput "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/output/json"
|
||||
configkit "go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
func TestLoadSourceRetainsMainFileAndLoadsImports(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
main := filepath.Join(dir, "main.yml")
|
||||
fragment := filepath.Join(dir, "fragment.yml")
|
||||
if err := os.WriteFile(fragment, []byte("output:\n directory: imported-output\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(main, []byte("version: 5\nconfig:\n imports:\n - path: fragment.yml\n required: true\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
loaded, err := LoadSource(configkit.File(main), LoadOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := loaded.MainSourcePath(); got != main {
|
||||
t.Fatalf("main source path = %q, want %q", got, main)
|
||||
}
|
||||
if got := loaded.Config.Output.Directory; got != filepath.Join(dir, "imported-output") {
|
||||
t.Fatalf("imported output directory = %q", got)
|
||||
}
|
||||
bytesLoaded, err := loadConfigSource(configkit.Bytes("memory.yml", []byte("version: 5\n")), LoadOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := bytesLoaded.MainSourcePath(); got != "" {
|
||||
t.Fatalf("non-file main source path = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadedConfigConvertsDocumentsAndResolvesRuntimePipeline(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -166,6 +197,33 @@ pipelines:
|
||||
}
|
||||
}
|
||||
|
||||
func TestNestedConfigOptionRemainsApplicationData(t *testing.T) {
|
||||
loaded, err := loadConfigSource(configkit.Bytes("options.yml", []byte(`version: 5
|
||||
pipelines:
|
||||
session:
|
||||
input:
|
||||
module: input
|
||||
options:
|
||||
config: {imports: ordinary-data}
|
||||
`)), LoadOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
value := loaded.Config.Pipelines["session"].Input.Options["config"].(map[string]any)["imports"]
|
||||
if value != "ordinary-data" {
|
||||
t.Fatalf("nested config option = %#v", value)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOverrideCannotRepairTypedDecodeFailure(t *testing.T) {
|
||||
_, err := loadConfigSource(configkit.Bytes("invalid.yml", []byte("version: 5\nconcurrency: {total_llm: invalid}\n")), LoadOptions{
|
||||
Overrides: []configkit.Override{{Path: configkit.Keys("concurrency", "total_llm"), Value: 2, Flag: "--total-llm"}},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("override unexpectedly repaired invalid typed input")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadedConfigRedactsSensitiveNestedOptions(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -6,8 +6,8 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
configkit "go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
func resolveDocumentOperations(document *Document, resolution *configkit.Resolution, state *documentLoadState) error {
|
||||
@@ -16,8 +16,8 @@ func resolveDocumentOperations(document *Document, resolution *configkit.Resolut
|
||||
}
|
||||
normalizeDocumentPromptKit(document.PromptKit, resolution)
|
||||
normalizeDocumentState(document, resolution)
|
||||
normalizeDocumentPipelines(document, resolution, state)
|
||||
normalizeDocumentPaths(document, resolution, state)
|
||||
normalizeDocumentPipelines(document, resolution, state)
|
||||
if document.Concurrency != nil {
|
||||
document.Concurrency.StageWorkers = normalizeDocumentMap(document.Concurrency.StageWorkers,
|
||||
configkit.Keys("concurrency", "stage_workers"), "stage worker identifier",
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
configkit "go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
func TestWorkerKeyNormalizationPreservesExplicitCount(t *testing.T) {
|
||||
|
||||
@@ -4,15 +4,14 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
configkit "go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
// documentLoadState keeps the spelling supplied by the user while resolver
|
||||
// operations move map entries to their canonical keys. It is intentionally
|
||||
// load-local: source-location handling consumes it during a later resolver
|
||||
// pass and it never becomes runtime configuration state.
|
||||
// documentLoadState keeps source-directory lookups and original map-key
|
||||
// spellings local to one load. It never becomes runtime configuration state.
|
||||
type documentLoadState struct {
|
||||
sourceDirectories map[string]string
|
||||
originalPaths map[string]configkit.Path
|
||||
mainDirectory string
|
||||
invocationDirectory string
|
||||
@@ -65,7 +64,6 @@ func normalizeDocumentPipelines(document *Document, resolution *configkit.Resolu
|
||||
}
|
||||
validateDocumentPipelinePresence(document, resolution)
|
||||
normalizer := documentNormalizer{Resolution: resolution}
|
||||
|
||||
pipelinesPath := configkit.Keys("pipelines")
|
||||
for _, id := range sortedPipelineDocumentKeys(document.Pipelines) {
|
||||
profile := document.Pipelines[id]
|
||||
@@ -192,18 +190,19 @@ func normalizePipelineDocument(profile *PipelineDocument, path configkit.Path, r
|
||||
profile.Artifacts[laneID] = lane
|
||||
}
|
||||
profile.Artifacts = normalizeDocumentMap(profile.Artifacts, path.Key("artifacts"), "artifact lane identifier", resolution, state)
|
||||
resolution = resolution.inSequence(path.Key("steps"))
|
||||
for index := range profile.Steps {
|
||||
step := &profile.Steps[index]
|
||||
stepPath := path.Key("steps").Index(index)
|
||||
normalizeDocumentValue(resolution, stepPath.Key("id"), &step.ID, strings.TrimSpace(step.ID), "trimmed step identifier")
|
||||
step.References = normalizeDocumentReferences(step.References, stepPath.Key("references"), resolution, state)
|
||||
stepNormalizer := resolution
|
||||
stepNormalizer.covered = step.ID != strings.TrimSpace(step.ID)
|
||||
normalizeDocumentValue(stepNormalizer, stepPath.Key("id"), &step.ID, strings.TrimSpace(step.ID), "trimmed step identifier")
|
||||
step.References = normalizeDocumentReferences(step.References, stepPath.Key("references"), stepNormalizer, state)
|
||||
for _, laneID := range sortedLaneDocumentKeys(step.Artifacts) {
|
||||
lane := step.Artifacts[laneID]
|
||||
normalizeArtifactLaneDocument(&lane, stepPath.Key("artifacts").Key(laneID), resolution, state)
|
||||
normalizeArtifactLaneDocument(&lane, stepPath.Key("artifacts").Key(laneID), stepNormalizer, state)
|
||||
step.Artifacts[laneID] = lane
|
||||
}
|
||||
step.Artifacts = normalizeDocumentMap(step.Artifacts, stepPath.Key("artifacts"), "artifact lane identifier", resolution, state)
|
||||
step.Artifacts = normalizeDocumentMap(step.Artifacts, stepPath.Key("artifacts"), "artifact lane identifier", stepNormalizer, state)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -215,7 +214,6 @@ func normalizeArtifactLaneDocument(lane *ArtifactLaneDocument, path configkit.Pa
|
||||
if lane.Normalize != nil {
|
||||
normalizeDocumentBinding(lane.Normalize, path.Key("normalize"), resolution, state)
|
||||
}
|
||||
resolution = resolution.inSequence(path.Key("validators"))
|
||||
for index := range lane.Validators {
|
||||
normalizeValidatorDocument(&lane.Validators[index], path.Key("validators").Index(index), resolution)
|
||||
}
|
||||
@@ -225,7 +223,6 @@ func normalizeDocumentBinding(binding *BindingDocument, path configkit.Path, res
|
||||
normalizeDocumentValue(resolution, path.Key("module"), &binding.Module, strings.TrimSpace(binding.Module), "trimmed module identifier")
|
||||
normalizePipelineDocumentString(resolution, path.Key("llm_profile"), &binding.LLMProfile, "trimmed binding LLM profile")
|
||||
binding.References = normalizeDocumentReferences(binding.References, path.Key("references"), resolution, state)
|
||||
resolution = resolution.inSequence(path.Key("validators"))
|
||||
for index := range binding.Validators {
|
||||
normalizeValidatorDocument(&binding.Validators[index], path.Key("validators").Index(index), resolution)
|
||||
}
|
||||
@@ -255,7 +252,7 @@ func normalizeDocumentValue[T comparable](resolution documentNormalizer, path co
|
||||
return
|
||||
}
|
||||
if *target != value {
|
||||
resolution.Record(path, configkit.ChangeNormalization, reason)
|
||||
resolution.RecordNormalization(path, reason)
|
||||
*target = value
|
||||
}
|
||||
}
|
||||
@@ -266,7 +263,7 @@ func normalizePipelineDocumentString(resolution documentNormalizer, path configk
|
||||
}
|
||||
value := strings.TrimSpace(**target)
|
||||
if **target != value {
|
||||
resolution.Record(path, configkit.ChangeNormalization, reason)
|
||||
resolution.RecordNormalization(path, reason)
|
||||
**target = value
|
||||
}
|
||||
}
|
||||
@@ -312,7 +309,7 @@ func normalizeDocumentMap[T any](values map[string]T, path configkit.Path, kind
|
||||
}
|
||||
}
|
||||
if changed {
|
||||
resolution.Record(path, configkit.ChangeNormalization, "normalized "+kind+" keys", dependencies...)
|
||||
resolution.RecordNormalization(path, "normalized "+kind+" keys", dependencies...)
|
||||
values = result
|
||||
}
|
||||
return values
|
||||
|
||||
@@ -5,7 +5,7 @@ import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
configkit "go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
func TestDocumentRejectsExplicitInvalidBranches(t *testing.T) {
|
||||
|
||||
@@ -4,8 +4,8 @@ import (
|
||||
"reflect"
|
||||
"strings"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
configkit "go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
func (e EffectiveConfig) RedactedResolvedPipelinePayload() pipeline.ResolvedPipeline {
|
||||
|
||||
@@ -83,7 +83,7 @@ func TestSequenceNormalizationPreservesRuntimeAndProvenance(t *testing.T) {
|
||||
}
|
||||
if !strings.Contains(effective.String(), test.want) ||
|
||||
!strings.Contains(explanation.String(), test.want) ||
|
||||
!strings.Contains(explanation.String(), path) ||
|
||||
(test.name != "step identifier" && !strings.Contains(explanation.String(), path)) ||
|
||||
!strings.Contains(explanation.String(), "normalization") {
|
||||
t.Fatalf("missing effective value or provenance:\n%s\n%s", effective.String(), explanation.String())
|
||||
}
|
||||
|
||||
@@ -4,8 +4,8 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
configkit "go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
func TestValidationPolicyFileConfigurationIsStrictAndPresenceAware(t *testing.T) {
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/readonlyfs"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
@@ -57,8 +58,8 @@ func (r *AssetRegistry) RegisterSchemaFS(fsys fs.FS, root string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// RegisterFallbackProfileFS registers profile assets that PromptKit uses only
|
||||
// when an operator-configured source does not provide a matching profile.
|
||||
// RegisterFallbackProfileFS registers lower-precedence application profiles.
|
||||
// Operator entries recursively overlay them, retaining omitted fields.
|
||||
func (r *AssetRegistry) RegisterFallbackProfileFS(fsys fs.FS, root string) error {
|
||||
if r == nil {
|
||||
return fmt.Errorf("asset registry must not be nil")
|
||||
@@ -99,43 +100,45 @@ func (r *AssetRegistry) FallbackProfileDigest() (string, error) {
|
||||
return digest, err
|
||||
}
|
||||
|
||||
func (r *AssetRegistry) PromptKitOptions() ([]promptkit.Option, error) {
|
||||
options, _, err := r.promptKitOptions()
|
||||
return options, err
|
||||
}
|
||||
|
||||
func (r *AssetRegistry) promptKitOptions() ([]promptkit.Option, string, error) {
|
||||
promptFS, err := r.PromptFS()
|
||||
// PromptKitConfigurationSources returns the registered prompt and fallback
|
||||
// profile assets in one strict native PromptKit underlay. Schema assets remain a
|
||||
// separate PromptKit option because they are runtime validation resources.
|
||||
func (r *AssetRegistry) PromptKitConfigurationSources() (promptkit.ConfigurationSources, []promptkit.Option, error) {
|
||||
if r == nil {
|
||||
return promptkit.ConfigurationSources{}, nil, fmt.Errorf("asset registry must not be nil")
|
||||
}
|
||||
inputs, err := promptKitFilesystemInputs(r.prompts, "notarius-prompts")
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("prepare prompt assets: %w", err)
|
||||
return promptkit.ConfigurationSources{}, nil, fmt.Errorf("prepare prompt configuration sources: %w", err)
|
||||
}
|
||||
profiles, err := promptKitFilesystemInputs(r.fallbackProfiles, "notarius-fallback-profiles")
|
||||
if err != nil {
|
||||
return promptkit.ConfigurationSources{}, nil, fmt.Errorf("prepare fallback profile configuration sources: %w", err)
|
||||
}
|
||||
inputs = append(inputs, profiles...)
|
||||
var underlays []configkit.Underlay
|
||||
if len(inputs) != 0 {
|
||||
underlays = []configkit.Underlay{{Inputs: inputs}}
|
||||
}
|
||||
schemaFS, err := r.SchemaFS()
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("prepare schema assets: %w", err)
|
||||
return promptkit.ConfigurationSources{}, nil, fmt.Errorf("prepare schema assets: %w", err)
|
||||
}
|
||||
options := []promptkit.Option{
|
||||
promptkit.WithPromptFS(promptFS, "."),
|
||||
return promptkit.ConfigurationSources{Underlays: underlays}, []promptkit.Option{
|
||||
promptkit.WithSchemaFS(schemaFS, "."),
|
||||
}
|
||||
fallbackFS, fallbackDigest, hasFallback, err := r.fallbackProfileAssets()
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if hasFallback {
|
||||
options = append(options, promptkit.WithFallbackProfileFS(fallbackFS, "."))
|
||||
}
|
||||
return options, fallbackDigest, nil
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *AssetRegistry) promptKitFallbackProfileOption() (promptkit.Option, bool, error) {
|
||||
fallbackFS, _, hasFallback, err := r.fallbackProfileAssets()
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
func promptKitFilesystemInputs(sources []AssetSource, label string) ([]configkit.Input, error) {
|
||||
var selected []configkit.Input
|
||||
for index, source := range sources {
|
||||
inputs, err := promptkit.FilesystemInputs(fmt.Sprintf("%s-%d", label, index), source.FS, source.Root)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("select filesystem inputs: %w", err)
|
||||
}
|
||||
selected = append(selected, inputs...)
|
||||
}
|
||||
if !hasFallback {
|
||||
return nil, false, nil
|
||||
}
|
||||
return promptkit.WithFallbackProfileFS(fallbackFS, "."), true, nil
|
||||
return selected, nil
|
||||
}
|
||||
|
||||
func (r *AssetRegistry) fallbackProfileAssets() (fs.FS, string, bool, error) {
|
||||
|
||||
53
internal/framework/llm/asset_registry_sources_test.go
Normal file
53
internal/framework/llm/asset_registry_sources_test.go
Normal file
@@ -0,0 +1,53 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
)
|
||||
|
||||
func TestApplicationAssetsComposeStrictlyBeforeOperatorOverlay(t *testing.T) {
|
||||
for _, conflict := range []bool{false, true} {
|
||||
t.Run(fmt.Sprint("conflict=", conflict), func(t *testing.T) {
|
||||
registry := NewAssetRegistry()
|
||||
for _, id := range []string{"first", "second"} {
|
||||
if conflict {
|
||||
id = "first"
|
||||
}
|
||||
files := fstest.MapFS{"profiles.yaml": {Data: []byte(fmt.Sprintf("profiles:\n %s:\n backend: openrouter\n model: openai/gpt-5.6-luna\n", id))}}
|
||||
if err := registry.RegisterFallbackProfileFS(files, "."); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
operator := filepath.Join(t.TempDir(), "operator.yml")
|
||||
if err := os.WriteFile(operator, []byte("profiles:\n first:\n api_key_required: false\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{Assets: registry, Source: PromptKitProfileSourceConfig{ProfileFile: operator}})
|
||||
if conflict {
|
||||
if err == nil {
|
||||
t.Fatal("conflicting application profiles were silently overlaid")
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, id := range []string{"first", "second"} {
|
||||
profile, err := inspector.InspectProfile(context.Background(), id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if profile.Model != "openai/gpt-5.6-luna" || profile.BackendID != "openrouter" {
|
||||
t.Fatalf("profile = %#v", profile)
|
||||
}
|
||||
if id == "first" && profile.CredentialRequired {
|
||||
t.Fatal("operator overlay was not applied")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -2,12 +2,14 @@ package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io/fs"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
"time"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
@@ -18,7 +20,7 @@ func TestAssetRegistryCombinesPromptAndSchemaSources(t *testing.T) {
|
||||
"prompts/messages/user.tmpl": {Data: []byte(`Input: {{ input "transcript" }}`)},
|
||||
"prompts/messages/task.tmpl": {Data: []byte("Return JSON.")},
|
||||
"schemas/ignored/schema.json": {Data: []byte(`{"type":"object"}`)},
|
||||
}, "prompts")
|
||||
}, ".")
|
||||
mustRegisterSchemaFS(t, registry, fstest.MapFS{
|
||||
"root/schemas/out.json": {Data: []byte(`{"type":"object"}`)},
|
||||
}, "root")
|
||||
@@ -42,7 +44,7 @@ func TestAssetRegistryCombinesPromptAndSchemaSources(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssetRegistryPrepareFailsForMissingPromptAsset(t *testing.T) {
|
||||
func TestAssetRegistryConstructionFailsForMissingPromptAsset(t *testing.T) {
|
||||
registry := NewAssetRegistry()
|
||||
mustRegisterPromptFS(t, registry, fstest.MapFS{
|
||||
"test.yaml": {Data: []byte(validPromptYAML("out.json"))},
|
||||
@@ -51,16 +53,21 @@ func TestAssetRegistryPrepareFailsForMissingPromptAsset(t *testing.T) {
|
||||
"out.json": {Data: []byte(`{"type":"object"}`)},
|
||||
}, ".")
|
||||
|
||||
engine := newPromptKitAssetTestEngine(t, registry)
|
||||
_, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "asset.test",
|
||||
ProfileID: "asset-test-profile",
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.Inline(`{"ok":true}`),
|
||||
},
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "content_file") {
|
||||
t.Fatalf("Prepare() error = %v, want missing content_file error", err)
|
||||
sources, options, err := registry.PromptKitConfigurationSources()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sources.Underlays = append(sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{
|
||||
Source: promptkit.ProfileSource("asset test profile", map[string]promptkit.ProfileConfig{
|
||||
"asset-test-profile": {
|
||||
Endpoint: promptKitTestString("http://127.0.0.1:1/v1"),
|
||||
Model: promptKitTestString("asset-test-model"),
|
||||
},
|
||||
}),
|
||||
}}})
|
||||
_, err = promptkit.NewEngine(promptkit.Config{Sources: sources}, options...)
|
||||
if err == nil || !errors.Is(err, promptkit.ErrInvalidConfig) || !strings.Contains(err.Error(), "messages/user.tmpl") {
|
||||
t.Fatalf("NewEngine() error = %v, want invalid configuration for missing prompt content", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -75,8 +82,23 @@ func TestAssetRegistryPrepareFailsForMissingSchemaAsset(t *testing.T) {
|
||||
"present.json": {Data: []byte(`{"type":"object"}`)},
|
||||
}, ".")
|
||||
|
||||
engine := newPromptKitAssetTestEngine(t, registry)
|
||||
_, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
sources, options, err := registry.PromptKitConfigurationSources()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sources.Underlays = append(sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{
|
||||
Source: promptkit.ProfileSource("asset test profile", map[string]promptkit.ProfileConfig{
|
||||
"asset-test-profile": {
|
||||
Endpoint: promptKitTestString("http://127.0.0.1:1/v1"),
|
||||
Model: promptKitTestString("asset-test-model"),
|
||||
},
|
||||
}),
|
||||
}}})
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: sources}, options...)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine() error = %v, want nil", err)
|
||||
}
|
||||
_, err = engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "asset.test",
|
||||
ProfileID: "asset-test-profile",
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
@@ -84,14 +106,14 @@ func TestAssetRegistryPrepareFailsForMissingSchemaAsset(t *testing.T) {
|
||||
},
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "missing.json") {
|
||||
t.Fatalf("Prepare() error = %v, want missing schema error", err)
|
||||
t.Fatalf("Prepare() error = %v, want missing schema asset failure", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssetRegistryRejectsDuplicateAssetPaths(t *testing.T) {
|
||||
registry := NewAssetRegistry()
|
||||
mustRegisterPromptFS(t, registry, fstest.MapFS{"one/prompt.yaml": {Data: []byte("id: one")}}, "one")
|
||||
mustRegisterPromptFS(t, registry, fstest.MapFS{"two/prompt.yaml": {Data: []byte("id: two")}}, "two")
|
||||
mustRegisterPromptFS(t, registry, fstest.MapFS{"one/prompt.yaml": {Data: []byte("prompts: {}")}}, "one")
|
||||
mustRegisterPromptFS(t, registry, fstest.MapFS{"two/prompt.yaml": {Data: []byte("prompts: {}")}}, "two")
|
||||
|
||||
_, err := registry.PromptFS()
|
||||
if err == nil || !strings.Contains(err.Error(), "duplicate asset path") {
|
||||
@@ -102,12 +124,12 @@ func TestAssetRegistryRejectsDuplicateAssetPaths(t *testing.T) {
|
||||
func TestAssetRegistryCombinesFallbackProfileSources(t *testing.T) {
|
||||
registry := NewAssetRegistry()
|
||||
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{
|
||||
"first/profiles/one.yaml": {Data: []byte("id: one\nmodel: first\n")},
|
||||
"first/profiles/one.yaml": {Data: []byte("profiles:\n one:\n model: first\n")},
|
||||
}, "first/profiles"); err != nil {
|
||||
t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err)
|
||||
}
|
||||
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{
|
||||
"second/two.yaml": {Data: []byte("id: two\nmodel: second\n")},
|
||||
"second/two.yaml": {Data: []byte("profiles:\n two:\n model: second\n")},
|
||||
}, "second"); err != nil {
|
||||
t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err)
|
||||
}
|
||||
@@ -144,10 +166,10 @@ func TestAssetRegistryRejectsUnreadableFallbackProfileAssets(t *testing.T) {
|
||||
|
||||
func TestAssetRegistryRejectsDuplicateFallbackProfilePaths(t *testing.T) {
|
||||
registry := NewAssetRegistry()
|
||||
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{"first/profile.yaml": {Data: []byte("id: first\n")}}, "first"); err != nil {
|
||||
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{"first/profile.yaml": {Data: []byte("profiles:\n first: {}\n")}}, "first"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{"second/profile.yaml": {Data: []byte("id: second\n")}}, "second"); err != nil {
|
||||
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{"second/profile.yaml": {Data: []byte("profiles:\n second: {}\n")}}, "second"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err := registry.FallbackProfileFS()
|
||||
@@ -172,8 +194,8 @@ func TestAssetRegistryFallbackProfileDigestTracksContentWithoutLeakingIt(t *test
|
||||
return digest
|
||||
}
|
||||
|
||||
first := digestFor("id: fallback\nmodel: model-one\n")
|
||||
second := digestFor("id: fallback\nmodel: model-two\n")
|
||||
first := digestFor("profiles:\n fallback:\n model: model-one\n")
|
||||
second := digestFor("profiles:\n fallback:\n model: model-two\n")
|
||||
if first == second {
|
||||
t.Fatalf("fallback profile digests = %q and %q, want content change", first, second)
|
||||
}
|
||||
@@ -231,16 +253,19 @@ func TestHashAssetsOmitsRawAssetContent(t *testing.T) {
|
||||
|
||||
func newPromptKitAssetTestEngine(t *testing.T, registry *AssetRegistry) *promptkit.Engine {
|
||||
t.Helper()
|
||||
options, err := registry.PromptKitOptions()
|
||||
sources, options, err := registry.PromptKitConfigurationSources()
|
||||
if err != nil {
|
||||
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
|
||||
t.Fatalf("PromptKitConfigurationSources() error = %v, want nil", err)
|
||||
}
|
||||
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
||||
ID: "asset-test-profile",
|
||||
Endpoint: "http://127.0.0.1:1/v1",
|
||||
Model: "asset-test-model",
|
||||
})))
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
||||
sources.Underlays = append(sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{
|
||||
Source: promptkit.ProfileSource("asset test profile", map[string]promptkit.ProfileConfig{
|
||||
"asset-test-profile": {
|
||||
Endpoint: promptKitTestString("http://127.0.0.1:1/v1"),
|
||||
Model: promptKitTestString("asset-test-model"),
|
||||
},
|
||||
}),
|
||||
}}})
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: sources, Timeout: time.Second}, options...)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine() error = %v, want nil", err)
|
||||
}
|
||||
@@ -262,22 +287,23 @@ func mustRegisterSchemaFS(t *testing.T, registry *AssetRegistry, fsys fstest.Map
|
||||
}
|
||||
|
||||
func validPromptYAML(schemaPath string) string {
|
||||
return `id: asset.test
|
||||
version: "v1"
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
messages:
|
||||
- role: user
|
||||
content_file: ./messages/user.tmpl
|
||||
- role: user
|
||||
content_file: ./messages/task.tmpl
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: ` + schemaPath + `
|
||||
repair_attempts: 0
|
||||
return `prompts:
|
||||
asset.test:
|
||||
v1:
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
messages:
|
||||
- role: user
|
||||
content_file: ./messages/user.tmpl
|
||||
- role: user
|
||||
content_file: ./messages/task.tmpl
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: ` + schemaPath + `
|
||||
repair_attempts: 0
|
||||
`
|
||||
}
|
||||
|
||||
|
||||
@@ -28,6 +28,7 @@ type PromptKitLocalBackendConfig struct {
|
||||
type PromptKitClientConfig struct {
|
||||
ProfileDir string
|
||||
ProfileFile string
|
||||
Sources promptkit.ConfigurationSources
|
||||
TransientRetries *int
|
||||
LocalBackend *PromptKitLocalBackendConfig
|
||||
Assets *AssetRegistry
|
||||
@@ -39,13 +40,10 @@ type PromptKitClientConfig struct {
|
||||
}
|
||||
|
||||
type PromptKitClient struct {
|
||||
engine *promptkit.Engine
|
||||
recorder *LLMProfileRecorder
|
||||
profileDir string
|
||||
profileFile string
|
||||
localEndpoint string
|
||||
fallbackProfileDigest string
|
||||
reasoningEffort *string
|
||||
engine *promptkit.Engine
|
||||
recorder *LLMProfileRecorder
|
||||
checkpointFingerprints []CheckpointFingerprint
|
||||
reasoningEffort *string
|
||||
}
|
||||
|
||||
type LLMProfileRecorder struct {
|
||||
@@ -53,40 +51,47 @@ type LLMProfileRecorder struct {
|
||||
profiles map[string]artifacts.LLMProfileManifest
|
||||
}
|
||||
|
||||
type promptKitClientConfigurationError struct {
|
||||
err error
|
||||
}
|
||||
|
||||
func (e *promptKitClientConfigurationError) Error() string {
|
||||
return "PromptKit client configuration is invalid or unreadable"
|
||||
}
|
||||
|
||||
func (e *promptKitClientConfigurationError) Unwrap() error {
|
||||
return e.err
|
||||
}
|
||||
|
||||
var _ contracts.StructuredLLMClient = (*PromptKitClient)(nil)
|
||||
var _ contracts.LLMProfileManifestProvider = (*PromptKitClient)(nil)
|
||||
|
||||
func PromptKitLocalBackendOption(cfg PromptKitLocalBackendConfig) promptkit.Option {
|
||||
return promptkit.WithBackend(promptkit.LocalBackend(cfg.Endpoint, cfg.ConcurrencyLimit))
|
||||
}
|
||||
|
||||
func NewPromptKitClient(cfg PromptKitClientConfig) (*PromptKitClient, error) {
|
||||
if cfg.Assets == nil {
|
||||
return nil, fmt.Errorf("PromptKit client assets must not be nil")
|
||||
}
|
||||
profileSource, profileOptions, err := promptKitProfileSourceEngineOptions(PromptKitProfileSourceConfig{
|
||||
_, sources, options, err := promptKitEngineConfiguration(PromptKitProfileSourceConfig{
|
||||
ProfileDir: cfg.ProfileDir,
|
||||
ProfileFile: cfg.ProfileFile,
|
||||
LocalBackend: cfg.LocalBackend,
|
||||
})
|
||||
}, cfg.Assets, cfg.Sources)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, &promptKitClientConfigurationError{err: err}
|
||||
}
|
||||
options, fallbackProfileDigest, err := cfg.Assets.promptKitOptions()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
options = append(options, profileOptions...)
|
||||
options = append(options, cfg.EngineOptions...)
|
||||
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||
ProfileDir: profileSource.ProfileDir,
|
||||
Sources: sources,
|
||||
Timeout: cfg.Timeout,
|
||||
HTTPClient: cfg.HTTPClient,
|
||||
TransientRetries: cfg.TransientRetries,
|
||||
}, options...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create PromptKit engine: %w", err)
|
||||
return nil, &promptKitClientConfigurationError{err: err}
|
||||
}
|
||||
fingerprint, err := promptKitConfigurationFingerprint(engine)
|
||||
if err != nil {
|
||||
return nil, &promptKitClientConfigurationError{err: err}
|
||||
}
|
||||
recorder := cfg.Recorder
|
||||
if recorder == nil {
|
||||
@@ -98,13 +103,10 @@ func NewPromptKitClient(cfg PromptKitClientConfig) (*PromptKitClient, error) {
|
||||
reasoningEffort = &value
|
||||
}
|
||||
return &PromptKitClient{
|
||||
engine: engine,
|
||||
recorder: recorder,
|
||||
profileDir: profileSource.ProfileDir,
|
||||
profileFile: profileSource.ProfileFile,
|
||||
localEndpoint: profileSource.localEndpoint(),
|
||||
fallbackProfileDigest: fallbackProfileDigest,
|
||||
reasoningEffort: reasoningEffort,
|
||||
engine: engine,
|
||||
recorder: recorder,
|
||||
checkpointFingerprints: []CheckpointFingerprint{fingerprint},
|
||||
reasoningEffort: reasoningEffort,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -463,15 +465,7 @@ func (c *PromptKitClient) LLMCheckpointFingerprints() ([]CheckpointFingerprint,
|
||||
if c == nil {
|
||||
return nil, nil
|
||||
}
|
||||
fingerprint, err := promptKitProfileFingerprint(c.profileDir, c.profileFile, c.fallbackProfileDigest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fingerprints := []CheckpointFingerprint{fingerprint}
|
||||
if c.localEndpoint != "" {
|
||||
fingerprints = append(fingerprints, promptKitLocalBackendFingerprint(c.localEndpoint))
|
||||
}
|
||||
return fingerprints, nil
|
||||
return append([]CheckpointFingerprint(nil), c.checkpointFingerprints...), nil
|
||||
}
|
||||
|
||||
func NewLLMProfileRecorder() *LLMProfileRecorder {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
129
internal/framework/llm/promptkit_configuration_fingerprint.go
Normal file
129
internal/framework/llm/promptkit_configuration_fingerprint.go
Normal file
@@ -0,0 +1,129 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
"go.yaml.in/yaml/v4"
|
||||
)
|
||||
|
||||
const promptKitConfigurationFingerprintName = "promptkit_configuration"
|
||||
|
||||
// This version describes the public definition graph captured by the engine.
|
||||
// Include the release and catalog revisions because they contribute built-in
|
||||
// profiles and other effective definitions without appearing in app sources.
|
||||
const promptKitConfigurationSnapshotMarker = "promptkit:v0.12.0:openrouter:v3.20260930.0:rakestrawhome:v3.20260930.0:configuration-snapshot:v1"
|
||||
|
||||
type promptKitConfigurationFingerprintPayload struct {
|
||||
SnapshotMarker string `json:"snapshot_marker"`
|
||||
Configuration map[string]any `json:"configuration"`
|
||||
Profiles []promptKitProfileFingerprintEntry `json:"profiles"`
|
||||
Prompts []promptKitPromptFingerprintEntry `json:"prompts"`
|
||||
}
|
||||
|
||||
type promptKitProfileFingerprintEntry struct {
|
||||
ID string `json:"id"`
|
||||
Target promptkit.ExecutionTarget `json:"target"`
|
||||
APIKeyRequired bool `json:"api_key_required"`
|
||||
}
|
||||
|
||||
type promptKitPromptFingerprintEntry struct {
|
||||
ID string `json:"id"`
|
||||
Version string `json:"version"`
|
||||
Hash string `json:"hash"`
|
||||
}
|
||||
|
||||
func promptKitConfigurationFingerprint(engine *promptkit.Engine) (CheckpointFingerprint, error) {
|
||||
if engine == nil {
|
||||
return CheckpointFingerprint{}, fmt.Errorf("PromptKit engine is unavailable for checkpoint identity")
|
||||
}
|
||||
|
||||
inspection, err := engine.InspectConfiguration()
|
||||
if err != nil {
|
||||
return CheckpointFingerprint{}, fmt.Errorf("inspect captured PromptKit configuration for checkpoint identity")
|
||||
}
|
||||
var report bytes.Buffer
|
||||
if err := inspection.WriteEffective(&report); err != nil {
|
||||
return CheckpointFingerprint{}, fmt.Errorf("write captured PromptKit configuration for checkpoint identity")
|
||||
}
|
||||
var configuration map[string]any
|
||||
if err := yaml.Unmarshal(report.Bytes(), &configuration); err != nil {
|
||||
return CheckpointFingerprint{}, fmt.Errorf("decode captured PromptKit configuration for checkpoint identity")
|
||||
}
|
||||
if configuration == nil {
|
||||
configuration = map[string]any{}
|
||||
}
|
||||
removeLocalSchedulingFields(configuration)
|
||||
|
||||
payload := promptKitConfigurationFingerprintPayload{
|
||||
SnapshotMarker: promptKitConfigurationSnapshotMarker,
|
||||
Configuration: configuration,
|
||||
}
|
||||
profileIDs := sortedMapKeys(configuration["profiles"])
|
||||
payload.Profiles = make([]promptKitProfileFingerprintEntry, 0, len(profileIDs))
|
||||
for _, id := range profileIDs {
|
||||
profile, err := engine.InspectProfile(context.Background(), id)
|
||||
if err != nil {
|
||||
return CheckpointFingerprint{}, fmt.Errorf("inspect captured PromptKit profile for checkpoint identity")
|
||||
}
|
||||
payload.Profiles = append(payload.Profiles, promptKitProfileFingerprintEntry{
|
||||
ID: profile.ProfileID,
|
||||
Target: profile.EffectiveModelParams,
|
||||
APIKeyRequired: profile.APIKeyRequired,
|
||||
})
|
||||
}
|
||||
|
||||
promptDefinitions, _ := configuration["prompts"].(map[string]any)
|
||||
var promptIDs []string
|
||||
for id := range promptDefinitions {
|
||||
promptIDs = append(promptIDs, id)
|
||||
}
|
||||
sort.Strings(promptIDs)
|
||||
for _, id := range promptIDs {
|
||||
versions := sortedMapKeys(promptDefinitions[id])
|
||||
for _, version := range versions {
|
||||
prompt, err := engine.InspectPrompt(context.Background(), id, version)
|
||||
if err != nil {
|
||||
return CheckpointFingerprint{}, fmt.Errorf("inspect captured PromptKit prompt for checkpoint identity")
|
||||
}
|
||||
payload.Prompts = append(payload.Prompts, promptKitPromptFingerprintEntry{
|
||||
ID: prompt.PromptID,
|
||||
Version: prompt.PromptVersion,
|
||||
Hash: prompt.PromptHash,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
encoded, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return CheckpointFingerprint{}, fmt.Errorf("encode captured PromptKit checkpoint identity")
|
||||
}
|
||||
digest := sha256.Sum256(encoded)
|
||||
return CheckpointFingerprint{
|
||||
Name: promptKitConfigurationFingerprintName,
|
||||
Value: "sha256:" + hex.EncodeToString(digest[:]),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func sortedMapKeys(value any) []string {
|
||||
values, _ := value.(map[string]any)
|
||||
keys := make([]string, 0, len(values))
|
||||
for key := range values {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
}
|
||||
|
||||
func removeLocalSchedulingFields(configuration map[string]any) {
|
||||
backends, _ := configuration["backends"].(map[string]any)
|
||||
local, _ := backends["local"].(map[string]any)
|
||||
delete(local, "concurrency_limit")
|
||||
delete(local, "queue_capacity")
|
||||
}
|
||||
155
internal/framework/llm/promptkit_inspection.go
Normal file
155
internal/framework/llm/promptkit_inspection.go
Normal file
@@ -0,0 +1,155 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
var ErrInvalidPromptKitConfigurationPointer = errors.New("invalid PromptKit configuration pointer")
|
||||
|
||||
// PromptKitEffectiveModelParams is the provider-neutral JSON projection of an
|
||||
// inspected profile target. Credential-shaped extra-parameter values are
|
||||
// redacted in this reporting copy.
|
||||
type PromptKitEffectiveModelParams struct {
|
||||
BackendID string `json:"backend_id"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
Model string `json:"model"`
|
||||
Temperature float64 `json:"temperature"`
|
||||
MaxTokens int `json:"max_tokens"`
|
||||
TopP float64 `json:"top_p"`
|
||||
TimeoutSeconds int `json:"timeout_seconds"`
|
||||
ServiceTier string `json:"service_tier"`
|
||||
ReasoningEffort string `json:"reasoning_effort"`
|
||||
APIKeyEnv string `json:"api_key_env"`
|
||||
ExtraParams map[string]any `json:"extra_params"`
|
||||
}
|
||||
|
||||
type invalidPromptKitConfigurationPointerError struct {
|
||||
cause error
|
||||
}
|
||||
|
||||
func (e *invalidPromptKitConfigurationPointerError) Error() string {
|
||||
return ErrInvalidPromptKitConfigurationPointer.Error()
|
||||
}
|
||||
|
||||
func (e *invalidPromptKitConfigurationPointerError) Unwrap() error {
|
||||
return e.cause
|
||||
}
|
||||
|
||||
func (e *invalidPromptKitConfigurationPointerError) Is(target error) bool {
|
||||
return target == ErrInvalidPromptKitConfigurationPointer
|
||||
}
|
||||
|
||||
func (i *PromptKitProfileInspector) EffectiveConfiguration() (string, error) {
|
||||
var output bytes.Buffer
|
||||
if err := i.WriteEffectiveConfiguration(&output); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return output.String(), nil
|
||||
}
|
||||
|
||||
func (i *PromptKitProfileInspector) WriteEffectiveConfiguration(output io.Writer) error {
|
||||
if i == nil || i.engine == nil {
|
||||
return fmt.Errorf("PromptKit inspector must not be nil")
|
||||
}
|
||||
inspection, err := i.engine.InspectConfiguration()
|
||||
if err != nil {
|
||||
return fmt.Errorf("inspect PromptKit configuration: %w", err)
|
||||
}
|
||||
return inspection.WriteEffective(output)
|
||||
}
|
||||
|
||||
func (i *PromptKitProfileInspector) ExplainConfiguration(pointer string) (string, bool, error) {
|
||||
var output bytes.Buffer
|
||||
found, err := i.WriteConfigurationExplanation(&output, pointer)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
return output.String(), found, nil
|
||||
}
|
||||
|
||||
func (i *PromptKitProfileInspector) WriteConfigurationExplanation(output io.Writer, pointer string) (bool, error) {
|
||||
if i == nil || i.engine == nil {
|
||||
return false, fmt.Errorf("PromptKit inspector must not be nil")
|
||||
}
|
||||
path, err := configkit.ParsePath(pointer)
|
||||
if err != nil || path.String() != pointer {
|
||||
if err == nil {
|
||||
err = fmt.Errorf("pointer spelling is not canonical")
|
||||
}
|
||||
return false, &invalidPromptKitConfigurationPointerError{cause: err}
|
||||
}
|
||||
inspection, err := i.engine.InspectConfiguration()
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("inspect PromptKit configuration: %w", err)
|
||||
}
|
||||
if _, found := inspection.Explain(path); !found {
|
||||
return false, nil
|
||||
}
|
||||
if err := inspection.WriteExplanation(output, path); err != nil {
|
||||
return true, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func promptKitEffectiveModelParams(target promptkit.ExecutionTarget) PromptKitEffectiveModelParams {
|
||||
return PromptKitEffectiveModelParams{
|
||||
BackendID: strings.TrimSpace(target.BackendID),
|
||||
Endpoint: strings.TrimSpace(target.Endpoint),
|
||||
Model: strings.TrimSpace(target.Model),
|
||||
Temperature: target.Temperature,
|
||||
MaxTokens: target.MaxTokens,
|
||||
TopP: target.TopP,
|
||||
TimeoutSeconds: target.TimeoutSeconds,
|
||||
ServiceTier: target.ServiceTier,
|
||||
ReasoningEffort: target.ReasoningEffort,
|
||||
APIKeyEnv: strings.TrimSpace(target.APIKeyEnv),
|
||||
ExtraParams: redactCredentialParameters(target.ExtraParams),
|
||||
}
|
||||
}
|
||||
|
||||
func redactCredentialParameters(values map[string]any) map[string]any {
|
||||
if values == nil {
|
||||
return nil
|
||||
}
|
||||
redacted, _ := redactCredentialParameterValue(values).(map[string]any)
|
||||
return redacted
|
||||
}
|
||||
|
||||
func redactCredentialParameterValue(value any) any {
|
||||
switch typed := value.(type) {
|
||||
case map[string]any:
|
||||
copy := make(map[string]any, len(typed))
|
||||
for key, nested := range typed {
|
||||
if isCredentialParameterName(key) {
|
||||
copy[key] = "[REDACTED]"
|
||||
continue
|
||||
}
|
||||
copy[key] = redactCredentialParameterValue(nested)
|
||||
}
|
||||
return copy
|
||||
case []any:
|
||||
copy := make([]any, len(typed))
|
||||
for index, nested := range typed {
|
||||
copy[index] = redactCredentialParameterValue(nested)
|
||||
}
|
||||
return copy
|
||||
default:
|
||||
return typed
|
||||
}
|
||||
}
|
||||
|
||||
func isCredentialParameterName(name string) bool {
|
||||
switch strings.ToLower(name) {
|
||||
case "api_key", "apikey", "authorization", "credential", "credentials", "password", "secret", "token", "access_token":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
143
internal/framework/llm/promptkit_inspection_test.go
Normal file
143
internal/framework/llm/promptkit_inspection_test.go
Normal file
@@ -0,0 +1,143 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
type promptKitInspectionErrorWriter struct{ err error }
|
||||
|
||||
func (w promptKitInspectionErrorWriter) Write([]byte) (int, error) { return 0, w.err }
|
||||
|
||||
func TestPromptKitProfileInspectionCopiesAndRedactsTarget(t *testing.T) {
|
||||
const (
|
||||
profileID = "inspection/profile~case"
|
||||
keyEnv = "NOT_SET_SYNTHETIC_PROFILE_KEY"
|
||||
)
|
||||
endpoint, initialModel, finalModel := "https://example.invalid/v1", "model-before", "model-after"
|
||||
params := map[string]any{
|
||||
"Authorization": "Bearer authorization-secret",
|
||||
"api_key": "api-key-secret",
|
||||
"ApiKey": "api-key-camel-secret",
|
||||
"apikey": "apikey-secret",
|
||||
"credential": "credential-secret",
|
||||
"credentials": "credentials-secret",
|
||||
"password": "password-secret",
|
||||
"secret": "secret-secret",
|
||||
"token": "token-secret",
|
||||
"access_token": "access-token-secret",
|
||||
"normal": "visible-value",
|
||||
"nested": map[string]any{
|
||||
"Access_Token": "nested-token-secret",
|
||||
"items": []any{map[string]any{"PASSWORD": "list-secret", "label": "ordinary"}},
|
||||
},
|
||||
}
|
||||
inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{
|
||||
{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("lower profile", map[string]promptkit.ProfileConfig{
|
||||
profileID: {Endpoint: &endpoint, Model: &initialModel, APIKeyEnv: promptKitTestString(keyEnv), ExtraParams: params},
|
||||
})}}},
|
||||
{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("profile overlay", map[string]promptkit.ProfileConfig{
|
||||
profileID: {Model: &finalModel},
|
||||
})}}},
|
||||
},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("NewPromptKitProfileInspector() error = %v", err)
|
||||
}
|
||||
profile, err := inspector.InspectProfile(t.Context(), profileID)
|
||||
if err != nil {
|
||||
t.Fatalf("InspectProfile() error = %v; an unset synthetic credential name must not be resolved", err)
|
||||
}
|
||||
if profile.ProfileID != profileID || profile.BackendID != "" || profile.Model != finalModel || profile.CredentialEnvironment != keyEnv || profile.CredentialRequired {
|
||||
t.Fatalf("profile inspection = %#v", profile)
|
||||
}
|
||||
paramsCopy := profile.EffectiveModelParams.ExtraParams
|
||||
for _, key := range []string{"Authorization", "api_key", "ApiKey", "apikey", "credential", "credentials", "password", "secret", "token", "access_token"} {
|
||||
if paramsCopy[key] != "[REDACTED]" {
|
||||
t.Errorf("effective provider parameter %q = %#v, want redacted", key, paramsCopy[key])
|
||||
}
|
||||
}
|
||||
if paramsCopy["normal"] != "visible-value" {
|
||||
t.Fatalf("top-level effective provider parameters = %#v", paramsCopy)
|
||||
}
|
||||
nested := paramsCopy["nested"].(map[string]any)
|
||||
if nested["Access_Token"] != "[REDACTED]" {
|
||||
t.Fatalf("nested effective provider parameters = %#v", nested)
|
||||
}
|
||||
items := nested["items"].([]any)
|
||||
if items[0].(map[string]any)["PASSWORD"] != "[REDACTED]" || items[0].(map[string]any)["label"] != "ordinary" {
|
||||
t.Fatalf("list provider parameters = %#v", items)
|
||||
}
|
||||
encoded, err := json.Marshal(profile.EffectiveModelParams)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, secret := range []string{"authorization-secret", "api-key-secret", "api-key-camel-secret", "apikey-secret", "credential-secret", "credentials-secret", "password-secret", "secret-secret", "token-secret", "access-token-secret", "nested-token-secret", "list-secret"} {
|
||||
if strings.Contains(string(encoded), secret) {
|
||||
t.Fatalf("profile reporting projection contains %q", secret)
|
||||
}
|
||||
}
|
||||
nested["ordinary"] = "caller mutation"
|
||||
paramsCopy["normal"] = "caller mutation"
|
||||
second, err := inspector.InspectProfile(t.Context(), profileID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if second.EffectiveModelParams.ExtraParams["normal"] != "visible-value" || second.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["ordinary"] != nil {
|
||||
t.Fatalf("profile inspection retained caller mutation: %#v", second.EffectiveModelParams.ExtraParams)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptKitConfigurationInspectionDistinguishesInvalidAndUnknownPointers(t *testing.T) {
|
||||
const profileID = "inspection/profile~case"
|
||||
endpoint, initialModel, finalModel := "https://example.invalid/v1", "model-before", "model-after"
|
||||
inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{
|
||||
{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("lower profile", map[string]promptkit.ProfileConfig{
|
||||
profileID: {Endpoint: &endpoint, Model: &initialModel},
|
||||
})}}},
|
||||
{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("profile overlay", map[string]promptkit.ProfileConfig{
|
||||
profileID: {Model: &finalModel},
|
||||
})}}},
|
||||
},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("NewPromptKitProfileInspector() error = %v", err)
|
||||
}
|
||||
|
||||
report, err := inspector.EffectiveConfiguration()
|
||||
if err != nil {
|
||||
t.Fatalf("EffectiveConfiguration() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(report, "model-after") || strings.Contains(report, "model-before") {
|
||||
t.Fatalf("effective report = %q, want only the effective model", report)
|
||||
}
|
||||
const escapedPointer = "/profiles/inspection~1profile~0case/model"
|
||||
explanation, found, err := inspector.ExplainConfiguration(escapedPointer)
|
||||
if err != nil || !found || !strings.Contains(explanation, "model-before") || !strings.Contains(explanation, "model-after") {
|
||||
t.Fatalf("ExplainConfiguration(%q) = (%q, %t, %v), want effective and historical values", escapedPointer, explanation, found, err)
|
||||
}
|
||||
if _, found, err := inspector.ExplainConfiguration("/profiles/not-configured"); err != nil || found {
|
||||
t.Fatalf("unknown configuration pointer = (found %t, error %v), want (false, nil)", found, err)
|
||||
}
|
||||
if _, _, err := inspector.ExplainConfiguration("/profiles/~2/model"); !errors.Is(err, ErrInvalidPromptKitConfigurationPointer) {
|
||||
t.Fatalf("invalid configuration pointer error = %v", err)
|
||||
}
|
||||
if _, found, err := inspector.ExplainConfiguration("/profiles/inspection~01profile~00case/model"); err != nil || found {
|
||||
t.Fatalf("configuration pointer with literal escape sequences = (found %t, error %v), want a valid unknown pointer", found, err)
|
||||
}
|
||||
|
||||
writerFailure := errors.New("writer failure")
|
||||
if err := inspector.WriteEffectiveConfiguration(promptKitInspectionErrorWriter{err: writerFailure}); !errors.Is(err, writerFailure) {
|
||||
t.Fatalf("WriteEffectiveConfiguration() error = %v, want writer cause", err)
|
||||
}
|
||||
if _, err := inspector.WriteConfigurationExplanation(promptKitInspectionErrorWriter{err: writerFailure}, escapedPointer); !errors.Is(err, writerFailure) {
|
||||
t.Fatalf("WriteConfigurationExplanation() error = %v, want writer cause", err)
|
||||
}
|
||||
}
|
||||
@@ -23,9 +23,9 @@ func newObservedHTTPClient(t *testing.T, handler http.HandlerFunc, retries int,
|
||||
client, err := NewPromptKitClient(PromptKitClientConfig{
|
||||
Assets: newTestPromptKitAssets(t), TransientRetries: &retries,
|
||||
HTTPClient: &http.Client{Transport: transport},
|
||||
EngineOptions: []promptkit.Option{promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
||||
ID: "default-profile", Endpoint: server.URL + "/v1", Model: "test-model",
|
||||
}))},
|
||||
Sources: promptKitTestSources(nil, map[string]promptkit.ProfileConfig{"default-profile": {
|
||||
Endpoint: promptKitTestString(server.URL + "/v1"), Model: promptKitTestString("test-model"),
|
||||
}}),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -1,110 +0,0 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
promptKitProfileFingerprintName = "promptkit_profile_source"
|
||||
promptKitLocalBackendFingerprintName = "promptkit_local_backend_target"
|
||||
promptKitLocalBackendMarker = "notarius:promptkit-local-backend:v1"
|
||||
// The built-in profile catalog is compiled into this pinned PromptKit
|
||||
// release. Update this identity when the dependency is upgraded.
|
||||
promptKitBuiltinProfileCatalogID = "promptkit:v0.11.2:builtin-profiles"
|
||||
)
|
||||
|
||||
func promptKitProfileFingerprint(profileDir, profileFile, fallbackProfileDigest string) (CheckpointFingerprint, error) {
|
||||
hasher := sha256.New()
|
||||
writeFingerprintPart(hasher, []byte(promptKitBuiltinProfileCatalogID))
|
||||
writeFingerprintPart(hasher, []byte(strings.TrimSpace(fallbackProfileDigest)))
|
||||
|
||||
switch {
|
||||
case strings.TrimSpace(profileFile) != "":
|
||||
cleanProfileFile := strings.TrimSpace(profileFile)
|
||||
data, err := os.ReadFile(cleanProfileFile)
|
||||
if err != nil {
|
||||
return CheckpointFingerprint{}, fmt.Errorf("read PromptKit profile file for checkpoint identity")
|
||||
}
|
||||
writeFingerprintPart(hasher, []byte(filepath.ToSlash(filepath.Base(cleanProfileFile))))
|
||||
writeFingerprintPart(hasher, data)
|
||||
case strings.TrimSpace(profileDir) != "":
|
||||
files, err := promptKitProfileFiles(strings.TrimSpace(profileDir))
|
||||
if err != nil {
|
||||
return CheckpointFingerprint{}, err
|
||||
}
|
||||
for _, file := range files {
|
||||
writeFingerprintPart(hasher, []byte(file.path))
|
||||
writeFingerprintPart(hasher, file.digest)
|
||||
}
|
||||
}
|
||||
|
||||
return CheckpointFingerprint{
|
||||
Name: promptKitProfileFingerprintName,
|
||||
Value: "sha256:" + hex.EncodeToString(hasher.Sum(nil)),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func promptKitLocalBackendFingerprint(endpoint string) CheckpointFingerprint {
|
||||
hasher := sha256.New()
|
||||
writeFingerprintPart(hasher, []byte(promptKitLocalBackendMarker))
|
||||
writeFingerprintPart(hasher, []byte(strings.TrimSpace(endpoint)))
|
||||
return CheckpointFingerprint{
|
||||
Name: promptKitLocalBackendFingerprintName,
|
||||
Value: "sha256:" + hex.EncodeToString(hasher.Sum(nil)),
|
||||
}
|
||||
}
|
||||
|
||||
type promptKitProfileFile struct {
|
||||
path string
|
||||
digest []byte
|
||||
}
|
||||
|
||||
func promptKitProfileFiles(root string) ([]promptKitProfileFile, error) {
|
||||
var files []promptKitProfileFile
|
||||
err := filepath.WalkDir(root, func(name string, entry fs.DirEntry, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
if entry.IsDir() {
|
||||
return nil
|
||||
}
|
||||
extension := filepath.Ext(entry.Name())
|
||||
if extension != ".yaml" && extension != ".yml" {
|
||||
return nil
|
||||
}
|
||||
data, err := os.ReadFile(name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
relativePath, err := filepath.Rel(root, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sum := sha256.Sum256(data)
|
||||
files = append(files, promptKitProfileFile{
|
||||
path: filepath.ToSlash(relativePath),
|
||||
digest: append([]byte(nil), sum[:]...),
|
||||
})
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read PromptKit profile directory for checkpoint identity")
|
||||
}
|
||||
sort.Slice(files, func(i, j int) bool {
|
||||
return files[i].path < files[j].path
|
||||
})
|
||||
return files, nil
|
||||
}
|
||||
|
||||
func writeFingerprintPart(hasher interface{ Write([]byte) (int, error) }, value []byte) {
|
||||
length := []byte(fmt.Sprintf("%d:", len(value)))
|
||||
_, _ = hasher.Write(length)
|
||||
_, _ = hasher.Write(value)
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
@@ -16,8 +17,9 @@ type PromptKitProfileSourceConfig struct {
|
||||
}
|
||||
|
||||
type PromptKitProfileInspectorConfig struct {
|
||||
Source PromptKitProfileSourceConfig
|
||||
Assets *AssetRegistry
|
||||
Source PromptKitProfileSourceConfig
|
||||
Assets *AssetRegistry
|
||||
Sources promptkit.ConfigurationSources
|
||||
}
|
||||
|
||||
func (c PromptKitProfileSourceConfig) localEndpoint() string {
|
||||
@@ -37,6 +39,7 @@ type PromptKitProfileInspection struct {
|
||||
Model string
|
||||
CredentialEnvironment string
|
||||
CredentialRequired bool
|
||||
EffectiveModelParams PromptKitEffectiveModelParams `json:"effective_model_params"`
|
||||
}
|
||||
|
||||
type PromptKitProfileInspectionError struct {
|
||||
@@ -74,22 +77,12 @@ func (e *promptKitProfileConfigurationError) Unwrap() error {
|
||||
}
|
||||
|
||||
func NewPromptKitProfileInspector(cfg PromptKitProfileInspectorConfig) (*PromptKitProfileInspector, error) {
|
||||
source, options, err := promptKitProfileSourceEngineOptions(cfg.Source)
|
||||
_, sources, options, err := promptKitEngineConfiguration(cfg.Source, cfg.Assets, cfg.Sources)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if cfg.Assets != nil {
|
||||
fallbackOption, hasFallback, err := cfg.Assets.promptKitFallbackProfileOption()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if hasFallback {
|
||||
options = append(options, fallbackOption)
|
||||
}
|
||||
return nil, &promptKitProfileConfigurationError{err: err}
|
||||
}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||
PromptDir: ".",
|
||||
ProfileDir: source.ProfileDir,
|
||||
Sources: sources,
|
||||
}, options...)
|
||||
if err != nil {
|
||||
return nil, &promptKitProfileConfigurationError{err: err}
|
||||
@@ -101,7 +94,6 @@ func (i *PromptKitProfileInspector) InspectProfile(ctx context.Context, profileI
|
||||
if i == nil || i.engine == nil {
|
||||
return PromptKitProfileInspection{}, fmt.Errorf("PromptKit profile inspector must not be nil")
|
||||
}
|
||||
profileID = strings.TrimSpace(profileID)
|
||||
inspection, err := i.engine.InspectProfile(ctx, profileID)
|
||||
if err != nil {
|
||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||
@@ -118,16 +110,38 @@ func (i *PromptKitProfileInspector) InspectProfile(ctx context.Context, profileI
|
||||
Model: strings.TrimSpace(inspection.EffectiveModelParams.Model),
|
||||
CredentialEnvironment: strings.TrimSpace(inspection.EffectiveModelParams.APIKeyEnv),
|
||||
CredentialRequired: inspection.APIKeyRequired,
|
||||
EffectiveModelParams: promptKitEffectiveModelParams(inspection.EffectiveModelParams),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func promptKitProfileSourceEngineOptions(cfg PromptKitProfileSourceConfig) (PromptKitProfileSourceConfig, []promptkit.Option, error) {
|
||||
func promptKitEngineConfiguration(cfg PromptKitProfileSourceConfig, assets *AssetRegistry, configuredSources promptkit.ConfigurationSources) (PromptKitProfileSourceConfig, promptkit.ConfigurationSources, []promptkit.Option, error) {
|
||||
profileSource, profileSources, err := promptKitProfileConfigurationSources(cfg)
|
||||
if err != nil {
|
||||
return PromptKitProfileSourceConfig{}, promptkit.ConfigurationSources{}, nil, err
|
||||
}
|
||||
sources := promptkit.ConfigurationSources{}
|
||||
var options []promptkit.Option
|
||||
if assets != nil {
|
||||
maintainedSources, maintainedOptions, err := assets.PromptKitConfigurationSources()
|
||||
if err != nil {
|
||||
return PromptKitProfileSourceConfig{}, promptkit.ConfigurationSources{}, nil, err
|
||||
}
|
||||
sources = maintainedSources
|
||||
options = append(options, maintainedOptions...)
|
||||
}
|
||||
sources.Underlays = append(sources.Underlays, configuredSources.Underlays...)
|
||||
sources.Underlays = append(sources.Underlays, profileSources.Underlays...)
|
||||
sources.Main = configuredSources.Main
|
||||
return profileSource, sources, options, nil
|
||||
}
|
||||
|
||||
func promptKitProfileConfigurationSources(cfg PromptKitProfileSourceConfig) (PromptKitProfileSourceConfig, promptkit.ConfigurationSources, error) {
|
||||
source := PromptKitProfileSourceConfig{
|
||||
ProfileDir: strings.TrimSpace(cfg.ProfileDir),
|
||||
ProfileFile: strings.TrimSpace(cfg.ProfileFile),
|
||||
}
|
||||
if source.ProfileDir != "" && source.ProfileFile != "" {
|
||||
return PromptKitProfileSourceConfig{}, nil, fmt.Errorf("PromptKit profile_dir and profile_file are mutually exclusive")
|
||||
return PromptKitProfileSourceConfig{}, promptkit.ConfigurationSources{}, fmt.Errorf("PromptKit profile_dir and profile_file are mutually exclusive")
|
||||
}
|
||||
if cfg.LocalBackend != nil {
|
||||
localBackend := *cfg.LocalBackend
|
||||
@@ -135,12 +149,31 @@ func promptKitProfileSourceEngineOptions(cfg PromptKitProfileSourceConfig) (Prom
|
||||
source.LocalBackend = &localBackend
|
||||
}
|
||||
|
||||
var options []promptkit.Option
|
||||
var underlays []configkit.Underlay
|
||||
if source.ProfileFile != "" {
|
||||
options = append(options, promptkit.WithProfileFile(source.ProfileFile))
|
||||
underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{{
|
||||
Source: configkit.File(source.ProfileFile),
|
||||
Imports: true,
|
||||
}}})
|
||||
} else if source.ProfileDir != "" {
|
||||
inputs, err := promptkit.DirectoryInputs(source.ProfileDir)
|
||||
if err != nil {
|
||||
return PromptKitProfileSourceConfig{}, promptkit.ConfigurationSources{}, fmt.Errorf("select PromptKit profile directory: %w", err)
|
||||
}
|
||||
for index := range inputs {
|
||||
inputs[index].Imports = true
|
||||
}
|
||||
if len(inputs) != 0 {
|
||||
underlays = append(underlays, configkit.Underlay{Inputs: inputs})
|
||||
}
|
||||
}
|
||||
if source.LocalBackend != nil {
|
||||
options = append(options, PromptKitLocalBackendOption(*source.LocalBackend))
|
||||
backend := promptkit.LocalBackend(source.LocalBackend.Endpoint, source.LocalBackend.ConcurrencyLimit)
|
||||
underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{{
|
||||
Source: promptkit.BackendSource("notarius-local-backend", map[string]promptkit.BackendConfig{
|
||||
promptkit.BackendLocal: backend,
|
||||
}),
|
||||
}}})
|
||||
}
|
||||
return source, options, nil
|
||||
return source, promptkit.ConfigurationSources{Underlays: underlays}, nil
|
||||
}
|
||||
|
||||
462
internal/framework/llm/promptkit_sources_integration_test.go
Normal file
462
internal/framework/llm/promptkit_sources_integration_test.go
Normal file
@@ -0,0 +1,462 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestPromptKitOperatorDocumentComposesForInspectionAndExecution(t *testing.T) {
|
||||
for _, name := range []string{
|
||||
"NOTARIUS_TEST_BACKEND_KEY",
|
||||
"NOTARIUS_TEST_PROFILE_KEY",
|
||||
"NOTARIUS_TEST_CATALOG_KEY",
|
||||
} {
|
||||
t.Setenv(name, "")
|
||||
}
|
||||
operatorDir := t.TempDir()
|
||||
profilePath := filepath.Join(operatorDir, "operator.yaml")
|
||||
if err := os.WriteFile(filepath.Join(operatorDir, "inherited.yaml"), []byte(`profiles:
|
||||
inherited-base:
|
||||
backend: custom-backend
|
||||
model: imported-model
|
||||
temperature: 0.7
|
||||
service_tier: flex
|
||||
extra_params:
|
||||
inherited: retained
|
||||
nested:
|
||||
inherited: true
|
||||
`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(profilePath, []byte(`configkit:
|
||||
imports:
|
||||
- path: inherited.yaml
|
||||
backends:
|
||||
custom-backend:
|
||||
endpoint: http://127.0.0.1:1/v1
|
||||
api_key_env: NOTARIUS_TEST_BACKEND_KEY
|
||||
concurrency_limit: 1
|
||||
queue_capacity: 0
|
||||
openrouter:
|
||||
api_key_env: NOTARIUS_TEST_CATALOG_KEY
|
||||
profiles:
|
||||
custom-profile:
|
||||
extends: inherited-base
|
||||
model: operator-model
|
||||
temperature: 0
|
||||
api_key_required: false
|
||||
service_tier: null
|
||||
extra_params: null
|
||||
empty-map-profile:
|
||||
extends: inherited-base
|
||||
extra_params: {}
|
||||
api_key_required: false
|
||||
profile-key-profile:
|
||||
backend: custom-backend
|
||||
model: profile-key-model
|
||||
api_key_env: NOTARIUS_TEST_PROFILE_KEY
|
||||
api_key_required: false
|
||||
required-profile:
|
||||
extends: inherited-base
|
||||
api_key_env: NOTARIUS_TEST_PROFILE_KEY
|
||||
api_key_required: true
|
||||
catalog-profile:
|
||||
backend: openrouter
|
||||
model: openai/gpt-5.6-luna
|
||||
api_key_required: false
|
||||
prompts:
|
||||
adapter.test:
|
||||
v1:
|
||||
output:
|
||||
repair_attempts: 1
|
||||
`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
assets := newTestPromptKitAssets(t)
|
||||
fake := &fakePromptKitLLM{}
|
||||
client, err := NewPromptKitClient(PromptKitClientConfig{
|
||||
Assets: assets,
|
||||
ProfileFile: profilePath,
|
||||
EngineOptions: []promptkit.Option{
|
||||
promptkit.WithLLMClient(fake),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewPromptKitClient() error = %v", err)
|
||||
}
|
||||
inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{
|
||||
Source: PromptKitProfileSourceConfig{ProfileFile: profilePath},
|
||||
Assets: assets,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewPromptKitProfileInspector() error = %v", err)
|
||||
}
|
||||
|
||||
for _, test := range []struct {
|
||||
id string
|
||||
backend string
|
||||
model string
|
||||
keyEnv string
|
||||
required bool
|
||||
}{
|
||||
{id: "custom-profile", backend: "custom-backend", model: "operator-model", keyEnv: "NOTARIUS_TEST_BACKEND_KEY"},
|
||||
{id: "inherited-base", backend: "custom-backend", model: "imported-model", keyEnv: "NOTARIUS_TEST_BACKEND_KEY"},
|
||||
{id: "empty-map-profile", backend: "custom-backend", model: "imported-model", keyEnv: "NOTARIUS_TEST_BACKEND_KEY"},
|
||||
{id: "profile-key-profile", backend: "custom-backend", model: "profile-key-model", keyEnv: "NOTARIUS_TEST_PROFILE_KEY"},
|
||||
{id: "required-profile", backend: "custom-backend", model: "imported-model", keyEnv: "NOTARIUS_TEST_PROFILE_KEY", required: true},
|
||||
{id: "catalog-profile", backend: promptkit.BackendOpenRouter, model: "openai/gpt-5.6-luna", keyEnv: "NOTARIUS_TEST_CATALOG_KEY"},
|
||||
} {
|
||||
inspection, err := inspector.InspectProfile(context.Background(), test.id)
|
||||
if err != nil {
|
||||
t.Fatalf("InspectProfile(%q) error = %v", test.id, err)
|
||||
}
|
||||
if inspection.ProfileID != test.id || inspection.BackendID != test.backend || inspection.Model != test.model ||
|
||||
inspection.CredentialEnvironment != test.keyEnv || inspection.CredentialRequired != test.required {
|
||||
t.Fatalf("InspectProfile(%q) = %#v", test.id, inspection)
|
||||
}
|
||||
}
|
||||
|
||||
var out map[string]any
|
||||
response, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
PromptID: "adapter.test",
|
||||
ProfileID: "custom-profile",
|
||||
SessionID: "source-composition-test",
|
||||
Inputs: contracts.LLMInputSet{
|
||||
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
||||
},
|
||||
}, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("CompleteStructured() with an unset optional credential error = %v", err)
|
||||
}
|
||||
if response.Debug == nil || response.Debug.Prompt == nil {
|
||||
t.Fatalf("completion debug = %#v, want captured prompt", response.Debug)
|
||||
}
|
||||
debug := response.Debug.Prompt
|
||||
if debug.SelectedBackendID != "custom-backend" || debug.SelectedProfileID != "custom-profile" {
|
||||
t.Fatalf("runtime selection = backend %q, profile %q", debug.SelectedBackendID, debug.SelectedProfileID)
|
||||
}
|
||||
if got := debug.EffectiveModelParams["model"]; got != "operator-model" {
|
||||
t.Fatalf("runtime model = %#v, want operator-model", got)
|
||||
}
|
||||
if got, ok := debug.EffectiveModelParams["temperature"].(float64); !ok || got != 0 {
|
||||
t.Fatalf("runtime temperature = %#v, want explicit zero", debug.EffectiveModelParams["temperature"])
|
||||
}
|
||||
if got := debug.EffectiveModelParams["service_tier"]; got == "flex" {
|
||||
t.Fatalf("explicit null retained inherited service tier: %#v", got)
|
||||
}
|
||||
if got, ok := debug.EffectiveModelParams["extra_params"]; ok && got != nil {
|
||||
t.Fatalf("explicit null retained inherited provider map: %#v", got)
|
||||
}
|
||||
if got := debug.OutputContract["repair_attempts"]; got != float64(1) {
|
||||
t.Fatalf("overlaid output repair attempts = %#v, want 1", got)
|
||||
}
|
||||
if len(debug.Messages) == 0 || !strings.Contains(debug.Messages[0].Content, "Transcript:") {
|
||||
t.Fatalf("rendered prompt messages = %#v", debug.Messages)
|
||||
}
|
||||
if got := atomic.LoadInt32(&fake.calls); got != 1 {
|
||||
t.Fatalf("provider calls after optional-credential execution = %d, want 1", got)
|
||||
}
|
||||
|
||||
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
PromptID: "adapter.test",
|
||||
ProfileID: "required-profile",
|
||||
SessionID: "source-composition-test",
|
||||
Inputs: contracts.LLMInputSet{
|
||||
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
||||
},
|
||||
}, &out)
|
||||
if err == nil || !strings.Contains(err.Error(), "api_key_env points to an unset environment variable") {
|
||||
t.Fatalf("CompleteStructured() with required unset credential error = %v, want native missing-environment failure", err)
|
||||
}
|
||||
if got := atomic.LoadInt32(&fake.calls); got != 1 {
|
||||
t.Fatalf("provider calls after required-credential failure = %d, want 1", got)
|
||||
}
|
||||
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
PromptID: "adapter.test",
|
||||
ProfileID: "empty-map-profile",
|
||||
SessionID: "source-composition-test",
|
||||
Inputs: contracts.LLMInputSet{
|
||||
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
||||
},
|
||||
}, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("CompleteStructured() with empty overlay map error = %v", err)
|
||||
}
|
||||
if got := fake.lastRequest().Target.ExtraParams["inherited"]; got != "retained" {
|
||||
t.Fatalf("empty provider-map overlay result = %#v, want inherited entry", fake.lastRequest().Target.ExtraParams)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptKitPromptContentUsesDeclaringAndReplacementSourceContexts(t *testing.T) {
|
||||
rootDirectory := t.TempDir()
|
||||
declaringDirectory := filepath.Join(rootDirectory, "child")
|
||||
replacementDirectory := t.TempDir()
|
||||
writeFile := func(path, content string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
rootConfig := filepath.Join(rootDirectory, "operator.yaml")
|
||||
declaringConfig := filepath.Join(declaringDirectory, "imported.yaml")
|
||||
replacementConfig := filepath.Join(replacementDirectory, "replacement.yaml")
|
||||
writeFile(rootConfig, `configkit:
|
||||
imports:
|
||||
- path: child/imported.yaml
|
||||
`)
|
||||
writeFile(filepath.Join(rootDirectory, "messages", "shared.tmpl"), "root decoy")
|
||||
writeFile(declaringConfig, `prompts:
|
||||
imported.capture:
|
||||
v1:
|
||||
messages:
|
||||
- role: user
|
||||
content_file: messages/shared.tmpl
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)
|
||||
writeFile(filepath.Join(declaringDirectory, "messages", "shared.tmpl"), "declared import")
|
||||
writeFile(replacementConfig, `prompts:
|
||||
adapter.test:
|
||||
v1:
|
||||
messages:
|
||||
- role: user
|
||||
content_file: messages/shared.tmpl
|
||||
`)
|
||||
writeFile(filepath.Join(replacementDirectory, "messages", "shared.tmpl"), "replacement original")
|
||||
|
||||
endpoint, model := "https://example.invalid/v1", "content-context-model"
|
||||
newClient := func() *PromptKitClient {
|
||||
t.Helper()
|
||||
client, err := NewPromptKitClient(PromptKitClientConfig{
|
||||
Assets: newTestPromptKitAssets(t),
|
||||
Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{
|
||||
{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("content context profile", map[string]promptkit.ProfileConfig{
|
||||
"content-context-profile": {Endpoint: &endpoint, Model: &model},
|
||||
})}}},
|
||||
{Inputs: []configkit.Input{{Source: configkit.File(rootConfig), Imports: true}}},
|
||||
{Inputs: []configkit.Input{{Source: configkit.File(replacementConfig), Imports: true}}},
|
||||
}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewPromptKitClient() error = %v", err)
|
||||
}
|
||||
return client
|
||||
}
|
||||
prepareMessage := func(client *PromptKitClient, promptID string) string {
|
||||
t.Helper()
|
||||
request := promptkit.RunRequest{PromptID: promptID, ProfileID: "content-context-profile", SessionID: "content-context-session"}
|
||||
if promptID == "adapter.test" {
|
||||
request.Inputs = map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.Inline(`{"source":true}`),
|
||||
}
|
||||
}
|
||||
prepared, err := client.engine.Prepare(context.Background(), request)
|
||||
if err != nil {
|
||||
t.Fatalf("Prepare(%q) error = %v", promptID, err)
|
||||
}
|
||||
if len(prepared.Messages) != 1 {
|
||||
t.Fatalf("Prepare(%q) messages = %#v, want one", promptID, prepared.Messages)
|
||||
}
|
||||
return prepared.Messages[0].Content
|
||||
}
|
||||
|
||||
first := newClient()
|
||||
if got := prepareMessage(first, "imported.capture"); got != "declared import" {
|
||||
t.Fatalf("imported prompt content = %q, want declaring-source content", got)
|
||||
}
|
||||
if got := prepareMessage(first, "adapter.test"); got != "replacement original" {
|
||||
t.Fatalf("partially overlaid application prompt content = %q, want replacement-source content", got)
|
||||
}
|
||||
writeFile(filepath.Join(declaringDirectory, "messages", "shared.tmpl"), "declared import updated")
|
||||
writeFile(filepath.Join(replacementDirectory, "messages", "shared.tmpl"), "replacement updated")
|
||||
if got := prepareMessage(first, "imported.capture"); got != "declared import" {
|
||||
t.Fatalf("existing engine imported prompt content after edit = %q", got)
|
||||
}
|
||||
if got := prepareMessage(first, "adapter.test"); got != "replacement original" {
|
||||
t.Fatalf("existing engine replacement content after edit = %q", got)
|
||||
}
|
||||
second := newClient()
|
||||
if got := prepareMessage(second, "imported.capture"); got != "declared import updated" {
|
||||
t.Fatalf("new engine imported prompt content = %q", got)
|
||||
}
|
||||
if got := prepareMessage(second, "adapter.test"); got != "replacement updated" {
|
||||
t.Fatalf("new engine replacement content = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptKitStaticMessageFailuresOccurAtConstruction(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
prompt string
|
||||
files fstest.MapFS
|
||||
}{
|
||||
{
|
||||
name: "missing message file",
|
||||
prompt: `prompts:
|
||||
construction.failure:
|
||||
v1:
|
||||
messages:
|
||||
- role: user
|
||||
content_file: messages/missing.tmpl
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`,
|
||||
},
|
||||
{
|
||||
name: "invalid static template",
|
||||
prompt: `prompts:
|
||||
construction.failure:
|
||||
v1:
|
||||
messages:
|
||||
- role: user
|
||||
content: '{{ .request_value'
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`,
|
||||
},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
assets := NewAssetRegistry()
|
||||
files := test.files
|
||||
if files == nil {
|
||||
files = fstest.MapFS{}
|
||||
}
|
||||
files["prompt.yaml"] = &fstest.MapFile{Data: []byte(test.prompt)}
|
||||
if err := assets.RegisterPromptFS(files, "."); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err := NewPromptKitClient(PromptKitClientConfig{
|
||||
Assets: assets,
|
||||
Sources: promptKitTestSources(nil, map[string]promptkit.ProfileConfig{"construction-profile": {
|
||||
Endpoint: promptKitTestString("https://example.invalid/v1"),
|
||||
Model: promptKitTestString("construction-model"),
|
||||
}}),
|
||||
})
|
||||
if err == nil || !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||
t.Fatalf("NewPromptKitClient() error = %v, want static construction failure", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptKitMissingRequestVariableFailsDuringPreparation(t *testing.T) {
|
||||
assets := NewAssetRegistry()
|
||||
if err := assets.RegisterPromptFS(fstest.MapFS{
|
||||
"prompt.yaml": {Data: []byte(`prompts:
|
||||
request.variable:
|
||||
v1:
|
||||
messages:
|
||||
- role: user
|
||||
content: 'Value: {{ .request_value }}'
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)},
|
||||
}, "."); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client, err := NewPromptKitClient(PromptKitClientConfig{
|
||||
Assets: assets,
|
||||
Sources: promptKitTestSources(nil, map[string]promptkit.ProfileConfig{"request-profile": {
|
||||
Endpoint: promptKitTestString("https://example.invalid/v1"),
|
||||
Model: promptKitTestString("request-model"),
|
||||
}}),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewPromptKitClient() error = %v, want valid static template", err)
|
||||
}
|
||||
if _, err := client.engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "request.variable", ProfileID: "request-profile",
|
||||
}); !errors.Is(err, promptkit.ErrPromptRender) {
|
||||
t.Fatalf("Prepare() error = %v, want missing request variable render failure", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptKitProfileSourceGroupsPreserveNativeLayerRules(t *testing.T) {
|
||||
t.Run("directory group conflicts", func(t *testing.T) {
|
||||
directory := t.TempDir()
|
||||
for name, model := range map[string]string{"first.yaml": "first-model", "second.yaml": "second-model"} {
|
||||
content := "profiles:\n duplicate:\n endpoint: http://127.0.0.1:1/v1\n model: " + model + "\n"
|
||||
if err := os.WriteFile(filepath.Join(directory, name), []byte(content), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
_, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{
|
||||
Source: PromptKitProfileSourceConfig{ProfileDir: directory},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("NewPromptKitProfileInspector() error = nil, want same-group conflict")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("later group repairs an invalid lower field and overlays supplied values", func(t *testing.T) {
|
||||
badEndpoint := "not an endpoint"
|
||||
goodEndpoint := "http://127.0.0.1:1/v1"
|
||||
lowerModel := "lower-model"
|
||||
sources := promptkit.ConfigurationSources{Underlays: []configkit.Underlay{
|
||||
{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("lower source", map[string]promptkit.ProfileConfig{
|
||||
"layered-profile": {Endpoint: &badEndpoint, Model: &lowerModel},
|
||||
})}}},
|
||||
{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("upper source", map[string]promptkit.ProfileConfig{
|
||||
"layered-profile": {Endpoint: &goodEndpoint},
|
||||
})}}},
|
||||
}}
|
||||
inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{Sources: sources})
|
||||
if err != nil {
|
||||
t.Fatalf("NewPromptKitProfileInspector() error = %v, want repaired final layer", err)
|
||||
}
|
||||
inspection, err := inspector.InspectProfile(context.Background(), "layered-profile")
|
||||
if err != nil || inspection.Model != lowerModel {
|
||||
t.Fatalf("InspectProfile() = (%#v, %v), want lower model retained", inspection, err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("empty and missing selected directories", func(t *testing.T) {
|
||||
if _, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{
|
||||
Source: PromptKitProfileSourceConfig{ProfileDir: filepath.Join(t.TempDir(), "missing")},
|
||||
}); err == nil {
|
||||
t.Fatal("missing selected directory was accepted")
|
||||
}
|
||||
empty := t.TempDir()
|
||||
inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{
|
||||
Source: PromptKitProfileSourceConfig{ProfileDir: empty},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("empty selected directory error = %v, want no contribution", err)
|
||||
}
|
||||
if _, err := inspector.InspectProfile(context.Background(), "not-present"); err == nil {
|
||||
t.Fatal("InspectProfile() in empty directory error = nil, want unknown profile")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("unused invalid final profile fails construction", func(t *testing.T) {
|
||||
badEndpoint := "not an endpoint"
|
||||
model := "unused-model"
|
||||
sources := promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{
|
||||
Source: promptkit.ProfileSource("unused invalid profile", map[string]promptkit.ProfileConfig{
|
||||
"unused-profile": {Endpoint: &badEndpoint, Model: &model},
|
||||
}),
|
||||
}}}}}
|
||||
_, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{Sources: sources})
|
||||
if err == nil || !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||
t.Fatalf("NewPromptKitProfileInspector() error = %v, want invalid final configuration", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
43
internal/framework/llm/promptkit_test_helpers_test.go
Normal file
43
internal/framework/llm/promptkit_test_helpers_test.go
Normal file
@@ -0,0 +1,43 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func promptKitTestString(value string) *string { return &value }
|
||||
|
||||
func promptKitTestInt(value int) *int { return &value }
|
||||
|
||||
func promptKitTestFloat(value float64) *float64 { return &value }
|
||||
|
||||
func promptKitTestInputs(t *testing.T, label string, files fstest.MapFS, root string) []configkit.Input {
|
||||
t.Helper()
|
||||
inputs, err := promptkit.FilesystemInputs(label, files, root)
|
||||
if err != nil {
|
||||
t.Fatalf("FilesystemInputs() error = %v", err)
|
||||
}
|
||||
return inputs
|
||||
}
|
||||
|
||||
func promptKitTestSources(
|
||||
backends map[string]promptkit.BackendConfig,
|
||||
profiles map[string]promptkit.ProfileConfig,
|
||||
otherInputs ...configkit.Input,
|
||||
) promptkit.ConfigurationSources {
|
||||
inputs := make([]configkit.Input, 0, len(otherInputs)+2)
|
||||
if len(backends) != 0 {
|
||||
inputs = append(inputs, configkit.Input{Source: promptkit.BackendSource("test backends", backends)})
|
||||
}
|
||||
if len(profiles) != 0 {
|
||||
inputs = append(inputs, configkit.Input{Source: promptkit.ProfileSource("test profiles", profiles)})
|
||||
}
|
||||
inputs = append(inputs, otherInputs...)
|
||||
if len(inputs) == 0 {
|
||||
return promptkit.ConfigurationSources{}
|
||||
}
|
||||
return promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: inputs}}}
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
@@ -17,17 +18,9 @@ func TestRegisterAssetsPreparesGenericPromptOffline(t *testing.T) {
|
||||
if err := RegisterAssets(registry); err != nil {
|
||||
t.Fatalf("RegisterAssets() error = %v, want nil", err)
|
||||
}
|
||||
options, err := registry.PromptKitOptions()
|
||||
if err != nil {
|
||||
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
|
||||
}
|
||||
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
||||
ID: "semantic-reconciliation-test", Endpoint: "http://127.0.0.1:1/v1", Model: "offline-test-model",
|
||||
})))
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine() error = %v, want nil", err)
|
||||
}
|
||||
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
|
||||
"semantic-reconciliation-test": promptkitfixture.Profile("http://127.0.0.1:1/v1", "offline-test-model"),
|
||||
})
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: PromptID, PromptVersion: PromptVersion, ProfileID: "semantic-reconciliation-test",
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
@@ -131,18 +132,7 @@ func prepareScenesPrompt(t *testing.T, transcript []byte, players string, party
|
||||
|
||||
func newScenesPromptEngine(t *testing.T, registry *llm.AssetRegistry) *promptkit.Engine {
|
||||
t.Helper()
|
||||
options, err := registry.PromptKitOptions()
|
||||
if err != nil {
|
||||
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
|
||||
}
|
||||
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
||||
ID: "scene-test-profile",
|
||||
Endpoint: "http://127.0.0.1:1/v1",
|
||||
Model: "scene-test-model",
|
||||
})))
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine() error = %v, want nil", err)
|
||||
}
|
||||
return engine
|
||||
return promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
|
||||
"scene-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "scene-test-model"),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
@@ -34,17 +35,9 @@ func TestPromptAssetsPrepareRequiredInputs(t *testing.T) {
|
||||
if err := RegisterPromptAssets(registry); err != nil {
|
||||
t.Fatalf("RegisterPromptAssets() error = %v, want nil", err)
|
||||
}
|
||||
options, err := registry.PromptKitOptions()
|
||||
if err != nil {
|
||||
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
|
||||
}
|
||||
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
||||
ID: "combat-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "combat-test-model",
|
||||
})))
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine() error = %v, want nil", err)
|
||||
}
|
||||
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
|
||||
"combat-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "combat-test-model"),
|
||||
})
|
||||
transcript := `{"units":[{"sentinel":"combat-transcript"}]}`
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "combat-test-profile",
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
@@ -63,18 +64,9 @@ func newEnemyEventPromptEngine(t *testing.T) *promptkit.Engine {
|
||||
if err := RegisterPromptAssets(registry); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
options, err := registry.PromptKitOptions()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
||||
ID: "dnd-extraction", Endpoint: "http://127.0.0.1:1/v1", Model: "enemy-test-model",
|
||||
})))
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return engine
|
||||
return promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
|
||||
"dnd-extraction": promptkitfixture.Profile("http://127.0.0.1:1/v1", "enemy-test-model"),
|
||||
})
|
||||
}
|
||||
|
||||
func enemyEventPromptInputs() map[string]promptkit.ArtifactRef {
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
@@ -16,17 +17,9 @@ func TestPromptAssetsPrepareItemOccurrencePrompt(t *testing.T) {
|
||||
if err := RegisterPromptAssets(registry); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
options, err := registry.PromptKitOptions()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
||||
ID: "item-occurrences-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "item-occurrences-test-model",
|
||||
})))
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
|
||||
"item-occurrences-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "item-occurrences-test-model"),
|
||||
})
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "item-occurrences-test-profile",
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
@@ -15,15 +16,9 @@ func TestRegisterPromptAssetsPreparesItemPrompt(t *testing.T) {
|
||||
if err := RegisterPromptAssets(registry); err != nil {
|
||||
t.Fatalf("RegisterPromptAssets() error = %v", err)
|
||||
}
|
||||
options, err := registry.PromptKitOptions()
|
||||
if err != nil {
|
||||
t.Fatalf("PromptKitOptions() error = %v", err)
|
||||
}
|
||||
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ID: "item-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "item-test-model"})))
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine() error = %v", err)
|
||||
}
|
||||
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
|
||||
"item-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "item-test-model"),
|
||||
})
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "item-test-profile", Inputs: map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.Inline(`{"units":[{"sentinel":"item-transcript"}]}`), "players": promptkit.Inline("item-player"), "party": promptkit.Inline(" "), "glossary": promptkit.Inline(" "),
|
||||
}})
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
@@ -15,17 +16,9 @@ func TestRegisterPromptAssetsPreparesLocationOccurrencePrompt(t *testing.T) {
|
||||
if err := RegisterPromptAssets(registry); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
options, err := registry.PromptKitOptions()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
||||
ID: "location-occurrences-test", Endpoint: "http://127.0.0.1:1/v1", Model: "test",
|
||||
})))
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
|
||||
"location-occurrences-test": promptkitfixture.Profile("http://127.0.0.1:1/v1", "test"),
|
||||
})
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "location-occurrences-test",
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
@@ -15,15 +16,9 @@ func TestRegisterPromptAssetsPreparesLocationPrompt(t *testing.T) {
|
||||
if err := RegisterPromptAssets(registry); err != nil {
|
||||
t.Fatalf("RegisterPromptAssets() error = %v", err)
|
||||
}
|
||||
options, err := registry.PromptKitOptions()
|
||||
if err != nil {
|
||||
t.Fatalf("PromptKitOptions() error = %v", err)
|
||||
}
|
||||
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ID: "location-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "location-test-model"})))
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine() error = %v", err)
|
||||
}
|
||||
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
|
||||
"location-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "location-test-model"),
|
||||
})
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "location-test-profile", Inputs: map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.Inline(`{"units":[{"sentinel":"location-transcript"}]}`), "players": promptkit.Inline("location-player"), "party": promptkit.Inline(" "), "glossary": promptkit.Inline(" "),
|
||||
}})
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
@@ -23,17 +24,9 @@ func TestRegisterPromptAssetsAndPrepareOccurrencePrompt(t *testing.T) {
|
||||
if _, err := fs.ReadFile(schemaFS, "dnd_npc_occurrences_llm.v1.json"); err != nil {
|
||||
t.Fatalf("response schema asset: %v", err)
|
||||
}
|
||||
options, err := registry.PromptKitOptions()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
||||
ID: "npc-occurrences-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "npc-occurrences-test-model",
|
||||
})))
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
|
||||
"npc-occurrences-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "npc-occurrences-test-model"),
|
||||
})
|
||||
transcript := `{"units":[{"sentinel":"occurrence-transcript"}]}`
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "npc-occurrences-test-profile",
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
@@ -16,17 +17,9 @@ func TestRegisterPromptAssetsAndPrepareNPCPrompt(t *testing.T) {
|
||||
if err := RegisterPromptAssets(registry); err != nil {
|
||||
t.Fatalf("RegisterPromptAssets() error = %v, want nil", err)
|
||||
}
|
||||
options, err := registry.PromptKitOptions()
|
||||
if err != nil {
|
||||
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
|
||||
}
|
||||
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
||||
ID: "npc-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "npc-test-model",
|
||||
})))
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine() error = %v, want nil", err)
|
||||
}
|
||||
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
|
||||
"npc-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "npc-test-model"),
|
||||
})
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "npc-test-profile",
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
@@ -16,17 +17,9 @@ func TestRegisterPromptAssetsPreparesSceneDescriptionPrompt(t *testing.T) {
|
||||
if err := RegisterPromptAssets(registry); err != nil {
|
||||
t.Fatalf("RegisterPromptAssets() error = %v, want nil", err)
|
||||
}
|
||||
options, err := registry.PromptKitOptions()
|
||||
if err != nil {
|
||||
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
|
||||
}
|
||||
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
||||
ID: "scene-description-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "scene-description-test-model",
|
||||
})))
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine() error = %v, want nil", err)
|
||||
}
|
||||
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
|
||||
"scene-description-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "scene-description-test-model"),
|
||||
})
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "scene-description-test-profile",
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
@@ -88,19 +89,9 @@ func prepareSpellsPrompt(t *testing.T, transcript []byte, players string, party
|
||||
if err := RegisterPromptAssets(registry); err != nil {
|
||||
t.Fatalf("register spell prompt assets: %v", err)
|
||||
}
|
||||
options, err := registry.PromptKitOptions()
|
||||
if err != nil {
|
||||
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
|
||||
}
|
||||
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
||||
ID: "spell-test-profile",
|
||||
Endpoint: "http://127.0.0.1:1/v1",
|
||||
Model: "spell-test-model",
|
||||
})))
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine() error = %v, want nil", err)
|
||||
}
|
||||
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
|
||||
"spell-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "spell-test-model"),
|
||||
})
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: PromptID,
|
||||
PromptVersion: SchemaVersion,
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user