Update examples and benchmarks for identified resources

This commit is contained in:
2026-11-02 12:56:05 +00:00
parent 130c4cc447
commit 0eee3ec962
17 changed files with 93 additions and 76 deletions

View File

@@ -71,7 +71,7 @@ func newAssembledWorkflowFixture(b *testing.B, name string) assembledWorkflowFix
message := "Hello {{.name}}"
response := "ok"
output := map[string]any{"format": "text", "validation_mode": "none"}
profile := map[string]any{"endpoint": "http://example.test/v1", "model": "benchmark-model"}
profile := map[string]any{"id": "profile", "endpoint": "http://example.test/v1", "model": "benchmark-model"}
options := []promptkit.Option{promptkit.WithLLMClient(assembledWorkflowClient{content: response})}
var expectedMessage string
@@ -99,9 +99,8 @@ func newAssembledWorkflowFixture(b *testing.B, name string) assembledWorkflowFix
}
config := map[string]any{
"profiles": map[string]any{"profile": profile},
"prompts": map[string]any{"workflow": map[string]any{
"version": "v1",
"profiles": []any{profile},
"prompts": []any{map[string]any{"id": "workflow",
"default_profile": "profile",
"messages": []any{map[string]any{"role": "user", "content": message}},
"output": output,
@@ -128,7 +127,7 @@ func verifyAssembledWorkflowFixture(b *testing.B, fixture assembledWorkflowFixtu
if err != nil {
b.Fatal(err)
}
if prepared.PromptID != "workflow" || prepared.PromptVersion == nil || *prepared.PromptVersion != "v1" || len(prepared.Messages) != 1 ||
if prepared.PromptID != "workflow" || len(prepared.Messages) != 1 ||
prepared.Messages[0].Content != fixture.expectedMessage {
b.Fatalf("prepared fixture output is invalid: %+v", prepared)
}

View File

@@ -38,10 +38,11 @@ func TestProfileEnvironmentLookupAcceptsNonShellNames(t *testing.T) {
}
cfg := simpleConfig("hello")
cfg.Sources.Underlays = []configkit.Underlay{{Inputs: []configkit.Input{*cfg.Sources.Main}}}
endpoint, model := "http://example.test/v1", "model"
lowerName := name
cfg.Sources.Underlays = append(cfg.Sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{
Source: promptkit.ProfileSource("default credentials", []promptkit.ProfileConfig{
{ID: "profile", APIKeyEnv: &lowerName},
{ID: "profile", Endpoint: &endpoint, Model: &model, APIKeyEnv: &lowerName},
}),
}}})
env := " " + name + " "
@@ -49,7 +50,7 @@ func TestProfileEnvironmentLookupAcceptsNonShellNames(t *testing.T) {
env = ""
}
cfg.Sources.Main = &configkit.Input{Source: promptkit.ProfileSource("credentials", []promptkit.ProfileConfig{
{ID: "profile", APIKeyEnv: &env},
{ID: "profile", Endpoint: &endpoint, Model: &model, APIKeyEnv: &env},
})}
var calls int
var auth string

View File

@@ -60,11 +60,10 @@ func newSimpleEngine(t *testing.T, content string, options ...promptkit.Option)
func simpleConfig(content string) promptkit.Config {
return promptkit.Config{Sources: promptkit.ConfigurationSources{
Main: &configkit.Input{Source: configkit.Mapping("consumer test", map[string]any{
"profiles": map[string]any{"profile": map[string]any{
"profiles": []any{map[string]any{"id": "profile",
"endpoint": "http://example.test/v1", "model": "model",
}},
"prompts": map[string]any{"prompt": map[string]any{
"version": "1",
"prompts": []any{map[string]any{"id": "prompt",
"default_profile": "profile",
"messages": []any{map[string]any{"role": "user", "content": content}},
"output": map[string]any{"format": "text", "validation_mode": "none"},

View File

@@ -803,6 +803,13 @@ Validation: run all three examples through the semantic checker once runnable.
Use a short focused construction benchmark if needed to investigate a suspected
repeated-scan regression, not to claim unmeasured performance improvements.
Completion: the three maintained examples and benchmark/shared fixtures now use
identified resources and ordered builders. The preparation example includes a
headerless item import and an inherited replacement profile. The semantic example
checker, full public contract suite, and short configuration benchmark fixture
runs passed on Go 1.27.0. The public run also verified the previously deferred
preparation and observation tests after updating their shared fixture.
## Stage 20: Document Formats And Internal Handoff
Scope: `docs/formats.md`, `docs/internal/sources.md`, and other current internal

View File

@@ -84,13 +84,15 @@ func run() (summary, error) {
return summary{}, err
}
backendID, proposalModel, assessmentModel := "offline-backend", "proposal-model", "assessment-model"
backend := promptkit.LocalBackend("https://example.invalid/v1", 1)
backend.ID = backendID
inputs = append(inputs,
configkit.Input{Source: promptkit.BackendSource("example backend", map[string]promptkit.BackendConfig{
backendID: promptkit.LocalBackend("https://example.invalid/v1", 1),
configkit.Input{Source: promptkit.BackendSource("example backend", []promptkit.BackendConfig{
backend,
})},
configkit.Input{Source: promptkit.ProfileSource("example profiles", map[string]promptkit.ProfileConfig{
"proposal-profile": {Backend: &backendID, Model: &proposalModel},
"assessment-profile": {Backend: &backendID, Model: &assessmentModel},
configkit.Input{Source: promptkit.ProfileSource("example profiles", []promptkit.ProfileConfig{
{ID: "proposal-profile", Backend: &backendID, Model: &proposalModel},
{ID: "assessment-profile", Backend: &backendID, Model: &assessmentModel},
})},
)
engine, err := promptkit.NewEngine(promptkit.Config{

View File

@@ -1,6 +1,5 @@
prompts:
example.assessment:
version: "1.0.0"
- id: example.assessment
default_profile: assessment-profile
description: Assess a structured proposal independently of schema validity.
inputs:

View File

@@ -1,6 +1,5 @@
prompts:
example.proposal:
version: "1.0.0"
- id: example.proposal
default_profile: proposal-profile
description: Propose a bounded improvement from a synthetic idea.
inputs:

View File

@@ -1,12 +1,12 @@
backends:
offline:
- id: offline
endpoint: https://review.example.invalid/v1
profiles:
audit:
- id: audit
backend: review/offline
model: review-model
prompts:
check:
- id: check
default_profile: review/audit
description: Inspect a review item without contacting a model provider.
messages:

View File

@@ -1,6 +1,10 @@
version: "1"
version: "2"
config:
imports:
- path: fragments/resources.yaml
required: true
- path: fragments/prepare.yaml
destination: [prompts]
placement: sequence-item
required: true
release_version: "1.4.0"

View File

@@ -0,0 +1,15 @@
id: prepare
default_profile: support/fast
description: Prepare a prompt without contacting a model provider.
inputs:
- name: note
required: true
content_type: text/plain
messages:
- role: system
content: Summarize the note in one sentence.
- role: user
content_file: ../messages/note.md
output:
format: text
validation_mode: basic

View File

@@ -1,28 +1,11 @@
backends:
offline:
- id: offline
endpoint: https://example.invalid/v1
profiles:
base:
- id: base
backend: support/offline
model: offline-model
timeout_seconds: 90
fast:
extends: base
- id: fast
extends: {id: base}
timeout_seconds: 60
prompts:
prepare:
version: "1.0.0"
default_profile: support/fast
description: Prepare a prompt without contacting a model provider.
inputs:
- name: note
required: true
content_type: text/plain
messages:
- role: system
content: Summarize the note in one sentence.
- role: user
content_file: ../messages/note.md
output:
format: text
validation_mode: basic

View File

@@ -1,4 +1,5 @@
# A later input intentionally replaces only the inherited profile timeout.
# A later input replaces the profile with an inherited variant.
profiles:
fast:
- id: fast
extends: {id: base}
timeout_seconds: 30

View File

@@ -12,7 +12,6 @@ import (
type summary struct {
PromptID string `json:"prompt_id"`
PromptVersion *string `json:"prompt_version,omitempty"`
SelectedProfile string `json:"selected_profile"`
Model string `json:"model"`
MessageCount int `json:"message_count"`
@@ -70,9 +69,7 @@ func main() {
if err != nil {
exit(err)
}
if _, found := view.Explain(configkit.Keys(
"collections", "support", "profiles", "fast", "timeout_seconds",
)); !found {
if _, found := view.Explain(configkit.Keys("collections", "support", "profiles").Index(1).Key("timeout_seconds")); !found {
exit(fmt.Errorf("collection profile override was not found"))
}
requirements, err := scoped.InspectRequirements(ctx, promptkit.RequirementInspectionRequest{
@@ -148,7 +145,6 @@ func main() {
encoder.SetIndent("", " ")
if err := encoder.Encode(summary{
PromptID: prepared.PromptID,
PromptVersion: prepared.PromptVersion,
SelectedProfile: prepared.SelectedProfileID,
Model: prepared.EffectiveModelParams.Model,
MessageCount: len(prepared.Messages),

View File

@@ -41,8 +41,8 @@ type summary struct {
func main() {
endpoint, model := "https://example.invalid/v1", "offline-model"
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("example profile", map[string]promptkit.ProfileConfig{
"offline-example": {Endpoint: &endpoint, Model: &model},
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("example profile", []promptkit.ProfileConfig{
{ID: "offline-example", Endpoint: &endpoint, Model: &model},
})}}}},
Main: &configkit.Input{Source: configkit.File("examples/go-library/run/prompt.yaml")},
}}, promptkit.WithLLMClient(deterministicClient{}))

View File

@@ -1,6 +1,5 @@
prompts:
example.run:
version: "1.0.0"
- id: example.run
default_profile: offline-example
description: Run a prompt with a deterministic injected model client.
inputs:

View File

@@ -31,13 +31,26 @@ type configurationShape struct {
func benchmarkPrompt(profileID string) map[string]any {
return map[string]any{
"version": "1",
"default_profile": profileID,
"messages": []any{map[string]any{"role": "user", "content": "Hello {{.name}}"}},
"output": map[string]any{"format": "text", "validation_mode": "none"},
}
}
func benchmarkRecords(values map[string]any) []any {
records := make([]any, 0, len(values))
for _, id := range sortedKeys(values) {
value := values[id].(map[string]any)
record := make(map[string]any, len(value)+1)
for key, field := range value {
record[key] = field
}
record["id"] = id
records = append(records, record)
}
return records
}
func independentDefinitions(count int) configurationShape {
profiles := make(map[string]any, count)
prompts := make(map[string]any, count)
@@ -48,7 +61,7 @@ func independentDefinitions(count int) configurationShape {
prompts[promptID] = benchmarkPrompt(profileID)
}
return configurationShape{
main: configkit.Input{Source: configkit.Mapping("independent definitions", map[string]any{"profiles": profiles, "prompts": prompts})},
main: configkit.Input{Source: configkit.Mapping("independent definitions", map[string]any{"profiles": benchmarkRecords(profiles), "prompts": benchmarkRecords(prompts)})},
inputEntries: count * 2,
effectiveEntries: count * 2,
tiers: 1,
@@ -71,7 +84,7 @@ func fileBackedDefinitions(count int) configurationShape {
})
return configurationShape{
main: configkit.Input{
Source: configkit.Mapping("file-backed definitions", map[string]any{"profiles": profiles, "prompts": prompts}),
Source: configkit.Mapping("file-backed definitions", map[string]any{"profiles": benchmarkRecords(profiles), "prompts": benchmarkRecords(prompts)}),
Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: ".", BaseDir: "assets"},
},
inputEntries: count * 2, effectiveEntries: count * 2, tiers: 1,
@@ -85,12 +98,12 @@ func sharedBaseDefinitions(count int) configurationShape {
"extra_params": map[string]any{"provider": map[string]any{"mode": "shared", "limit": 4}},
}
for index := range count {
profiles[fmt.Sprintf("child-%03d", index)] = map[string]any{"extends": "base", "temperature": float64(index%10) / 10}
profiles[fmt.Sprintf("child-%03d", index)] = map[string]any{"extends": map[string]any{"id": "base"}, "temperature": float64(index%10) / 10}
}
return configurationShape{
main: configkit.Input{Source: configkit.Mapping("inherited definitions", map[string]any{
"profiles": profiles,
"prompts": map[string]any{"shared": benchmarkPrompt(fmt.Sprintf("child-%03d", count-1))},
"profiles": benchmarkRecords(profiles),
"prompts": benchmarkRecords(map[string]any{"shared": benchmarkPrompt(fmt.Sprintf("child-%03d", count-1))}),
})},
inputEntries: count + 2,
effectiveEntries: count + 2,
@@ -106,19 +119,19 @@ func layeredDefinitions(count int) configurationShape {
for index := range count {
id := fmt.Sprintf("profile-%03d", index)
base[id] = map[string]any{"endpoint": "https://example.test/v1", "model": "base-model"}
operator[id] = map[string]any{"model": "operator-model"}
programmatic[id] = map[string]any{"extra_params": map[string]any{"provider": map[string]any{"mode": "programmatic"}}}
final[id] = map[string]any{"temperature": float64(index%10) / 10}
operator[id] = map[string]any{"endpoint": "https://example.test/v1", "model": "operator-model"}
programmatic[id] = map[string]any{"endpoint": "https://example.test/v1", "model": "operator-model", "extra_params": map[string]any{"provider": map[string]any{"mode": "programmatic"}}}
final[id] = map[string]any{"endpoint": "https://example.test/v1", "model": "operator-model", "extra_params": map[string]any{"provider": map[string]any{"mode": "programmatic"}}, "temperature": float64(index%10) / 10}
}
return configurationShape{
underlays: []configkit.Underlay{
{Inputs: []configkit.Input{{Source: configkit.Mapping("fallback", map[string]any{"profiles": base})}}},
{Inputs: []configkit.Input{{Source: configkit.Mapping("operator", map[string]any{"profiles": operator})}}},
{Inputs: []configkit.Input{{Source: configkit.Mapping("programmatic", map[string]any{"profiles": programmatic})}}},
{Inputs: []configkit.Input{{Source: configkit.Mapping("fallback", map[string]any{"profiles": benchmarkRecords(base)})}}},
{Inputs: []configkit.Input{{Source: configkit.Mapping("operator", map[string]any{"profiles": benchmarkRecords(operator)})}}},
{Inputs: []configkit.Input{{Source: configkit.Mapping("programmatic", map[string]any{"profiles": benchmarkRecords(programmatic)})}}},
},
main: configkit.Input{Source: configkit.Mapping("main", map[string]any{
"profiles": final,
"prompts": map[string]any{"layered": benchmarkPrompt(fmt.Sprintf("profile-%03d", count-1))},
"profiles": benchmarkRecords(final),
"prompts": benchmarkRecords(map[string]any{"layered": benchmarkPrompt(fmt.Sprintf("profile-%03d", count-1))}),
})},
inputEntries: count*4 + 1,
effectiveEntries: count + 1,
@@ -132,8 +145,8 @@ func BenchmarkCollectionFileBackedConstruction(b *testing.B) {
collections := make(map[string]any, count)
for index := range count {
collections[fmt.Sprintf("collection-%03d", index)] = map[string]any{
"prompts": map[string]any{
"example": map[string]any{
"prompts": []any{
map[string]any{"id": "example",
"messages": []any{map[string]any{"role": "user", "content_file": "body.md"}},
"output": map[string]any{"format": "text", "validation_mode": "none"},
},
@@ -240,8 +253,8 @@ func inspectionDefinitions(extraParams map[string]any) configurationShape {
}
return configurationShape{
main: configkit.Input{Source: configkit.Mapping("inspection definitions", map[string]any{
"profiles": profiles,
"prompts": prompts,
"profiles": benchmarkRecords(profiles),
"prompts": benchmarkRecords(prompts),
})},
inputEntries: 2, effectiveEntries: 2, tiers: 1,
}

View File

@@ -38,8 +38,8 @@ func BenchmarkRequirementInspection(b *testing.B) {
}, "", noValidationOutput)
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
Source: configkit.Mapping("requirement inspection benchmark", map[string]any{
"prompts": oneVersionPrompt("inspection", "1", definition),
"profiles": profiles,
"prompts": identifiedRequirementRecords(onePrompt("inspection", definition)),
"profiles": identifiedRequirementRecords(profiles),
}),
}}})
if err != nil {
@@ -51,7 +51,7 @@ func BenchmarkRequirementInspection(b *testing.B) {
PromptID: "inspection", ProfileID: profileID, Vars: map[string]string{"name": "Ada"},
}
inspection, err := engine.InspectRequirements(ctx, request)
if err != nil || inspection.PromptID != "inspection" || inspection.PromptVersion == nil || *inspection.PromptVersion != "1" ||
if err != nil || inspection.PromptID != "inspection" ||
inspection.ProfileID != profileID || !inspection.Complete || len(inspection.Inputs) != 1 ||
inspection.Inputs[0].Name != "transcript" || inspection.Inputs[0].Kind != promptkit.RequirementRequired ||
len(inspection.Variables) != 1 || inspection.Variables[0].Name != "name" || !inspection.Variables[0].Supplied {