From 0eee3ec962bcd9303e16fdff8b63636b2f32e874 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Mon, 2 Nov 2026 12:56:05 +0000 Subject: [PATCH] Update examples and benchmarks for identified resources --- assembled_workflow_benchmark_test.go | 9 ++-- construction_contract_test.go | 5 +- consumer_test_helpers_test.go | 5 +- docs/roadmap/implementation.md | 7 +++ examples/go-library/observe/main.go | 12 +++-- .../observe/prompts/assessment.yaml | 3 +- .../go-library/observe/prompts/proposal.yaml | 3 +- .../collections/review/collection.yaml | 6 +-- .../collections/support/collection.yaml | 6 ++- .../support/fragments/prepare.yaml | 15 ++++++ .../support/fragments/resources.yaml | 25 ++-------- .../support/operator-overrides.yaml | 5 +- examples/go-library/prepare/main.go | 6 +-- examples/go-library/run/main.go | 4 +- examples/go-library/run/prompt.yaml | 3 +- internal/configuration/benchmark_test.go | 49 ++++++++++++------- requirements_inspection_benchmark_test.go | 6 +-- 17 files changed, 93 insertions(+), 76 deletions(-) create mode 100644 examples/go-library/prepare/collections/support/fragments/prepare.yaml diff --git a/assembled_workflow_benchmark_test.go b/assembled_workflow_benchmark_test.go index e3cad25..ac64add 100644 --- a/assembled_workflow_benchmark_test.go +++ b/assembled_workflow_benchmark_test.go @@ -71,7 +71,7 @@ func newAssembledWorkflowFixture(b *testing.B, name string) assembledWorkflowFix message := "Hello {{.name}}" response := "ok" output := map[string]any{"format": "text", "validation_mode": "none"} - profile := map[string]any{"endpoint": "http://example.test/v1", "model": "benchmark-model"} + profile := map[string]any{"id": "profile", "endpoint": "http://example.test/v1", "model": "benchmark-model"} options := []promptkit.Option{promptkit.WithLLMClient(assembledWorkflowClient{content: response})} var expectedMessage string @@ -99,9 +99,8 @@ func newAssembledWorkflowFixture(b *testing.B, name string) assembledWorkflowFix } config := map[string]any{ - "profiles": map[string]any{"profile": profile}, - "prompts": map[string]any{"workflow": map[string]any{ - "version": "v1", + "profiles": []any{profile}, + "prompts": []any{map[string]any{"id": "workflow", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": message}}, "output": output, @@ -128,7 +127,7 @@ func verifyAssembledWorkflowFixture(b *testing.B, fixture assembledWorkflowFixtu if err != nil { b.Fatal(err) } - if prepared.PromptID != "workflow" || prepared.PromptVersion == nil || *prepared.PromptVersion != "v1" || len(prepared.Messages) != 1 || + if prepared.PromptID != "workflow" || len(prepared.Messages) != 1 || prepared.Messages[0].Content != fixture.expectedMessage { b.Fatalf("prepared fixture output is invalid: %+v", prepared) } diff --git a/construction_contract_test.go b/construction_contract_test.go index 3abecd7..44a63a4 100644 --- a/construction_contract_test.go +++ b/construction_contract_test.go @@ -38,10 +38,11 @@ func TestProfileEnvironmentLookupAcceptsNonShellNames(t *testing.T) { } cfg := simpleConfig("hello") cfg.Sources.Underlays = []configkit.Underlay{{Inputs: []configkit.Input{*cfg.Sources.Main}}} + endpoint, model := "http://example.test/v1", "model" lowerName := name cfg.Sources.Underlays = append(cfg.Sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{ Source: promptkit.ProfileSource("default credentials", []promptkit.ProfileConfig{ - {ID: "profile", APIKeyEnv: &lowerName}, + {ID: "profile", Endpoint: &endpoint, Model: &model, APIKeyEnv: &lowerName}, }), }}}) env := " " + name + " " @@ -49,7 +50,7 @@ func TestProfileEnvironmentLookupAcceptsNonShellNames(t *testing.T) { env = "" } cfg.Sources.Main = &configkit.Input{Source: promptkit.ProfileSource("credentials", []promptkit.ProfileConfig{ - {ID: "profile", APIKeyEnv: &env}, + {ID: "profile", Endpoint: &endpoint, Model: &model, APIKeyEnv: &env}, })} var calls int var auth string diff --git a/consumer_test_helpers_test.go b/consumer_test_helpers_test.go index 00e5253..545566d 100644 --- a/consumer_test_helpers_test.go +++ b/consumer_test_helpers_test.go @@ -60,11 +60,10 @@ func newSimpleEngine(t *testing.T, content string, options ...promptkit.Option) func simpleConfig(content string) promptkit.Config { return promptkit.Config{Sources: promptkit.ConfigurationSources{ Main: &configkit.Input{Source: configkit.Mapping("consumer test", map[string]any{ - "profiles": map[string]any{"profile": map[string]any{ + "profiles": []any{map[string]any{"id": "profile", "endpoint": "http://example.test/v1", "model": "model", }}, - "prompts": map[string]any{"prompt": map[string]any{ - "version": "1", + "prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": content}}, "output": map[string]any{"format": "text", "validation_mode": "none"}, diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 146f3d6..3c05dcc 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -803,6 +803,13 @@ Validation: run all three examples through the semantic checker once runnable. Use a short focused construction benchmark if needed to investigate a suspected repeated-scan regression, not to claim unmeasured performance improvements. +Completion: the three maintained examples and benchmark/shared fixtures now use +identified resources and ordered builders. The preparation example includes a +headerless item import and an inherited replacement profile. The semantic example +checker, full public contract suite, and short configuration benchmark fixture +runs passed on Go 1.27.0. The public run also verified the previously deferred +preparation and observation tests after updating their shared fixture. + ## Stage 20: Document Formats And Internal Handoff Scope: `docs/formats.md`, `docs/internal/sources.md`, and other current internal diff --git a/examples/go-library/observe/main.go b/examples/go-library/observe/main.go index 29f22f2..052b9f3 100644 --- a/examples/go-library/observe/main.go +++ b/examples/go-library/observe/main.go @@ -84,13 +84,15 @@ func run() (summary, error) { return summary{}, err } backendID, proposalModel, assessmentModel := "offline-backend", "proposal-model", "assessment-model" + backend := promptkit.LocalBackend("https://example.invalid/v1", 1) + backend.ID = backendID inputs = append(inputs, - configkit.Input{Source: promptkit.BackendSource("example backend", map[string]promptkit.BackendConfig{ - backendID: promptkit.LocalBackend("https://example.invalid/v1", 1), + configkit.Input{Source: promptkit.BackendSource("example backend", []promptkit.BackendConfig{ + backend, })}, - configkit.Input{Source: promptkit.ProfileSource("example profiles", map[string]promptkit.ProfileConfig{ - "proposal-profile": {Backend: &backendID, Model: &proposalModel}, - "assessment-profile": {Backend: &backendID, Model: &assessmentModel}, + configkit.Input{Source: promptkit.ProfileSource("example profiles", []promptkit.ProfileConfig{ + {ID: "proposal-profile", Backend: &backendID, Model: &proposalModel}, + {ID: "assessment-profile", Backend: &backendID, Model: &assessmentModel}, })}, ) engine, err := promptkit.NewEngine(promptkit.Config{ diff --git a/examples/go-library/observe/prompts/assessment.yaml b/examples/go-library/observe/prompts/assessment.yaml index 1bb75d7..285720f 100644 --- a/examples/go-library/observe/prompts/assessment.yaml +++ b/examples/go-library/observe/prompts/assessment.yaml @@ -1,6 +1,5 @@ prompts: - example.assessment: - version: "1.0.0" + - id: example.assessment default_profile: assessment-profile description: Assess a structured proposal independently of schema validity. inputs: diff --git a/examples/go-library/observe/prompts/proposal.yaml b/examples/go-library/observe/prompts/proposal.yaml index 2ba2ea9..2c686bf 100644 --- a/examples/go-library/observe/prompts/proposal.yaml +++ b/examples/go-library/observe/prompts/proposal.yaml @@ -1,6 +1,5 @@ prompts: - example.proposal: - version: "1.0.0" + - id: example.proposal default_profile: proposal-profile description: Propose a bounded improvement from a synthetic idea. inputs: diff --git a/examples/go-library/prepare/collections/review/collection.yaml b/examples/go-library/prepare/collections/review/collection.yaml index cd7da05..55c0620 100644 --- a/examples/go-library/prepare/collections/review/collection.yaml +++ b/examples/go-library/prepare/collections/review/collection.yaml @@ -1,12 +1,12 @@ backends: - offline: + - id: offline endpoint: https://review.example.invalid/v1 profiles: - audit: + - id: audit backend: review/offline model: review-model prompts: - check: + - id: check default_profile: review/audit description: Inspect a review item without contacting a model provider. messages: diff --git a/examples/go-library/prepare/collections/support/collection.yaml b/examples/go-library/prepare/collections/support/collection.yaml index 2452f38..34b6b2c 100644 --- a/examples/go-library/prepare/collections/support/collection.yaml +++ b/examples/go-library/prepare/collections/support/collection.yaml @@ -1,6 +1,10 @@ -version: "1" +version: "2" config: imports: - path: fragments/resources.yaml required: true + - path: fragments/prepare.yaml + destination: [prompts] + placement: sequence-item + required: true release_version: "1.4.0" diff --git a/examples/go-library/prepare/collections/support/fragments/prepare.yaml b/examples/go-library/prepare/collections/support/fragments/prepare.yaml new file mode 100644 index 0000000..7b3ae66 --- /dev/null +++ b/examples/go-library/prepare/collections/support/fragments/prepare.yaml @@ -0,0 +1,15 @@ +id: prepare +default_profile: support/fast +description: Prepare a prompt without contacting a model provider. +inputs: + - name: note + required: true + content_type: text/plain +messages: + - role: system + content: Summarize the note in one sentence. + - role: user + content_file: ../messages/note.md +output: + format: text + validation_mode: basic diff --git a/examples/go-library/prepare/collections/support/fragments/resources.yaml b/examples/go-library/prepare/collections/support/fragments/resources.yaml index d4f36fb..496dc18 100644 --- a/examples/go-library/prepare/collections/support/fragments/resources.yaml +++ b/examples/go-library/prepare/collections/support/fragments/resources.yaml @@ -1,28 +1,11 @@ backends: - offline: + - id: offline endpoint: https://example.invalid/v1 profiles: - base: + - id: base backend: support/offline model: offline-model timeout_seconds: 90 - fast: - extends: base + - id: fast + extends: {id: base} timeout_seconds: 60 -prompts: - prepare: - version: "1.0.0" - default_profile: support/fast - description: Prepare a prompt without contacting a model provider. - inputs: - - name: note - required: true - content_type: text/plain - messages: - - role: system - content: Summarize the note in one sentence. - - role: user - content_file: ../messages/note.md - output: - format: text - validation_mode: basic diff --git a/examples/go-library/prepare/collections/support/operator-overrides.yaml b/examples/go-library/prepare/collections/support/operator-overrides.yaml index acac9fc..7ab5063 100644 --- a/examples/go-library/prepare/collections/support/operator-overrides.yaml +++ b/examples/go-library/prepare/collections/support/operator-overrides.yaml @@ -1,4 +1,5 @@ -# A later input intentionally replaces only the inherited profile timeout. +# A later input replaces the profile with an inherited variant. profiles: - fast: + - id: fast + extends: {id: base} timeout_seconds: 30 diff --git a/examples/go-library/prepare/main.go b/examples/go-library/prepare/main.go index 171954a..4066f88 100644 --- a/examples/go-library/prepare/main.go +++ b/examples/go-library/prepare/main.go @@ -12,7 +12,6 @@ import ( type summary struct { PromptID string `json:"prompt_id"` - PromptVersion *string `json:"prompt_version,omitempty"` SelectedProfile string `json:"selected_profile"` Model string `json:"model"` MessageCount int `json:"message_count"` @@ -70,9 +69,7 @@ func main() { if err != nil { exit(err) } - if _, found := view.Explain(configkit.Keys( - "collections", "support", "profiles", "fast", "timeout_seconds", - )); !found { + if _, found := view.Explain(configkit.Keys("collections", "support", "profiles").Index(1).Key("timeout_seconds")); !found { exit(fmt.Errorf("collection profile override was not found")) } requirements, err := scoped.InspectRequirements(ctx, promptkit.RequirementInspectionRequest{ @@ -148,7 +145,6 @@ func main() { encoder.SetIndent("", " ") if err := encoder.Encode(summary{ PromptID: prepared.PromptID, - PromptVersion: prepared.PromptVersion, SelectedProfile: prepared.SelectedProfileID, Model: prepared.EffectiveModelParams.Model, MessageCount: len(prepared.Messages), diff --git a/examples/go-library/run/main.go b/examples/go-library/run/main.go index a432b9f..a4c92c8 100644 --- a/examples/go-library/run/main.go +++ b/examples/go-library/run/main.go @@ -41,8 +41,8 @@ type summary struct { func main() { endpoint, model := "https://example.invalid/v1", "offline-model" engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ - Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("example profile", map[string]promptkit.ProfileConfig{ - "offline-example": {Endpoint: &endpoint, Model: &model}, + Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("example profile", []promptkit.ProfileConfig{ + {ID: "offline-example", Endpoint: &endpoint, Model: &model}, })}}}}, Main: &configkit.Input{Source: configkit.File("examples/go-library/run/prompt.yaml")}, }}, promptkit.WithLLMClient(deterministicClient{})) diff --git a/examples/go-library/run/prompt.yaml b/examples/go-library/run/prompt.yaml index 6c5f679..328d4fd 100644 --- a/examples/go-library/run/prompt.yaml +++ b/examples/go-library/run/prompt.yaml @@ -1,6 +1,5 @@ prompts: - example.run: - version: "1.0.0" + - id: example.run default_profile: offline-example description: Run a prompt with a deterministic injected model client. inputs: diff --git a/internal/configuration/benchmark_test.go b/internal/configuration/benchmark_test.go index 2734212..3a7d3bd 100644 --- a/internal/configuration/benchmark_test.go +++ b/internal/configuration/benchmark_test.go @@ -31,13 +31,26 @@ type configurationShape struct { func benchmarkPrompt(profileID string) map[string]any { return map[string]any{ - "version": "1", "default_profile": profileID, "messages": []any{map[string]any{"role": "user", "content": "Hello {{.name}}"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}, } } +func benchmarkRecords(values map[string]any) []any { + records := make([]any, 0, len(values)) + for _, id := range sortedKeys(values) { + value := values[id].(map[string]any) + record := make(map[string]any, len(value)+1) + for key, field := range value { + record[key] = field + } + record["id"] = id + records = append(records, record) + } + return records +} + func independentDefinitions(count int) configurationShape { profiles := make(map[string]any, count) prompts := make(map[string]any, count) @@ -48,7 +61,7 @@ func independentDefinitions(count int) configurationShape { prompts[promptID] = benchmarkPrompt(profileID) } return configurationShape{ - main: configkit.Input{Source: configkit.Mapping("independent definitions", map[string]any{"profiles": profiles, "prompts": prompts})}, + main: configkit.Input{Source: configkit.Mapping("independent definitions", map[string]any{"profiles": benchmarkRecords(profiles), "prompts": benchmarkRecords(prompts)})}, inputEntries: count * 2, effectiveEntries: count * 2, tiers: 1, @@ -71,7 +84,7 @@ func fileBackedDefinitions(count int) configurationShape { }) return configurationShape{ main: configkit.Input{ - Source: configkit.Mapping("file-backed definitions", map[string]any{"profiles": profiles, "prompts": prompts}), + Source: configkit.Mapping("file-backed definitions", map[string]any{"profiles": benchmarkRecords(profiles), "prompts": benchmarkRecords(prompts)}), Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: ".", BaseDir: "assets"}, }, inputEntries: count * 2, effectiveEntries: count * 2, tiers: 1, @@ -85,12 +98,12 @@ func sharedBaseDefinitions(count int) configurationShape { "extra_params": map[string]any{"provider": map[string]any{"mode": "shared", "limit": 4}}, } for index := range count { - profiles[fmt.Sprintf("child-%03d", index)] = map[string]any{"extends": "base", "temperature": float64(index%10) / 10} + profiles[fmt.Sprintf("child-%03d", index)] = map[string]any{"extends": map[string]any{"id": "base"}, "temperature": float64(index%10) / 10} } return configurationShape{ main: configkit.Input{Source: configkit.Mapping("inherited definitions", map[string]any{ - "profiles": profiles, - "prompts": map[string]any{"shared": benchmarkPrompt(fmt.Sprintf("child-%03d", count-1))}, + "profiles": benchmarkRecords(profiles), + "prompts": benchmarkRecords(map[string]any{"shared": benchmarkPrompt(fmt.Sprintf("child-%03d", count-1))}), })}, inputEntries: count + 2, effectiveEntries: count + 2, @@ -106,19 +119,19 @@ func layeredDefinitions(count int) configurationShape { for index := range count { id := fmt.Sprintf("profile-%03d", index) base[id] = map[string]any{"endpoint": "https://example.test/v1", "model": "base-model"} - operator[id] = map[string]any{"model": "operator-model"} - programmatic[id] = map[string]any{"extra_params": map[string]any{"provider": map[string]any{"mode": "programmatic"}}} - final[id] = map[string]any{"temperature": float64(index%10) / 10} + operator[id] = map[string]any{"endpoint": "https://example.test/v1", "model": "operator-model"} + programmatic[id] = map[string]any{"endpoint": "https://example.test/v1", "model": "operator-model", "extra_params": map[string]any{"provider": map[string]any{"mode": "programmatic"}}} + final[id] = map[string]any{"endpoint": "https://example.test/v1", "model": "operator-model", "extra_params": map[string]any{"provider": map[string]any{"mode": "programmatic"}}, "temperature": float64(index%10) / 10} } return configurationShape{ underlays: []configkit.Underlay{ - {Inputs: []configkit.Input{{Source: configkit.Mapping("fallback", map[string]any{"profiles": base})}}}, - {Inputs: []configkit.Input{{Source: configkit.Mapping("operator", map[string]any{"profiles": operator})}}}, - {Inputs: []configkit.Input{{Source: configkit.Mapping("programmatic", map[string]any{"profiles": programmatic})}}}, + {Inputs: []configkit.Input{{Source: configkit.Mapping("fallback", map[string]any{"profiles": benchmarkRecords(base)})}}}, + {Inputs: []configkit.Input{{Source: configkit.Mapping("operator", map[string]any{"profiles": benchmarkRecords(operator)})}}}, + {Inputs: []configkit.Input{{Source: configkit.Mapping("programmatic", map[string]any{"profiles": benchmarkRecords(programmatic)})}}}, }, main: configkit.Input{Source: configkit.Mapping("main", map[string]any{ - "profiles": final, - "prompts": map[string]any{"layered": benchmarkPrompt(fmt.Sprintf("profile-%03d", count-1))}, + "profiles": benchmarkRecords(final), + "prompts": benchmarkRecords(map[string]any{"layered": benchmarkPrompt(fmt.Sprintf("profile-%03d", count-1))}), })}, inputEntries: count*4 + 1, effectiveEntries: count + 1, @@ -132,8 +145,8 @@ func BenchmarkCollectionFileBackedConstruction(b *testing.B) { collections := make(map[string]any, count) for index := range count { collections[fmt.Sprintf("collection-%03d", index)] = map[string]any{ - "prompts": map[string]any{ - "example": map[string]any{ + "prompts": []any{ + map[string]any{"id": "example", "messages": []any{map[string]any{"role": "user", "content_file": "body.md"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}, }, @@ -240,8 +253,8 @@ func inspectionDefinitions(extraParams map[string]any) configurationShape { } return configurationShape{ main: configkit.Input{Source: configkit.Mapping("inspection definitions", map[string]any{ - "profiles": profiles, - "prompts": prompts, + "profiles": benchmarkRecords(profiles), + "prompts": benchmarkRecords(prompts), })}, inputEntries: 2, effectiveEntries: 2, tiers: 1, } diff --git a/requirements_inspection_benchmark_test.go b/requirements_inspection_benchmark_test.go index 0b7749a..52b137b 100644 --- a/requirements_inspection_benchmark_test.go +++ b/requirements_inspection_benchmark_test.go @@ -38,8 +38,8 @@ func BenchmarkRequirementInspection(b *testing.B) { }, "", noValidationOutput) engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{ Source: configkit.Mapping("requirement inspection benchmark", map[string]any{ - "prompts": oneVersionPrompt("inspection", "1", definition), - "profiles": profiles, + "prompts": identifiedRequirementRecords(onePrompt("inspection", definition)), + "profiles": identifiedRequirementRecords(profiles), }), }}}) if err != nil { @@ -51,7 +51,7 @@ func BenchmarkRequirementInspection(b *testing.B) { PromptID: "inspection", ProfileID: profileID, Vars: map[string]string{"name": "Ada"}, } inspection, err := engine.InspectRequirements(ctx, request) - if err != nil || inspection.PromptID != "inspection" || inspection.PromptVersion == nil || *inspection.PromptVersion != "1" || + if err != nil || inspection.PromptID != "inspection" || inspection.ProfileID != profileID || !inspection.Complete || len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "transcript" || inspection.Inputs[0].Kind != promptkit.RequirementRequired || len(inspection.Variables) != 1 || inspection.Variables[0].Name != "name" || !inspection.Variables[0].Supplied {