From 130c4cc44718a81f7ade34705e27653d2742d89b Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Mon, 2 Nov 2026 12:54:10 +0000 Subject: [PATCH] Update diagnostic and observation contracts --- docs/roadmap/implementation.md | 7 +++++ observation_boundary_contract_test.go | 5 ++-- observation_contract_test.go | 10 +------ prepared_observation_contract_test.go | 5 ++-- smoke_integration_contract_test.go | 38 +++++++++++---------------- validate_request_contract_test.go | 20 +++++++------- 6 files changed, 37 insertions(+), 48 deletions(-) diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 879b9f2..146f3d6 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -774,6 +774,13 @@ boundary fixtures not completed by Stage 10's serialization changes. Validation: run these public contract tests offline; diagnostic operations in tests use controlled transports, not actual backend endpoints or API keys. +Completion: public diagnostic, scoped-capacity, and observation fixtures use +identified lists and retain selection, disclosure, and ownership checks without +prompt labels. Corrected remaining request-validation callers of the migrated +requirements helper. Focused offline diagnostic and prepared-observation tests +passed on Go 1.27.0; other observation tests still rely on the shared legacy +fixture assigned to the next assignment and will be rerun after its conversion. + ## Stage 19: Update Maintained Examples And Benchmarks Scope: the three `examples/go-library` workflows, their assets, and configuration- diff --git a/observation_boundary_contract_test.go b/observation_boundary_contract_test.go index 30ce635..2841e7c 100644 --- a/observation_boundary_contract_test.go +++ b/observation_boundary_contract_test.go @@ -26,9 +26,8 @@ func TestDelegatedOperationsIsolateObservations(t *testing.T) { deadline, _ := ctx.Deadline() child, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{ Source: configkit.Mapping("child execution", map[string]any{ - "profiles": map[string]any{"profile": map[string]any{"endpoint": "https://example.invalid/v1", "model": "child-model"}}, - "prompts": map[string]any{"child": map[string]any{ - "version": "1", + "profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "child-model"}}, + "prompts": []any{map[string]any{"id": "child", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "child message"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}, diff --git a/observation_contract_test.go b/observation_contract_test.go index 995b8e7..d6a8924 100644 --- a/observation_contract_test.go +++ b/observation_contract_test.go @@ -56,17 +56,9 @@ func TestRunObservationHasResolvedIdentitiesAndAccounting(t *testing.T) { if len(events) != 3 || events[0].Kind != promptkit.ExecutionStarted || events[1].Kind != promptkit.ExecutionAttempt || events[2].Kind != promptkit.ExecutionFinished { t.Fatalf("events=%+v", events) } - if events[0].PromptID != "" || events[1].PromptID != "prompt" || events[1].PromptVersion == nil || - *events[1].PromptVersion != "1" || events[1].ProfileID != "profile" || events[1].Model != "model" { + if events[0].PromptID != "" || events[1].PromptID != "prompt" || events[1].ProfileID != "profile" || events[1].Model != "model" { t.Fatalf("selection=%+v", events) } - if events[1].PromptVersion == events[2].PromptVersion { - t.Fatal("execution events share a mutable version label") - } - *events[1].PromptVersion = "mutated event" - if events[2].PromptVersion == nil || *events[2].PromptVersion != "1" { - t.Fatal("mutating one event changed another event's version label") - } if result.RunID == "" || events[0].RunID != result.RunID || events[2].RunID != result.RunID || events[2].Attempts != 1 || events[2].TransientRetries != 0 || events[2].RepairCalls != 0 || events[2].Outcome != promptkit.ExecutionOutcomeSuccess { t.Fatalf("result=%+v terminal=%+v", result, events[2]) } diff --git a/prepared_observation_contract_test.go b/prepared_observation_contract_test.go index d9321b9..2a37e27 100644 --- a/prepared_observation_contract_test.go +++ b/prepared_observation_contract_test.go @@ -134,11 +134,10 @@ func TestPreparedObservationFinishesAfterCredentialAndCapacityFailures(t *testin t.Setenv(envName, "available") engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{ Source: configkit.Mapping("credential preparation", map[string]any{ - "profiles": map[string]any{"profile": map[string]any{ + "profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model", "api_key_required": true, }}, - "prompts": map[string]any{"prepared": map[string]any{ - "version": "1", + "prompts": []any{map[string]any{"id": "prepared", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}, diff --git a/smoke_integration_contract_test.go b/smoke_integration_contract_test.go index 53c8ba5..37bad2a 100644 --- a/smoke_integration_contract_test.go +++ b/smoke_integration_contract_test.go @@ -65,11 +65,9 @@ func TestScopedCollectionRunsShareBackendCapacity(t *testing.T) { var inputs []configkit.Input for _, namespace := range []string{"north", "south"} { input, err := promptkit.CollectionInput(namespace, configkit.Mapping(namespace+" resources", map[string]any{ - "profiles": map[string]any{ - "shared": map[string]any{"backend": "capacity/backend", "model": namespace + "-model"}, - }, - "prompts": map[string]any{ - "shared": map[string]any{ + "profiles": []any{map[string]any{"id": "shared", "backend": "capacity/backend", "model": namespace + "-model"}}, + "prompts": []any{ + map[string]any{"id": "shared", "default_profile": namespace + "/shared", "messages": []any{map[string]any{"role": "user", "content": "Hello"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}, @@ -83,8 +81,8 @@ func TestScopedCollectionRunsShareBackendCapacity(t *testing.T) { inputs = append(inputs, input) } capacityInput, err := promptkit.CollectionInput("capacity", configkit.Mapping("shared capacity", map[string]any{ - "backends": map[string]any{ - "backend": map[string]any{ + "backends": []any{ + map[string]any{"id": "backend", "endpoint": "https://example.invalid/v1", "concurrency_limit": 1, "queue_capacity": 0, @@ -334,9 +332,8 @@ func TestSmokeTestUsesPrivateInferenceWithoutApplicationResourcesOrListingChange schemaSource := &failingSchemaFS{} client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: outputMarker}} input := configkit.Input{Source: configkit.Mapping("private diagnostic test", map[string]any{ - "profiles": map[string]any{"profile": map[string]any{"endpoint": "http://example.test/v1", "model": "model"}}, - "prompts": map[string]any{"application-prompt": map[string]any{ - "version": "1", + "profiles": []any{map[string]any{"id": "profile", "endpoint": "http://example.test/v1", "model": "model"}}, + "prompts": []any{map[string]any{"id": "application-prompt", "default_profile": "profile", "session_id": `application-session {{input "document"}}`, "inputs": []any{map[string]any{"name": "document", "required": true}}, @@ -374,9 +371,6 @@ func TestSmokeTestUsesPrivateInferenceWithoutApplicationResourcesOrListingChange if err != nil || report == nil || !report.Completed { t.Fatalf("structured smoke test=(%+v, %v)", report, err) } - if report.Target.PromptVersion == nil || *report.Target.PromptVersion != "1" { - t.Fatalf("smoke target label = %v, want 1", report.Target.PromptVersion) - } if reader.calls != 0 || schemaSource.openCount() != schemaOpensBefore { t.Fatalf("application resources used: artifact reads=%d schema opens before=%d after=%d", reader.calls, schemaOpensBefore, schemaSource.openCount()) } @@ -408,14 +402,12 @@ func TestSmokeTestUsesPrivateInferenceWithoutApplicationResourcesOrListingChange t.Errorf("smoke report exposed %q", private) } } - firstLabel := report.Target.PromptVersion - *firstLabel = "mutated report" + report.Target.EffectiveSettings.Model = "mutated report" secondReport, err := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{ Kind: promptkit.SmokeTestStructuredOutput, PromptID: "application-prompt", APIKey: "synthetic-smoke-key-marker", }) - if err != nil || secondReport == nil || !secondReport.Completed || secondReport.Target.PromptVersion == nil || - *secondReport.Target.PromptVersion != "1" || secondReport.Target.PromptVersion == firstLabel { - t.Fatalf("smoke target label was not independently owned: (%+v, %v)", secondReport, err) + if err != nil || secondReport == nil || !secondReport.Completed || secondReport.Target.EffectiveSettings.Model != "model" { + t.Fatalf("smoke target was not independently owned: (%+v, %v)", secondReport, err) } profilesAfter, err := engine.ListProfiles(context.Background()) if err != nil { @@ -489,14 +481,14 @@ func TestSmokeTestCrossBackendDefaultUsesEndpointOverrideThroughFacade(t *testin }, nil })} input := configkit.Input{Source: configkit.Mapping("cross-backend smoke test", map[string]any{ - "backends": map[string]any{ - "backend-a": map[string]any{ + "backends": []any{ + map[string]any{"id": "backend-a", "endpoint": "http://backend-a.example/v1", "smoke_tests": map[string]any{"default_profile": "profile-b"}, }, - "backend-b": map[string]any{"endpoint": "http://backend-b.example/v1", "concurrency_limit": 1}, + map[string]any{"id": "backend-b", "endpoint": "http://backend-b.example/v1", "concurrency_limit": 1}, }, - "profiles": map[string]any{"profile-b": map[string]any{"backend": "backend-b", "model": "model"}}, + "profiles": []any{map[string]any{"id": "profile-b", "backend": "backend-b", "model": "model"}}, })} engine, err := promptkit.NewEngine(promptkit.Config{ Sources: promptkit.ConfigurationSources{Main: &input}, HTTPClient: httpClient, @@ -544,7 +536,7 @@ func TestSmokeTestInferenceEndpointsDistinguishRedirectFromRequestComposition(t }, nil })} input := configkit.Input{Source: configkit.Mapping("redirect diagnostic", map[string]any{ - "profiles": map[string]any{"profile": map[string]any{"endpoint": base, "model": "model"}}, + "profiles": []any{map[string]any{"id": "profile", "endpoint": base, "model": "model"}}, })} engine, err := promptkit.NewEngine(promptkit.Config{ Sources: promptkit.ConfigurationSources{Main: &input}, HTTPClient: client, diff --git a/validate_request_contract_test.go b/validate_request_contract_test.go index ff943df..76908ea 100644 --- a/validate_request_contract_test.go +++ b/validate_request_contract_test.go @@ -18,7 +18,7 @@ func TestValidateRequestMatchesPrepareContract(t *testing.T) { } newEngine := func(t *testing.T, definition map[string]any) *promptkit.Engine { t.Helper() - return requirementsEngine(t, oneVersionPrompt("synthetic", "v1", definition), profiles, + return requirementsEngine(t, onePrompt("synthetic", definition), profiles, promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls})) } @@ -92,10 +92,10 @@ func TestValidateRequestUsesSelectedTemplateBranchesAndVariablePresence(t *testi map[string]any{"name": "details", "required": true}, }, "", noValidationOutput) engine := requirementsEngine(t, map[string]any{ - "optional": oneVersionPrompt("optional", "v1", optionalDefinition)["optional"], - "required": oneVersionPrompt("required", "v1", requiredDefinition)["required"], - "direct": oneVersionPrompt("direct", "v1", contractPrompt("offline", `{{.audience}}`, nil, "", noValidationOutput))["direct"], - "index": oneVersionPrompt("index", "v1", contractPrompt("offline", `{{index . "audience"}}`, nil, "", noValidationOutput))["index"], + "optional": onePrompt("optional", optionalDefinition)["optional"], + "required": onePrompt("required", requiredDefinition)["required"], + "direct": onePrompt("direct", contractPrompt("offline", `{{.audience}}`, nil, "", noValidationOutput))["direct"], + "index": onePrompt("index", contractPrompt("offline", `{{index . "audience"}}`, nil, "", noValidationOutput))["index"], }, profiles, promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls})) cases := []struct { @@ -141,9 +141,9 @@ func TestValidateRequestPreservesResourceAndSelectionFailures(t *testing.T) { "format": "json", "validation_mode": "json_schema", "schema_path": "missing-schema.json", }) prompts := map[string]any{ - "branch": oneVersionPrompt("branch", "v1", definition)["branch"], - "profile": oneVersionPrompt("profile", "v1", profilePrompt)["profile"], - "schema": oneVersionPrompt("schema", "v1", schemaPrompt)["schema"], + "branch": onePrompt("branch", definition)["branch"], + "profile": onePrompt("profile", profilePrompt)["profile"], + "schema": onePrompt("schema", schemaPrompt)["schema"], } engine := requirementsEngine(t, prompts, profiles, promptkit.WithSchemaFS(fstest.MapFS{}, "."), @@ -177,7 +177,7 @@ func TestValidateRequestPreservesCanceledLoaderCategory(t *testing.T) { definition := contractPrompt("offline", `{{if eq .mode "detailed"}}{{input "details"}}{{end}}`, []any{ map[string]any{"name": "details", "required": false}, }, "", noValidationOutput) - engine := requirementsEngine(t, oneVersionPrompt("synthetic", "v1", definition), map[string]any{ + engine := requirementsEngine(t, onePrompt("synthetic", definition), map[string]any{ "offline": map[string]any{"endpoint": "http://example.test/v1", "model": "offline-model"}, }, promptkit.WithArtifactReader(canceledValidationArtifactReader{}), promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls})) @@ -195,7 +195,7 @@ func TestValidateRequestPreservesCanceledLoaderCategory(t *testing.T) { func TestValidateRequestForwardsProfileOverridesAndRequestSettings(t *testing.T) { var modelCalls atomic.Int64 definition := contractPrompt("partial", "base message", nil, "", noValidationOutput) - engine := requirementsEngine(t, oneVersionPrompt("synthetic", "v1", definition), map[string]any{ + engine := requirementsEngine(t, onePrompt("synthetic", definition), map[string]any{ "partial": map[string]any{"model": "partial-model"}, }, promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls})) request := promptkit.RunRequest{