From ea63a7b0ffe3ec8ed6c90fdb7013cbc5b6ccd72c Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Fri, 6 Nov 2026 15:26:16 +0000 Subject: [PATCH] Verify resource inspection isolation and selection --- docs/roadmap/implementation.md | 4 ++ namespaces_contract_test.go | 20 +++++++ resource_isolation_test.go | 101 +++++++++++++++++++++++++++++++++ 3 files changed, 125 insertions(+) create mode 100644 resource_isolation_test.go diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 6066daf..dd5f2c5 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -438,6 +438,10 @@ add narrowly scoped encoding support; do not move adaptation into consumers. ## Stage 8: Verify Selection, Isolation, And Offline Boundaries +**Completed:** Public namespace, offline reference, ownership, concurrent +inspection, execution fallback, and retained null/provenance checks pass under +the race detector. Existing captured-content and requirement tests remain in use. + **Scope:** Public behavior risks not already covered by the migrated suite. 1. Extend existing namespace tests for backend inspection: scoped slashless ID, diff --git a/namespaces_contract_test.go b/namespaces_contract_test.go index 2cb6bdb..fb1821e 100644 --- a/namespaces_contract_test.go +++ b/namespaces_contract_test.go @@ -87,6 +87,26 @@ func TestWithNamespacesRoutesOrdinarySelectionsAndPreservesViews(t *testing.T) { } } +func TestInspectBackendNamespaceSelection(t *testing.T) { + engine := newNamespacedContractEngine(t, &fakeLLMClient{}) + scoped, err := engine.WithNamespaces(promptkit.Namespaces{Backend: "alpha"}) + if err != nil { + t.Fatal(err) + } + for id, want := range map[string]string{"backend": "alpha/backend", "beta/backend": "beta/backend"} { + got, err := scoped.InspectBackend(context.Background(), id) + if err != nil || got.BackendID != want { + t.Errorf("backend %q = %+v, %v", id, got, err) + } + } + if _, err := scoped.InspectBackend(context.Background(), "openrouter"); !errors.Is(err, promptkit.ErrBackendNotFound) { + t.Fatalf("unexpected fallback: %v", err) + } + if got, err := engine.InspectBackend(context.Background(), "openrouter"); err != nil || got.BackendID != "openrouter" { + t.Fatalf("bundled backend = %+v, %v", got, err) + } +} + func TestWithNamespacesHasNoFallbackAndValidatesAtTheCorrectBoundary(t *testing.T) { engine := newNamespacedContractEngine(t, &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "reply"}}) scoped, err := engine.WithNamespaces(promptkit.Namespaces{Prompt: "alpha", Profile: "alpha"}) diff --git a/resource_isolation_test.go b/resource_isolation_test.go new file mode 100644 index 0000000..3729eed --- /dev/null +++ b/resource_isolation_test.go @@ -0,0 +1,101 @@ +package promptkit_test + +import ( + "context" + "strings" + "sync" + "testing" + + "go.maximumdirect.net/configkit" + "go.maximumdirect.net/promptkit" +) + +func TestResourceInspectionIsolationAndNullInheritance(t *testing.T) { + client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "unexpected"}} + engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{Source: configkit.Mapping("resources", map[string]any{ + "backends": []any{map[string]any{"id": "local", "endpoint": "https://backend.test/v1", "api_key_env": "UNSET_INSPECTION_KEY", "extra_params": map[string]any{"nested": map[string]any{"array": []any{"captured"}}}, "smoke_tests": map[string]any{"default_profile": "missing", "reachability": map[string]any{}}}}, + "profiles": []any{map[string]any{"id": "parent", "backend": "local", "endpoint": "https://parent.test/v1", "extra_params": map[string]any{"parent": true}, "model": "model"}, map[string]any{"id": "child", "extends": map[string]any{"id": "parent"}, "endpoint": nil, "extra_params": nil, "api_key_env": nil}, map[string]any{"id": "partial"}}, + "prompts": []any{map[string]any{"id": "reference", "default_profile": "missing", "messages": []any{map[string]any{"role": "user", "content": "{{.missing}}"}}, "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "missing.json"}}, map[string]any{"id": "ready", "inputs": []any{map[string]any{"name": "x", "required": false}}, "messages": []any{map[string]any{"role": "user", "content": "hello", "cache_control": map[string]any{"type": "ephemeral", "ttl": ""}}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}}, + })}}}, promptkit.WithLLMClient(client), promptkit.WithBundledOpenRouter(false)) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + if got, err := engine.InspectPrompt(ctx, "reference"); err != nil || got.DefaultProfileID == nil || *got.DefaultProfileID != "missing" || got.OutputContract.SchemaPath == nil || *got.OutputContract.SchemaPath != "missing.json" || got.Messages[0].Content != "{{.missing}}" { + t.Fatalf("reference = %+v, %v", got, err) + } + if got, err := engine.InspectProfile(ctx, "partial"); err != nil || got.Endpoint != nil || got.Model != nil { + t.Fatalf("partial = %+v, %v", got, err) + } + profile, err := engine.InspectProfile(ctx, "child") + if err != nil || profile.Endpoint != nil || profile.ExtraParams != nil || profile.APIKeyEnv != nil { + t.Fatalf("child = %+v, %v", profile, err) + } + view, err := engine.InspectConfiguration() + if err != nil { + t.Fatal(err) + } + for _, field := range []string{"endpoint", "extra_params", "api_key_env"} { + path := configkit.Keys("profiles").Index(1).Key(field) + explanation, ok := view.Explain(path) + if !ok || !strings.Contains(explanation.Effective.YAML, "null") || len(explanation.History) == 0 { + t.Errorf("%s provenance = %+v, %t", field, explanation, ok) + } + } + backend, err := engine.InspectBackend(ctx, "local") + if err != nil { + t.Fatal(err) + } + if backend.SmokeTests == nil || backend.SmokeTests.DefaultProfile == nil || *backend.SmokeTests.DefaultProfile != "missing" { + t.Fatalf("diagnostics = %+v", backend) + } + *backend.APIKeyEnv = "MUTATED" + *backend.SmokeTests.DefaultProfile = "mutated" + (*backend.ExtraParams)["nested"].(map[string]any)["array"].([]any)[0] = "mutated" + *profile.Model = "mutated" + prompt, err := engine.InspectPrompt(ctx, "ready") + if err != nil { + t.Fatal(err) + } + (*prompt.Inputs)[0].Name = "changed" + *(*prompt.Inputs)[0].Required = true + prompt.Messages[0].Content = "changed" + *prompt.Messages[0].CacheControl.TTL = "changed" + again, err := engine.InspectBackend(ctx, "local") + if err != nil || *again.APIKeyEnv != "UNSET_INSPECTION_KEY" || *again.SmokeTests.DefaultProfile != "missing" || (*again.ExtraParams)["nested"].(map[string]any)["array"].([]any)[0] != "captured" { + t.Fatalf("backend mutation leaked: %+v, %v", again, err) + } + againPrompt, err := engine.InspectPrompt(ctx, "ready") + if err != nil || (*againPrompt.Inputs)[0].Name != "x" || *(*againPrompt.Inputs)[0].Required || againPrompt.Messages[0].Content != "hello" || *againPrompt.Messages[0].CacheControl.TTL != "" { + t.Fatalf("prompt mutation leaked: %+v, %v", againPrompt, err) + } + prepared, err := engine.Prepare(ctx, promptkit.RunRequest{PromptID: "ready", ProfileID: "child"}) + if err != nil || prepared.EffectiveModelParams.Endpoint != "https://backend.test/v1" || prepared.EffectiveModelParams.APIKeyEnv != "UNSET_INSPECTION_KEY" || prepared.EffectiveModelParams.Model != "model" || prepared.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["array"].([]any)[0] != "captured" { + t.Fatalf("prepared = %+v, %v", prepared, err) + } + if len(client.requests) != 0 { + t.Fatal("inspection invoked provider") + } + var wg sync.WaitGroup + for i := 0; i < 12; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 20; j++ { + b, err := engine.InspectBackend(ctx, "local") + if err != nil { + t.Error(err) + return + } + (*b.ExtraParams)["nested"].(map[string]any)["array"].([]any)[0] = "local" + p, err := engine.InspectPrompt(ctx, "ready") + if err != nil { + t.Error(err) + return + } + p.Messages[0].Content = "local" + } + }() + } + wg.Wait() +}