diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 2b65dddc..2ea7f0f8 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -360,6 +360,12 @@ Prompt size: mechanical consumer fixture sweep, no new domain behavior. ## Stage 6 — Prove Native Operator Composition And Credential Policy +> Completion: complete. The source-composition tests exercise imported combined +> operator documents through runtime and preflight, layered and strict-group +> behavior, local-backend precedence, and optional/required synthetic +> credentials. `GOWORK=off go test ./internal/framework/llm ./internal/cli` and +> `git diff --check` pass. + Goal: verify accepted unified sources through the shared boundary. Inspect source assembly, profile/preflight tests, and adapter source tests. diff --git a/internal/framework/llm/promptkit_client_test.go b/internal/framework/llm/promptkit_client_test.go index eb85d6d5..6b2f1525 100644 --- a/internal/framework/llm/promptkit_client_test.go +++ b/internal/framework/llm/promptkit_client_test.go @@ -20,6 +20,7 @@ import ( "testing/fstest" "time" + configkit "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/promptkit" @@ -880,7 +881,12 @@ func TestPromptKitClientUsesConfiguredLocalBackend(t *testing.T) { defer server.Close() profilePath := filepath.Join(t.TempDir(), "profiles.yml") - if err := os.WriteFile(profilePath, []byte(`profiles: + if err := os.WriteFile(profilePath, []byte(`backends: + local: + endpoint: http://operator.example.test/v1 + concurrency_limit: 9 + queue_capacity: 1 +profiles: local-profile: backend: local model: local-model @@ -899,6 +905,27 @@ func TestPromptKitClientUsesConfiguredLocalBackend(t *testing.T) { if err != nil { t.Fatalf("NewPromptKitClient() error = %v, want nil", err) } + configuration, err := client.engine.InspectConfiguration() + if err != nil { + t.Fatalf("InspectConfiguration() error = %v", err) + } + for _, test := range []struct { + path string + want string + }{ + {path: "/backends/local/endpoint", want: server.URL + "/v1"}, + {path: "/backends/local/concurrency_limit", want: "2"}, + {path: "/backends/local/queue_capacity", want: "1"}, + } { + path, err := configkit.ParsePath(test.path) + if err != nil { + t.Fatal(err) + } + explanation, found := configuration.Explain(path) + if !found || !explanation.Effective.Present || !strings.Contains(explanation.Effective.YAML, test.want) { + t.Fatalf("effective local setting %s = %+v, want %q", test.path, explanation.Effective, test.want) + } + } localBackend.Endpoint = "http://127.0.0.1:1/v1" request := contracts.StructuredCompletionRequest{ @@ -929,14 +956,22 @@ func TestPromptKitClientUsesConfiguredLocalBackend(t *testing.T) { t.Fatalf("profile manifests = %#v, want local backend", manifests) } + missingRegistrationPath := filepath.Join(t.TempDir(), "profiles.yml") + if err := os.WriteFile(missingRegistrationPath, []byte(`profiles: + local-profile: + backend: local + model: local-model +`), 0o600); err != nil { + t.Fatal(err) + } clientWithoutRegistration, err := NewPromptKitClient(PromptKitClientConfig{ Assets: newTestPromptKitAssets(t), - ProfileFile: profilePath, + ProfileFile: missingRegistrationPath, }) if clientWithoutRegistration != nil || err == nil || !errors.Is(err, promptkit.ErrInvalidConfig) { t.Fatalf("NewPromptKitClient() without local registration = (%v, %v), want invalid configuration", clientWithoutRegistration, err) } - if strings.Contains(err.Error(), "local-profile") || strings.Contains(err.Error(), profilePath) { + if strings.Contains(err.Error(), "local-profile") || strings.Contains(err.Error(), missingRegistrationPath) { t.Fatalf("configuration error exposes profile details: %v", err) } if providerCalls.Load() != 1 { diff --git a/internal/framework/llm/promptkit_sources_integration_test.go b/internal/framework/llm/promptkit_sources_integration_test.go new file mode 100644 index 00000000..c240f23a --- /dev/null +++ b/internal/framework/llm/promptkit_sources_integration_test.go @@ -0,0 +1,270 @@ +package llm + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + + configkit "gitea.maximumdirect.net/eric/ConfigKit" + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/promptkit" +) + +func TestPromptKitOperatorDocumentComposesForInspectionAndExecution(t *testing.T) { + for _, name := range []string{ + "NOTARIUS_TEST_BACKEND_KEY", + "NOTARIUS_TEST_PROFILE_KEY", + "NOTARIUS_TEST_CATALOG_KEY", + } { + t.Setenv(name, "") + } + operatorDir := t.TempDir() + profilePath := filepath.Join(operatorDir, "operator.yaml") + if err := os.WriteFile(filepath.Join(operatorDir, "inherited.yaml"), []byte(`profiles: + inherited-base: + backend: custom-backend + model: imported-model + temperature: 0.7 + service_tier: flex + extra_params: + inherited: retained + nested: + inherited: true +`), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(profilePath, []byte(`configkit: + imports: + - path: inherited.yaml +backends: + custom-backend: + endpoint: http://127.0.0.1:1/v1 + api_key_env: NOTARIUS_TEST_BACKEND_KEY + concurrency_limit: 1 + queue_capacity: 0 + openrouter: + api_key_env: NOTARIUS_TEST_CATALOG_KEY +profiles: + custom-profile: + extends: inherited-base + model: operator-model + temperature: 0 + api_key_required: false + service_tier: null + extra_params: null + empty-map-profile: + extends: inherited-base + extra_params: {} + api_key_required: false + profile-key-profile: + backend: custom-backend + model: profile-key-model + api_key_env: NOTARIUS_TEST_PROFILE_KEY + api_key_required: false + required-profile: + extends: inherited-base + api_key_env: NOTARIUS_TEST_PROFILE_KEY + api_key_required: true + catalog-profile: + backend: openrouter + model: openai/gpt-5.6-luna + api_key_required: false +prompts: + adapter.test: + v1: + output: + repair_attempts: 1 +`), 0o600); err != nil { + t.Fatal(err) + } + + assets := newTestPromptKitAssets(t) + fake := &fakePromptKitLLM{} + client, err := NewPromptKitClient(PromptKitClientConfig{ + Assets: assets, + ProfileFile: profilePath, + EngineOptions: []promptkit.Option{ + promptkit.WithLLMClient(fake), + }, + }) + if err != nil { + t.Fatalf("NewPromptKitClient() error = %v", err) + } + inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{ + Source: PromptKitProfileSourceConfig{ProfileFile: profilePath}, + Assets: assets, + }) + if err != nil { + t.Fatalf("NewPromptKitProfileInspector() error = %v", err) + } + + for _, test := range []struct { + id string + backend string + model string + keyEnv string + required bool + }{ + {id: "custom-profile", backend: "custom-backend", model: "operator-model", keyEnv: "NOTARIUS_TEST_BACKEND_KEY"}, + {id: "inherited-base", backend: "custom-backend", model: "imported-model", keyEnv: "NOTARIUS_TEST_BACKEND_KEY"}, + {id: "empty-map-profile", backend: "custom-backend", model: "imported-model", keyEnv: "NOTARIUS_TEST_BACKEND_KEY"}, + {id: "profile-key-profile", backend: "custom-backend", model: "profile-key-model", keyEnv: "NOTARIUS_TEST_PROFILE_KEY"}, + {id: "required-profile", backend: "custom-backend", model: "imported-model", keyEnv: "NOTARIUS_TEST_PROFILE_KEY", required: true}, + {id: "catalog-profile", backend: promptkit.BackendOpenRouter, model: "openai/gpt-5.6-luna", keyEnv: "NOTARIUS_TEST_CATALOG_KEY"}, + } { + inspection, err := inspector.InspectProfile(context.Background(), test.id) + if err != nil { + t.Fatalf("InspectProfile(%q) error = %v", test.id, err) + } + if inspection.ProfileID != test.id || inspection.BackendID != test.backend || inspection.Model != test.model || + inspection.CredentialEnvironment != test.keyEnv || inspection.CredentialRequired != test.required { + t.Fatalf("InspectProfile(%q) = %#v", test.id, inspection) + } + } + + var out map[string]any + response, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ + PromptID: "adapter.test", + ProfileID: "custom-profile", + SessionID: "source-composition-test", + Inputs: contracts.LLMInputSet{ + "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), + }, + }, &out) + if err != nil { + t.Fatalf("CompleteStructured() with an unset optional credential error = %v", err) + } + if response.Debug == nil || response.Debug.Prompt == nil { + t.Fatalf("completion debug = %#v, want captured prompt", response.Debug) + } + debug := response.Debug.Prompt + if debug.SelectedBackendID != "custom-backend" || debug.SelectedProfileID != "custom-profile" { + t.Fatalf("runtime selection = backend %q, profile %q", debug.SelectedBackendID, debug.SelectedProfileID) + } + if got := debug.EffectiveModelParams["model"]; got != "operator-model" { + t.Fatalf("runtime model = %#v, want operator-model", got) + } + if got, ok := debug.EffectiveModelParams["temperature"].(float64); !ok || got != 0 { + t.Fatalf("runtime temperature = %#v, want explicit zero", debug.EffectiveModelParams["temperature"]) + } + if got := debug.EffectiveModelParams["service_tier"]; got == "flex" { + t.Fatalf("explicit null retained inherited service tier: %#v", got) + } + if got, ok := debug.EffectiveModelParams["extra_params"]; ok && got != nil { + t.Fatalf("explicit null retained inherited provider map: %#v", got) + } + if got := debug.OutputContract["repair_attempts"]; got != float64(1) { + t.Fatalf("overlaid output repair attempts = %#v, want 1", got) + } + if len(debug.Messages) == 0 || !strings.Contains(debug.Messages[0].Content, "Transcript:") { + t.Fatalf("rendered prompt messages = %#v", debug.Messages) + } + if got := atomic.LoadInt32(&fake.calls); got != 1 { + t.Fatalf("provider calls after optional-credential execution = %d, want 1", got) + } + + _, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ + PromptID: "adapter.test", + ProfileID: "required-profile", + SessionID: "source-composition-test", + Inputs: contracts.LLMInputSet{ + "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), + }, + }, &out) + if err == nil || !strings.Contains(err.Error(), "api_key_env points to an unset environment variable") { + t.Fatalf("CompleteStructured() with required unset credential error = %v, want native missing-environment failure", err) + } + if got := atomic.LoadInt32(&fake.calls); got != 1 { + t.Fatalf("provider calls after required-credential failure = %d, want 1", got) + } + _, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ + PromptID: "adapter.test", + ProfileID: "empty-map-profile", + SessionID: "source-composition-test", + Inputs: contracts.LLMInputSet{ + "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), + }, + }, &out) + if err != nil { + t.Fatalf("CompleteStructured() with empty overlay map error = %v", err) + } + if got := fake.lastRequest().Target.ExtraParams["inherited"]; got != "retained" { + t.Fatalf("empty provider-map overlay result = %#v, want inherited entry", fake.lastRequest().Target.ExtraParams) + } +} + +func TestPromptKitProfileSourceGroupsPreserveNativeLayerRules(t *testing.T) { + t.Run("directory group conflicts", func(t *testing.T) { + directory := t.TempDir() + for name, model := range map[string]string{"first.yaml": "first-model", "second.yaml": "second-model"} { + content := "profiles:\n duplicate:\n endpoint: http://127.0.0.1:1/v1\n model: " + model + "\n" + if err := os.WriteFile(filepath.Join(directory, name), []byte(content), 0o600); err != nil { + t.Fatal(err) + } + } + _, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{ + Source: PromptKitProfileSourceConfig{ProfileDir: directory}, + }) + if err == nil { + t.Fatal("NewPromptKitProfileInspector() error = nil, want same-group conflict") + } + }) + + t.Run("later group repairs an invalid lower field and overlays supplied values", func(t *testing.T) { + badEndpoint := "not an endpoint" + goodEndpoint := "http://127.0.0.1:1/v1" + lowerModel := "lower-model" + sources := promptkit.ConfigurationSources{Underlays: []configkit.Underlay{ + {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("lower source", map[string]promptkit.ProfileConfig{ + "layered-profile": {Endpoint: &badEndpoint, Model: &lowerModel}, + })}}}, + {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("upper source", map[string]promptkit.ProfileConfig{ + "layered-profile": {Endpoint: &goodEndpoint}, + })}}}, + }} + inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{Sources: sources}) + if err != nil { + t.Fatalf("NewPromptKitProfileInspector() error = %v, want repaired final layer", err) + } + inspection, err := inspector.InspectProfile(context.Background(), "layered-profile") + if err != nil || inspection.Model != lowerModel { + t.Fatalf("InspectProfile() = (%#v, %v), want lower model retained", inspection, err) + } + }) + + t.Run("empty and missing selected directories", func(t *testing.T) { + if _, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{ + Source: PromptKitProfileSourceConfig{ProfileDir: filepath.Join(t.TempDir(), "missing")}, + }); err == nil { + t.Fatal("missing selected directory was accepted") + } + empty := t.TempDir() + inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{ + Source: PromptKitProfileSourceConfig{ProfileDir: empty}, + }) + if err != nil { + t.Fatalf("empty selected directory error = %v, want no contribution", err) + } + if _, err := inspector.InspectProfile(context.Background(), "not-present"); err == nil { + t.Fatal("InspectProfile() in empty directory error = nil, want unknown profile") + } + }) + + t.Run("unused invalid final profile fails construction", func(t *testing.T) { + badEndpoint := "not an endpoint" + model := "unused-model" + sources := promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{ + Source: promptkit.ProfileSource("unused invalid profile", map[string]promptkit.ProfileConfig{ + "unused-profile": {Endpoint: &badEndpoint, Model: &model}, + }), + }}}}} + _, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{Sources: sources}) + if err == nil || !errors.Is(err, promptkit.ErrInvalidConfig) { + t.Fatalf("NewPromptKitProfileInspector() error = %v, want invalid final configuration", err) + } + }) +}