diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 88c37abb..4e943ac6 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -329,6 +329,17 @@ and diff status `3`. Test semantics, not entire upstream help text. ## Stage 7: Install ckprompt And Reusable Run Overrides +**Status:** Complete. Installed native resource listings/inspection, requirements, +request validation/preparation, namespace selection, and explicit smoke operations +with ckprompt's builders and scoped convenience aliases. Added reusable native +profile/session/execution flag input and request application, including omitted +versus empty reasoning and native transient-retry overrides. Removed the copied +CLI profile-inspection path and replaced its tests with native command coverage. +The isolated real-kit override test, build-info/asset tests, and +`git diff --check` pass. Assembled CLI inspection/preview/fake-provider smoke tests +remain blocked by the recorded legacy dependency/caller compilation blockers; +their results are not claimed passing. Full pipeline binding remains Stage 33. + **Goal:** adopt the full selected native diagnostics and execution input suite. **Inspect/change:** CLI composition from Stage 6, old `promptkit_profiles.go`, and diff --git a/go.mod b/go.mod index 955b5a53..3611ef9c 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( gitea.maximumdirect.net/eric/promptkit v0.12.0 github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 go.maximumdirect.net/ckconfig v0.10.0 + go.maximumdirect.net/ckprompt v0.11.0 go.maximumdirect.net/commandkit v0.7.0 go.maximumdirect.net/configkit v1.4.0 go.maximumdirect.net/promptkit v0.24.0 @@ -20,5 +21,4 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/spf13/cobra v1.10.2 // indirect github.com/spf13/pflag v1.1.0 // indirect - go.maximumdirect.net/ckprompt v0.11.0 // indirect ) diff --git a/internal/cli/application.go b/internal/cli/application.go index 672e4955..1590260b 100644 --- a/internal/cli/application.go +++ b/internal/cli/application.go @@ -7,14 +7,22 @@ import ( "gitea.maximumdirect.net/eric/notarius/internal/core/config" "go.maximumdirect.net/ckconfig" + "go.maximumdirect.net/ckprompt" "go.maximumdirect.net/commandkit" "go.maximumdirect.net/configkit" + "go.maximumdirect.net/promptkit" ) // NewApplication declares the native command surface. Acquisition is deferred to // the selected handler; construction performs no configuration or engine work. func NewApplication(version string, options Options) (*commandkit.App, error) { - newState := func() *nativeLoaders { return &nativeLoaders{lookupEnv: options.LookupEnv} } + newState := func() *nativeLoaders { + state := &nativeLoaders{lookupEnv: options.LookupEnv, runtime: options.LLMRuntime} + if options.LLMClient != nil { + state.engineOptions = []promptkit.Option{promptkit.WithLLMClient(options.LLMClient)} + } + return state + } load := func(ctx context.Context, selection ckconfig.Selection, _ struct{}, state *nativeLoaders) (*configkit.Result[config.Document], error) { return state.application(ctx, selection) } @@ -22,17 +30,33 @@ func NewApplication(version string, options Options) (*commandkit.App, error) { appScope.Features.Overrides = true promptScope := ckconfig.FromConfigKitSubtree("promptkit", "Native PromptKit configuration.", configkit.Keys("promptkit"), load) promptScope.Features.Overrides = true - return commandkit.New(commandkit.Identity{Name: "notarius", Version: version}, + loadEngine := func(ctx context.Context, selection ckconfig.Selection, _ struct{}, state *nativeLoaders) (*promptkit.Engine, error) { + return state.engine(ctx, selection) + } + definitions := ckprompt.FromEngine(loadEngine) + definitions.NewState = newState + definitions.CheckInput = func(selection ckconfig.Selection, _ struct{}) error { return checkConfigSelection(selection) } + definitions = ckprompt.WithRequestValidation(definitions, loadEngine, nil) + definitions = ckprompt.WithRequestPreparation(definitions, loadEngine, nil) + definitions = ckprompt.WithSmokeTesting(definitions, loadEngine, nil) + definitions = ckprompt.WithNamespaces(definitions, promptkit.Namespaces{}) + declarations := []commandkit.Declaration{ commandkit.Completion("notarius"), ckconfig.Command(ckconfig.Suite[struct{}, *nativeLoaders]{ NewState: newState, CheckInput: func(_ string, selection ckconfig.Selection, _ struct{}) error { - if selection.Config.Set && strings.TrimSpace(selection.Config.Value) == "" { - return fmt.Errorf("--config must not be empty") - } - return nil + return checkConfigSelection(selection) }, Scopes: []ckconfig.Scope[struct{}, *nativeLoaders]{appScope, promptScope}, }), - ) + } + declarations = append(declarations, ckprompt.Declarations(definitions)...) + return commandkit.New(commandkit.Identity{Name: "notarius", Version: version}, declarations...) +} + +func checkConfigSelection(selection ckconfig.Selection) error { + if selection.Config.Set && strings.TrimSpace(selection.Config.Value) == "" { + return fmt.Errorf("--config must not be empty") + } + return nil } diff --git a/internal/cli/execution_overrides.go b/internal/cli/execution_overrides.go new file mode 100644 index 00000000..b73d0b36 --- /dev/null +++ b/internal/cli/execution_overrides.go @@ -0,0 +1,38 @@ +package cli + +import ( + "go.maximumdirect.net/ckprompt" + "go.maximumdirect.net/commandkit" + "go.maximumdirect.net/promptkit" +) + +// executionOverrides is invocation input, not configuration. Keep the native +// supplied values until binding a concrete producer or validator request. +type executionOverrides struct { + Resources ckprompt.ResourceInput + Execution ckprompt.ExecutionInput +} + +func executionOverrideFlags[I any](selectInput func(*I) *executionOverrides) []commandkit.Flag[I] { + resources := ckprompt.NewResourceFlags(func(input *I) *ckprompt.ResourceInput { return &selectInput(input).Resources }) + execution := ckprompt.NewExecutionFlags(func(input *I) *ckprompt.ExecutionInput { return &selectInput(input).Execution }) + return append(execution.All(), resources.Profile, resources.SessionID) +} + +// apply leaves operation-owned prompt/input/capture fields intact. Engine +// preparation captures native override values without changing loaded settings. +func (overrides executionOverrides) apply(request promptkit.RunRequest) promptkit.RunRequest { + if overrides.Resources.Profile.Set { + request.ProfileID = overrides.Resources.Profile.Value + } + if overrides.Resources.SessionID.Set { + request.SessionID = overrides.Resources.SessionID.Value + } + if overrides.Execution.Execution != nil { + request.Execution = overrides.Execution.Execution + } + if overrides.Execution.TransientRetries != nil { + request.TransientRetries = overrides.Execution.TransientRetries + } + return request +} diff --git a/internal/cli/execution_overrides_test.go b/internal/cli/execution_overrides_test.go new file mode 100644 index 00000000..97fad52e --- /dev/null +++ b/internal/cli/execution_overrides_test.go @@ -0,0 +1,54 @@ +package cli + +import ( + "context" + "io" + "testing" + + "go.maximumdirect.net/commandkit" + "go.maximumdirect.net/promptkit" +) + +func TestRunOverrideFlagsPreserveNativeSuppliedValues(t *testing.T) { + type input struct{ Overrides executionOverrides } + var received executionOverrides + app, err := commandkit.New(commandkit.Identity{Name: "fixture"}, commandkit.NewCommand(commandkit.CommandSpec[input]{ + Name: "execute", + Flags: executionOverrideFlags(func(value *input) *executionOverrides { return &value.Overrides }), + Run: func(_ commandkit.Invocation, value input) commandkit.Outcome { + received = value.Overrides + return commandkit.Success() + }, + })) + if err != nil { + t.Fatal(err) + } + execute := func(args ...string) commandkit.Outcome { + return app.Execute(context.Background(), append([]string{"execute"}, args...), commandkit.Streams{Out: io.Discard, Err: io.Discard}) + } + if outcome := execute(); outcome.Status != 0 { + t.Fatal(outcome) + } + request := promptkit.RunRequest{PromptID: "notarius/producer", ProfileID: "bound", SessionID: "generated"} + unchanged := received.apply(request) + if unchanged.ProfileID != "bound" || unchanged.SessionID != "generated" || unchanged.Execution != nil { + t.Fatalf("omission changed the request: %#v", unchanged) + } + if outcome := execute("--llm-profile=chosen", "--llm-session-id=operator", "--llm-reasoning-effort=", "--llm-temperature=0", "--llm-transient-retries=0", `--llm-extra-params={"enabled":false}`); outcome.Status != 0 { + t.Fatal(outcome) + } + applied := received.apply(request) + if applied.PromptID != request.PromptID || applied.ProfileID != "chosen" || applied.SessionID != "operator" || applied.Execution == nil || applied.Execution.ReasoningEffort == nil || *applied.Execution.ReasoningEffort != "" || applied.Execution.Temperature == nil || *applied.Execution.Temperature != 0 || applied.TransientRetries == nil || *applied.TransientRetries != 0 || applied.Execution.ExtraParams["enabled"] != false { + t.Fatalf("native values lost: %#v", applied) + } + if request.ProfileID != "bound" || request.Execution != nil { + t.Fatal("application changed operation input") + } + // Every singleton occurrence must be valid, even if a later value parses. + if outcome := execute("--llm-temperature=invalid", "--llm-temperature=0"); outcome.Status != 2 { + t.Fatalf("invalid occurrence skipped: %#v", outcome) + } + if outcome := execute("--llm-profile=first", "--llm-profile=last"); outcome.Status != 0 || received.Resources.Profile.Value != "last" { + t.Fatalf("last supplied profile lost: %#v", outcome) + } +} diff --git a/internal/cli/promptkit_commands_test.go b/internal/cli/promptkit_commands_test.go new file mode 100644 index 00000000..7bc13bc6 --- /dev/null +++ b/internal/cli/promptkit_commands_test.go @@ -0,0 +1,92 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "path/filepath" + "strings" + "sync/atomic" + "testing" + + "go.maximumdirect.net/commandkit" + "go.maximumdirect.net/promptkit" +) + +type diagnosticClient struct{ calls atomic.Int32 } + +func (client *diagnosticClient) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + client.calls.Add(1) + return &promptkit.GenerateResponse{Content: "PROMPTKIT_OK"}, nil +} + +func TestNativePromptCommandsInspectPreviewAndSmokeExplicitly(t *testing.T) { + t.Parallel() + main := filepath.Join(t.TempDir(), "config.yml") + writeConfigKitFile(t, main, `version: 6 +promptkit: + profiles: + - id: partial + model: fixture-model + - id: ready + endpoint: https://example.invalid/v1 + model: fixture-model + api_key_required: false + prompts: + - id: preview + default_profile: ready + inputs: [{name: topic, required: true}] + messages: [{role: user, content: 'Topic: {{input "topic"}}'}] + output: {format: text, validation_mode: none} +`) + client := &diagnosticClient{} + app, err := NewApplication("development", Options{LLMClient: client}) + if err != nil { + t.Fatal(err) + } + execute := func(args ...string) (commandkit.Outcome, []byte, string) { + var out, diagnostics bytes.Buffer + args = append(args, "--config", main, "--format", "json") + outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &out, Err: &diagnostics}) + return outcome, out.Bytes(), diagnostics.String() + } + for _, args := range [][]string{ + {"promptkit", "list", "prompts", "--all-namespaces"}, + {"promptkit", "list", "profiles", "--all-namespaces"}, + {"promptkit", "list", "backends", "--all-namespaces"}, + {"promptkit", "list", "namespaces"}, + {"promptkit", "list", "collections"}, + {"promptkit", "show", "profile", "partial"}, + {"promptkit", "show", "prompt", "preview"}, + {"promptkit", "show", "backend", "openrouter"}, + {"promptkit", "requirements", "show", "preview"}, + {"promptkit", "requirements", "validate", "preview", "--llm-input-text=topic=fixture-topic"}, + // The short input spelling comes from the native scoped alias declarations. + {"promptkit", "show", "prepared", "preview", "--input=topic=fixture-topic", "--reasoning-effort=", "--transient-retries=0"}, + } { + outcome, out, diagnostics := execute(args...) + if outcome.Status != 0 || !json.Valid(out) { + t.Fatalf("%v: %#v, %s, %s", args, outcome, out, diagnostics) + } + if len(args) > 2 && args[1] == "show" && args[2] == "prepared" && !strings.Contains(string(out), "fixture-topic") { + t.Fatalf("preview lost explicit input: %s", out) + } + } + if client.calls.Load() != 0 { + t.Fatal("inspection implicitly called provider") + } + outcome, out, diagnostics := execute("promptkit", "test", "text-inference", "--llm-profile=ready") + if outcome.Status != 0 || client.calls.Load() == 0 { + t.Fatalf("explicit smoke: %#v, %s, %s", outcome, out, diagnostics) + } + var report struct { + Supported bool `json:"supported"` + Completed bool `json:"completed"` + } + if err := json.Unmarshal(out, &report); err != nil { + t.Fatal(err) + } + if !report.Supported || !report.Completed { + t.Fatalf("native smoke distinctions lost: %s", out) + } +} diff --git a/internal/cli/promptkit_profiles.go b/internal/cli/promptkit_profiles.go deleted file mode 100644 index 825714cf..00000000 --- a/internal/cli/promptkit_profiles.go +++ /dev/null @@ -1,46 +0,0 @@ -package cli - -import ( - "context" - "fmt" - - "gitea.maximumdirect.net/eric/notarius/internal/core/config" - "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" -) - -func validateExplicitPromptKitProfiles(ctx context.Context, cfg config.Config, profileIDs []string, assets *llm.AssetRegistry) error { - if len(profileIDs) == 0 { - return nil - } - inspector, err := llm.NewPromptKitProfileInspector(llm.PromptKitProfileInspectorConfig{ - Source: promptKitProfileSourceConfig(cfg), - Assets: assets, - }) - if err != nil { - return fmt.Errorf("load PromptKit profiles: %w", err) - } - for _, profileID := range profileIDs { - if _, err := inspector.InspectProfile(ctx, profileID); err != nil { - return err - } - } - return nil -} - -func promptKitProfileSourceConfig(cfg config.Config) llm.PromptKitProfileSourceConfig { - return llm.PromptKitProfileSourceConfig{ - ProfileDir: cfg.PromptKit.ProfileDir, - ProfileFile: cfg.PromptKit.ProfileFile, - LocalBackend: mapPromptKitLocalBackend(cfg.PromptKit.LocalBackend), - } -} - -func mapPromptKitLocalBackend(cfg *config.PromptKitLocalBackendConfig) *llm.PromptKitLocalBackendConfig { - if cfg == nil { - return nil - } - return &llm.PromptKitLocalBackendConfig{ - Endpoint: cfg.Endpoint, - ConcurrencyLimit: cfg.ConcurrencyLimit, - } -} diff --git a/internal/cli/promptkit_profiles_test.go b/internal/cli/promptkit_profiles_test.go deleted file mode 100644 index 20c016e0..00000000 --- a/internal/cli/promptkit_profiles_test.go +++ /dev/null @@ -1,182 +0,0 @@ -package cli - -import ( - "context" - "errors" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "strings" - "sync/atomic" - "testing" - "testing/fstest" - - "gitea.maximumdirect.net/eric/notarius/internal/core/config" - "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" -) - -func TestExplicitPromptKitProfileValidationInspectsProfilesWithoutGeneration(t *testing.T) { - var providerCalls atomic.Int32 - server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { - providerCalls.Add(1) - })) - defer server.Close() - - writeProfile := func(t *testing.T, name, content string) string { - t.Helper() - profilePath := filepath.Join(t.TempDir(), name+".yaml") - if err := os.WriteFile(profilePath, []byte(content), 0o600); err != nil { - t.Fatal(err) - } - return profilePath - } - localProfile := "profiles:\n local-profile:\n backend: local\n model: local-model\n" - credentialProfile := `profiles: - credential-profile: - endpoint: ` + server.URL + `/v1 - model: credential-model - api_key_env: NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY -` - t.Setenv("NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY", "") - - tests := []struct { - name string - profilePath string - profileID string - profileDir bool - localBackend bool - canceled bool - wantErr []string - rejectErr []string - }{ - { - name: "configured local backend", - profilePath: writeProfile(t, "local-profile", localProfile), - profileID: "local-profile", - profileDir: true, - localBackend: true, - }, - { - name: "missing local backend registration", - profilePath: writeProfile(t, "local-profile", localProfile), - profileID: "local-profile", - wantErr: []string{"load PromptKit profiles", "configuration is invalid or unreadable"}, - }, - { - name: "absent profile", - profilePath: writeProfile(t, "local-profile", localProfile), - profileID: "absent-profile", - localBackend: true, - wantErr: []string{`PromptKit profile "absent-profile" is not configured`}, - }, - { - name: "malformed profile", - profilePath: writeProfile(t, "malformed-profile", "profiles:\n malformed-profile:\n endpoint: https://provider.example/v1?credential=forbidden\n model: malformed-model\n"), - profileID: "malformed-profile", - wantErr: []string{"load PromptKit profiles", "configuration is invalid or unreadable"}, - rejectErr: []string{"malformed-profile.yaml", "credential=forbidden"}, - }, - { - name: "invalid profile source", - profilePath: filepath.Join(t.TempDir(), "missing-profile.yaml"), - profileID: "missing-profile", - wantErr: []string{"load PromptKit profiles", "profile configuration is invalid or unreadable"}, - }, - { - name: "credential environment intentionally unset", - profilePath: writeProfile(t, "credential-profile", credentialProfile), - profileID: "credential-profile", - }, - { - name: "canceled inspection", - profilePath: writeProfile(t, "local-profile", localProfile), - profileID: "local-profile", - localBackend: true, - canceled: true, - wantErr: []string{"context canceled"}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - cfg := config.Default() - if tt.profileDir { - cfg.PromptKit.ProfileDir = filepath.Dir(tt.profilePath) - } else { - cfg.PromptKit.ProfileFile = tt.profilePath - } - if tt.localBackend { - cfg.PromptKit.LocalBackend = &config.PromptKitLocalBackendConfig{ - Endpoint: server.URL + "/v1", - ConcurrencyLimit: 2, - } - } - ctx := context.Background() - if tt.canceled { - var cancel context.CancelFunc - ctx, cancel = context.WithCancel(ctx) - cancel() - } - err := validateExplicitPromptKitProfiles(ctx, cfg, []string{tt.profileID}, nil) - if len(tt.wantErr) == 0 { - if err != nil { - t.Fatalf("validateExplicitPromptKitProfiles() error = %v, want nil", err) - } - return - } - if err == nil { - t.Fatal("validateExplicitPromptKitProfiles() error = nil, want failure") - } - if tt.canceled && !errors.Is(err, context.Canceled) { - t.Fatalf("canceled inspection error = %v, want context canceled", err) - } - for _, want := range tt.wantErr { - if !strings.Contains(err.Error(), want) { - t.Fatalf("validation error = %q, want %q", err, want) - } - } - for _, rejected := range append(tt.rejectErr, tt.profilePath) { - if rejected != "" && strings.Contains(err.Error(), rejected) { - t.Fatalf("validation error = %q, must not expose %q", err, rejected) - } - } - }) - } - if providerCalls.Load() != 0 { - t.Fatalf("provider calls during profile inspection = %d, want 0", providerCalls.Load()) - } -} - -func TestExplicitPromptKitProfileValidationUsesFallbackAssets(t *testing.T) { - assets := llm.NewAssetRegistry() - if err := assets.RegisterFallbackProfileFS(fstest.MapFS{ - "profiles/fallback.yaml": {Data: []byte("profiles:\n fallback-profile:\n endpoint: http://promptkit.test/v1\n model: fallback-model\n")}, - }, "profiles"); err != nil { - t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err) - } - if err := validateExplicitPromptKitProfiles(context.Background(), config.Default(), []string{"fallback-profile"}, assets); err != nil { - t.Fatalf("validateExplicitPromptKitProfiles() error = %v, want nil", err) - } -} - -func TestExplicitPromptKitProfileValidationRejectsInvalidInheritanceBeforeGeneration(t *testing.T) { - for _, profiles := range []string{ - "profiles:\n child:\n extends: missing\n", - "profiles:\n first:\n extends: second\n second:\n extends: first\n", - } { - t.Run("invalid inheritance", func(t *testing.T) { - profilePath := filepath.Join(t.TempDir(), "profiles.yaml") - if err := os.WriteFile(profilePath, []byte(profiles), 0o600); err != nil { - t.Fatal(err) - } - profileID := "child" - if strings.Contains(profiles, "first:") { - profileID = "first" - } - err := validateExplicitPromptKitProfiles(context.Background(), config.Config{PromptKit: config.PromptKitConfig{ProfileFile: profilePath}}, []string{profileID}, nil) - if err == nil || !strings.Contains(err.Error(), "invalid or unreadable") || strings.Contains(err.Error(), profilePath) { - t.Fatalf("profile preflight error = %v", err) - } - }) - } -} diff --git a/internal/cli/run.go b/internal/cli/run.go index 2bff7f82..c842a576 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -28,9 +28,12 @@ import ( "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "go.maximumdirect.net/commandkit" configkit "go.maximumdirect.net/configkit" + "go.maximumdirect.net/promptkit" ) type Options struct { + LLMClient promptkit.LLMClient + LLMRuntime promptkit.RuntimeConfig Catalog pipeline.ModuleCatalog Registries pipeline.Registries LLMClientFactory LLMClientFactory