Install native PromptKit diagnostics and reusable execution overrides

This commit is contained in:
2026-11-21 23:02:57 +00:00
parent 91869ba191
commit b899ee32a6
9 changed files with 230 additions and 236 deletions

View File

@@ -329,6 +329,17 @@ and diff status `3`. Test semantics, not entire upstream help text.
## Stage 7: Install ckprompt And Reusable Run Overrides
**Status:** Complete. Installed native resource listings/inspection, requirements,
request validation/preparation, namespace selection, and explicit smoke operations
with ckprompt's builders and scoped convenience aliases. Added reusable native
profile/session/execution flag input and request application, including omitted
versus empty reasoning and native transient-retry overrides. Removed the copied
CLI profile-inspection path and replaced its tests with native command coverage.
The isolated real-kit override test, build-info/asset tests, and
`git diff --check` pass. Assembled CLI inspection/preview/fake-provider smoke tests
remain blocked by the recorded legacy dependency/caller compilation blockers;
their results are not claimed passing. Full pipeline binding remains Stage 33.
**Goal:** adopt the full selected native diagnostics and execution input suite.
**Inspect/change:** CLI composition from Stage 6, old `promptkit_profiles.go`, and

2
go.mod
View File

@@ -7,6 +7,7 @@ require (
gitea.maximumdirect.net/eric/promptkit v0.12.0
github.com/santhosh-tekuri/jsonschema/v6 v6.0.3
go.maximumdirect.net/ckconfig v0.10.0
go.maximumdirect.net/ckprompt v0.11.0
go.maximumdirect.net/commandkit v0.7.0
go.maximumdirect.net/configkit v1.4.0
go.maximumdirect.net/promptkit v0.24.0
@@ -20,5 +21,4 @@ require (
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/spf13/cobra v1.10.2 // indirect
github.com/spf13/pflag v1.1.0 // indirect
go.maximumdirect.net/ckprompt v0.11.0 // indirect
)

View File

@@ -7,14 +7,22 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"go.maximumdirect.net/ckconfig"
"go.maximumdirect.net/ckprompt"
"go.maximumdirect.net/commandkit"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
// NewApplication declares the native command surface. Acquisition is deferred to
// the selected handler; construction performs no configuration or engine work.
func NewApplication(version string, options Options) (*commandkit.App, error) {
newState := func() *nativeLoaders { return &nativeLoaders{lookupEnv: options.LookupEnv} }
newState := func() *nativeLoaders {
state := &nativeLoaders{lookupEnv: options.LookupEnv, runtime: options.LLMRuntime}
if options.LLMClient != nil {
state.engineOptions = []promptkit.Option{promptkit.WithLLMClient(options.LLMClient)}
}
return state
}
load := func(ctx context.Context, selection ckconfig.Selection, _ struct{}, state *nativeLoaders) (*configkit.Result[config.Document], error) {
return state.application(ctx, selection)
}
@@ -22,17 +30,33 @@ func NewApplication(version string, options Options) (*commandkit.App, error) {
appScope.Features.Overrides = true
promptScope := ckconfig.FromConfigKitSubtree("promptkit", "Native PromptKit configuration.", configkit.Keys("promptkit"), load)
promptScope.Features.Overrides = true
return commandkit.New(commandkit.Identity{Name: "notarius", Version: version},
loadEngine := func(ctx context.Context, selection ckconfig.Selection, _ struct{}, state *nativeLoaders) (*promptkit.Engine, error) {
return state.engine(ctx, selection)
}
definitions := ckprompt.FromEngine(loadEngine)
definitions.NewState = newState
definitions.CheckInput = func(selection ckconfig.Selection, _ struct{}) error { return checkConfigSelection(selection) }
definitions = ckprompt.WithRequestValidation(definitions, loadEngine, nil)
definitions = ckprompt.WithRequestPreparation(definitions, loadEngine, nil)
definitions = ckprompt.WithSmokeTesting(definitions, loadEngine, nil)
definitions = ckprompt.WithNamespaces(definitions, promptkit.Namespaces{})
declarations := []commandkit.Declaration{
commandkit.Completion("notarius"),
ckconfig.Command(ckconfig.Suite[struct{}, *nativeLoaders]{
NewState: newState,
CheckInput: func(_ string, selection ckconfig.Selection, _ struct{}) error {
if selection.Config.Set && strings.TrimSpace(selection.Config.Value) == "" {
return fmt.Errorf("--config must not be empty")
}
return nil
return checkConfigSelection(selection)
},
Scopes: []ckconfig.Scope[struct{}, *nativeLoaders]{appScope, promptScope},
}),
)
}
declarations = append(declarations, ckprompt.Declarations(definitions)...)
return commandkit.New(commandkit.Identity{Name: "notarius", Version: version}, declarations...)
}
func checkConfigSelection(selection ckconfig.Selection) error {
if selection.Config.Set && strings.TrimSpace(selection.Config.Value) == "" {
return fmt.Errorf("--config must not be empty")
}
return nil
}

View File

@@ -0,0 +1,38 @@
package cli
import (
"go.maximumdirect.net/ckprompt"
"go.maximumdirect.net/commandkit"
"go.maximumdirect.net/promptkit"
)
// executionOverrides is invocation input, not configuration. Keep the native
// supplied values until binding a concrete producer or validator request.
type executionOverrides struct {
Resources ckprompt.ResourceInput
Execution ckprompt.ExecutionInput
}
func executionOverrideFlags[I any](selectInput func(*I) *executionOverrides) []commandkit.Flag[I] {
resources := ckprompt.NewResourceFlags(func(input *I) *ckprompt.ResourceInput { return &selectInput(input).Resources })
execution := ckprompt.NewExecutionFlags(func(input *I) *ckprompt.ExecutionInput { return &selectInput(input).Execution })
return append(execution.All(), resources.Profile, resources.SessionID)
}
// apply leaves operation-owned prompt/input/capture fields intact. Engine
// preparation captures native override values without changing loaded settings.
func (overrides executionOverrides) apply(request promptkit.RunRequest) promptkit.RunRequest {
if overrides.Resources.Profile.Set {
request.ProfileID = overrides.Resources.Profile.Value
}
if overrides.Resources.SessionID.Set {
request.SessionID = overrides.Resources.SessionID.Value
}
if overrides.Execution.Execution != nil {
request.Execution = overrides.Execution.Execution
}
if overrides.Execution.TransientRetries != nil {
request.TransientRetries = overrides.Execution.TransientRetries
}
return request
}

View File

@@ -0,0 +1,54 @@
package cli
import (
"context"
"io"
"testing"
"go.maximumdirect.net/commandkit"
"go.maximumdirect.net/promptkit"
)
func TestRunOverrideFlagsPreserveNativeSuppliedValues(t *testing.T) {
type input struct{ Overrides executionOverrides }
var received executionOverrides
app, err := commandkit.New(commandkit.Identity{Name: "fixture"}, commandkit.NewCommand(commandkit.CommandSpec[input]{
Name: "execute",
Flags: executionOverrideFlags(func(value *input) *executionOverrides { return &value.Overrides }),
Run: func(_ commandkit.Invocation, value input) commandkit.Outcome {
received = value.Overrides
return commandkit.Success()
},
}))
if err != nil {
t.Fatal(err)
}
execute := func(args ...string) commandkit.Outcome {
return app.Execute(context.Background(), append([]string{"execute"}, args...), commandkit.Streams{Out: io.Discard, Err: io.Discard})
}
if outcome := execute(); outcome.Status != 0 {
t.Fatal(outcome)
}
request := promptkit.RunRequest{PromptID: "notarius/producer", ProfileID: "bound", SessionID: "generated"}
unchanged := received.apply(request)
if unchanged.ProfileID != "bound" || unchanged.SessionID != "generated" || unchanged.Execution != nil {
t.Fatalf("omission changed the request: %#v", unchanged)
}
if outcome := execute("--llm-profile=chosen", "--llm-session-id=operator", "--llm-reasoning-effort=", "--llm-temperature=0", "--llm-transient-retries=0", `--llm-extra-params={"enabled":false}`); outcome.Status != 0 {
t.Fatal(outcome)
}
applied := received.apply(request)
if applied.PromptID != request.PromptID || applied.ProfileID != "chosen" || applied.SessionID != "operator" || applied.Execution == nil || applied.Execution.ReasoningEffort == nil || *applied.Execution.ReasoningEffort != "" || applied.Execution.Temperature == nil || *applied.Execution.Temperature != 0 || applied.TransientRetries == nil || *applied.TransientRetries != 0 || applied.Execution.ExtraParams["enabled"] != false {
t.Fatalf("native values lost: %#v", applied)
}
if request.ProfileID != "bound" || request.Execution != nil {
t.Fatal("application changed operation input")
}
// Every singleton occurrence must be valid, even if a later value parses.
if outcome := execute("--llm-temperature=invalid", "--llm-temperature=0"); outcome.Status != 2 {
t.Fatalf("invalid occurrence skipped: %#v", outcome)
}
if outcome := execute("--llm-profile=first", "--llm-profile=last"); outcome.Status != 0 || received.Resources.Profile.Value != "last" {
t.Fatalf("last supplied profile lost: %#v", outcome)
}
}

View File

@@ -0,0 +1,92 @@
package cli
import (
"bytes"
"context"
"encoding/json"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"go.maximumdirect.net/commandkit"
"go.maximumdirect.net/promptkit"
)
type diagnosticClient struct{ calls atomic.Int32 }
func (client *diagnosticClient) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
client.calls.Add(1)
return &promptkit.GenerateResponse{Content: "PROMPTKIT_OK"}, nil
}
func TestNativePromptCommandsInspectPreviewAndSmokeExplicitly(t *testing.T) {
t.Parallel()
main := filepath.Join(t.TempDir(), "config.yml")
writeConfigKitFile(t, main, `version: 6
promptkit:
profiles:
- id: partial
model: fixture-model
- id: ready
endpoint: https://example.invalid/v1
model: fixture-model
api_key_required: false
prompts:
- id: preview
default_profile: ready
inputs: [{name: topic, required: true}]
messages: [{role: user, content: 'Topic: {{input "topic"}}'}]
output: {format: text, validation_mode: none}
`)
client := &diagnosticClient{}
app, err := NewApplication("development", Options{LLMClient: client})
if err != nil {
t.Fatal(err)
}
execute := func(args ...string) (commandkit.Outcome, []byte, string) {
var out, diagnostics bytes.Buffer
args = append(args, "--config", main, "--format", "json")
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &out, Err: &diagnostics})
return outcome, out.Bytes(), diagnostics.String()
}
for _, args := range [][]string{
{"promptkit", "list", "prompts", "--all-namespaces"},
{"promptkit", "list", "profiles", "--all-namespaces"},
{"promptkit", "list", "backends", "--all-namespaces"},
{"promptkit", "list", "namespaces"},
{"promptkit", "list", "collections"},
{"promptkit", "show", "profile", "partial"},
{"promptkit", "show", "prompt", "preview"},
{"promptkit", "show", "backend", "openrouter"},
{"promptkit", "requirements", "show", "preview"},
{"promptkit", "requirements", "validate", "preview", "--llm-input-text=topic=fixture-topic"},
// The short input spelling comes from the native scoped alias declarations.
{"promptkit", "show", "prepared", "preview", "--input=topic=fixture-topic", "--reasoning-effort=", "--transient-retries=0"},
} {
outcome, out, diagnostics := execute(args...)
if outcome.Status != 0 || !json.Valid(out) {
t.Fatalf("%v: %#v, %s, %s", args, outcome, out, diagnostics)
}
if len(args) > 2 && args[1] == "show" && args[2] == "prepared" && !strings.Contains(string(out), "fixture-topic") {
t.Fatalf("preview lost explicit input: %s", out)
}
}
if client.calls.Load() != 0 {
t.Fatal("inspection implicitly called provider")
}
outcome, out, diagnostics := execute("promptkit", "test", "text-inference", "--llm-profile=ready")
if outcome.Status != 0 || client.calls.Load() == 0 {
t.Fatalf("explicit smoke: %#v, %s, %s", outcome, out, diagnostics)
}
var report struct {
Supported bool `json:"supported"`
Completed bool `json:"completed"`
}
if err := json.Unmarshal(out, &report); err != nil {
t.Fatal(err)
}
if !report.Supported || !report.Completed {
t.Fatalf("native smoke distinctions lost: %s", out)
}
}

View File

@@ -1,46 +0,0 @@
package cli
import (
"context"
"fmt"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
)
func validateExplicitPromptKitProfiles(ctx context.Context, cfg config.Config, profileIDs []string, assets *llm.AssetRegistry) error {
if len(profileIDs) == 0 {
return nil
}
inspector, err := llm.NewPromptKitProfileInspector(llm.PromptKitProfileInspectorConfig{
Source: promptKitProfileSourceConfig(cfg),
Assets: assets,
})
if err != nil {
return fmt.Errorf("load PromptKit profiles: %w", err)
}
for _, profileID := range profileIDs {
if _, err := inspector.InspectProfile(ctx, profileID); err != nil {
return err
}
}
return nil
}
func promptKitProfileSourceConfig(cfg config.Config) llm.PromptKitProfileSourceConfig {
return llm.PromptKitProfileSourceConfig{
ProfileDir: cfg.PromptKit.ProfileDir,
ProfileFile: cfg.PromptKit.ProfileFile,
LocalBackend: mapPromptKitLocalBackend(cfg.PromptKit.LocalBackend),
}
}
func mapPromptKitLocalBackend(cfg *config.PromptKitLocalBackendConfig) *llm.PromptKitLocalBackendConfig {
if cfg == nil {
return nil
}
return &llm.PromptKitLocalBackendConfig{
Endpoint: cfg.Endpoint,
ConcurrencyLimit: cfg.ConcurrencyLimit,
}
}

View File

@@ -1,182 +0,0 @@
package cli
import (
"context"
"errors"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"testing/fstest"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
)
func TestExplicitPromptKitProfileValidationInspectsProfilesWithoutGeneration(t *testing.T) {
var providerCalls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
providerCalls.Add(1)
}))
defer server.Close()
writeProfile := func(t *testing.T, name, content string) string {
t.Helper()
profilePath := filepath.Join(t.TempDir(), name+".yaml")
if err := os.WriteFile(profilePath, []byte(content), 0o600); err != nil {
t.Fatal(err)
}
return profilePath
}
localProfile := "profiles:\n local-profile:\n backend: local\n model: local-model\n"
credentialProfile := `profiles:
credential-profile:
endpoint: ` + server.URL + `/v1
model: credential-model
api_key_env: NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY
`
t.Setenv("NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY", "")
tests := []struct {
name string
profilePath string
profileID string
profileDir bool
localBackend bool
canceled bool
wantErr []string
rejectErr []string
}{
{
name: "configured local backend",
profilePath: writeProfile(t, "local-profile", localProfile),
profileID: "local-profile",
profileDir: true,
localBackend: true,
},
{
name: "missing local backend registration",
profilePath: writeProfile(t, "local-profile", localProfile),
profileID: "local-profile",
wantErr: []string{"load PromptKit profiles", "configuration is invalid or unreadable"},
},
{
name: "absent profile",
profilePath: writeProfile(t, "local-profile", localProfile),
profileID: "absent-profile",
localBackend: true,
wantErr: []string{`PromptKit profile "absent-profile" is not configured`},
},
{
name: "malformed profile",
profilePath: writeProfile(t, "malformed-profile", "profiles:\n malformed-profile:\n endpoint: https://provider.example/v1?credential=forbidden\n model: malformed-model\n"),
profileID: "malformed-profile",
wantErr: []string{"load PromptKit profiles", "configuration is invalid or unreadable"},
rejectErr: []string{"malformed-profile.yaml", "credential=forbidden"},
},
{
name: "invalid profile source",
profilePath: filepath.Join(t.TempDir(), "missing-profile.yaml"),
profileID: "missing-profile",
wantErr: []string{"load PromptKit profiles", "profile configuration is invalid or unreadable"},
},
{
name: "credential environment intentionally unset",
profilePath: writeProfile(t, "credential-profile", credentialProfile),
profileID: "credential-profile",
},
{
name: "canceled inspection",
profilePath: writeProfile(t, "local-profile", localProfile),
profileID: "local-profile",
localBackend: true,
canceled: true,
wantErr: []string{"context canceled"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg := config.Default()
if tt.profileDir {
cfg.PromptKit.ProfileDir = filepath.Dir(tt.profilePath)
} else {
cfg.PromptKit.ProfileFile = tt.profilePath
}
if tt.localBackend {
cfg.PromptKit.LocalBackend = &config.PromptKitLocalBackendConfig{
Endpoint: server.URL + "/v1",
ConcurrencyLimit: 2,
}
}
ctx := context.Background()
if tt.canceled {
var cancel context.CancelFunc
ctx, cancel = context.WithCancel(ctx)
cancel()
}
err := validateExplicitPromptKitProfiles(ctx, cfg, []string{tt.profileID}, nil)
if len(tt.wantErr) == 0 {
if err != nil {
t.Fatalf("validateExplicitPromptKitProfiles() error = %v, want nil", err)
}
return
}
if err == nil {
t.Fatal("validateExplicitPromptKitProfiles() error = nil, want failure")
}
if tt.canceled && !errors.Is(err, context.Canceled) {
t.Fatalf("canceled inspection error = %v, want context canceled", err)
}
for _, want := range tt.wantErr {
if !strings.Contains(err.Error(), want) {
t.Fatalf("validation error = %q, want %q", err, want)
}
}
for _, rejected := range append(tt.rejectErr, tt.profilePath) {
if rejected != "" && strings.Contains(err.Error(), rejected) {
t.Fatalf("validation error = %q, must not expose %q", err, rejected)
}
}
})
}
if providerCalls.Load() != 0 {
t.Fatalf("provider calls during profile inspection = %d, want 0", providerCalls.Load())
}
}
func TestExplicitPromptKitProfileValidationUsesFallbackAssets(t *testing.T) {
assets := llm.NewAssetRegistry()
if err := assets.RegisterFallbackProfileFS(fstest.MapFS{
"profiles/fallback.yaml": {Data: []byte("profiles:\n fallback-profile:\n endpoint: http://promptkit.test/v1\n model: fallback-model\n")},
}, "profiles"); err != nil {
t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err)
}
if err := validateExplicitPromptKitProfiles(context.Background(), config.Default(), []string{"fallback-profile"}, assets); err != nil {
t.Fatalf("validateExplicitPromptKitProfiles() error = %v, want nil", err)
}
}
func TestExplicitPromptKitProfileValidationRejectsInvalidInheritanceBeforeGeneration(t *testing.T) {
for _, profiles := range []string{
"profiles:\n child:\n extends: missing\n",
"profiles:\n first:\n extends: second\n second:\n extends: first\n",
} {
t.Run("invalid inheritance", func(t *testing.T) {
profilePath := filepath.Join(t.TempDir(), "profiles.yaml")
if err := os.WriteFile(profilePath, []byte(profiles), 0o600); err != nil {
t.Fatal(err)
}
profileID := "child"
if strings.Contains(profiles, "first:") {
profileID = "first"
}
err := validateExplicitPromptKitProfiles(context.Background(), config.Config{PromptKit: config.PromptKitConfig{ProfileFile: profilePath}}, []string{profileID}, nil)
if err == nil || !strings.Contains(err.Error(), "invalid or unreadable") || strings.Contains(err.Error(), profilePath) {
t.Fatalf("profile preflight error = %v", err)
}
})
}
}

View File

@@ -28,9 +28,12 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"go.maximumdirect.net/commandkit"
configkit "go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
type Options struct {
LLMClient promptkit.LLMClient
LLMRuntime promptkit.RuntimeConfig
Catalog pipeline.ModuleCatalog
Registries pipeline.Registries
LLMClientFactory LLMClientFactory