From c848befb531e12f8911e85c2348fdaceb38a30b2 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 4 Oct 2026 13:26:54 +0000 Subject: [PATCH] Expose provider-neutral PromptKit inspection --- docs/roadmap/implementation.md | 8 + .../framework/llm/promptkit_inspection.go | 155 ++++++++++++++++++ .../llm/promptkit_inspection_test.go | 143 ++++++++++++++++ internal/framework/llm/promptkit_profiles.go | 2 + 4 files changed, 308 insertions(+) create mode 100644 internal/framework/llm/promptkit_inspection.go create mode 100644 internal/framework/llm/promptkit_inspection_test.go diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index f43e59cf..58248314 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -539,6 +539,14 @@ resolution, provider calls, automatic dumps, or mutation of identity. Validation: `GOWORK=off go test ./internal/framework/llm`; `git diff --check`. +> Completion: complete. Added provider-neutral effective-configuration and +> explanation string/writer operations, with canonical pointer parsing and +> distinct invalid/unknown results. Profile inspection now returns a detached +> copy of the effective target and recursively redacts credential-shaped +> provider parameters without resolving credentials. Escaped-pointer, history, +> writer-error, unset-credential, redaction, and ownership coverage passes with +> `go test ./internal/framework/llm`; `git diff --check` passes. + Prompt size: one facade/DTO, not CLI parsing. ## Stage 11 — Add Scoped Effective And Explanation Commands diff --git a/internal/framework/llm/promptkit_inspection.go b/internal/framework/llm/promptkit_inspection.go new file mode 100644 index 00000000..3f9b6fee --- /dev/null +++ b/internal/framework/llm/promptkit_inspection.go @@ -0,0 +1,155 @@ +package llm + +import ( + "bytes" + "errors" + "fmt" + "io" + "strings" + + configkit "gitea.maximumdirect.net/eric/ConfigKit" + "gitea.maximumdirect.net/eric/promptkit" +) + +var ErrInvalidPromptKitConfigurationPointer = errors.New("invalid PromptKit configuration pointer") + +// PromptKitEffectiveModelParams is the provider-neutral JSON projection of an +// inspected profile target. Credential-shaped extra-parameter values are +// redacted in this reporting copy. +type PromptKitEffectiveModelParams struct { + BackendID string `json:"backend_id"` + Endpoint string `json:"endpoint"` + Model string `json:"model"` + Temperature float64 `json:"temperature"` + MaxTokens int `json:"max_tokens"` + TopP float64 `json:"top_p"` + TimeoutSeconds int `json:"timeout_seconds"` + ServiceTier string `json:"service_tier"` + ReasoningEffort string `json:"reasoning_effort"` + APIKeyEnv string `json:"api_key_env"` + ExtraParams map[string]any `json:"extra_params"` +} + +type invalidPromptKitConfigurationPointerError struct { + cause error +} + +func (e *invalidPromptKitConfigurationPointerError) Error() string { + return ErrInvalidPromptKitConfigurationPointer.Error() +} + +func (e *invalidPromptKitConfigurationPointerError) Unwrap() error { + return e.cause +} + +func (e *invalidPromptKitConfigurationPointerError) Is(target error) bool { + return target == ErrInvalidPromptKitConfigurationPointer +} + +func (i *PromptKitProfileInspector) EffectiveConfiguration() (string, error) { + var output bytes.Buffer + if err := i.WriteEffectiveConfiguration(&output); err != nil { + return "", err + } + return output.String(), nil +} + +func (i *PromptKitProfileInspector) WriteEffectiveConfiguration(output io.Writer) error { + if i == nil || i.engine == nil { + return fmt.Errorf("PromptKit inspector must not be nil") + } + inspection, err := i.engine.InspectConfiguration() + if err != nil { + return fmt.Errorf("inspect PromptKit configuration: %w", err) + } + return inspection.WriteEffective(output) +} + +func (i *PromptKitProfileInspector) ExplainConfiguration(pointer string) (string, bool, error) { + var output bytes.Buffer + found, err := i.WriteConfigurationExplanation(&output, pointer) + if err != nil { + return "", false, err + } + return output.String(), found, nil +} + +func (i *PromptKitProfileInspector) WriteConfigurationExplanation(output io.Writer, pointer string) (bool, error) { + if i == nil || i.engine == nil { + return false, fmt.Errorf("PromptKit inspector must not be nil") + } + path, err := configkit.ParsePath(pointer) + if err != nil || path.String() != pointer { + if err == nil { + err = fmt.Errorf("pointer spelling is not canonical") + } + return false, &invalidPromptKitConfigurationPointerError{cause: err} + } + inspection, err := i.engine.InspectConfiguration() + if err != nil { + return false, fmt.Errorf("inspect PromptKit configuration: %w", err) + } + if _, found := inspection.Explain(path); !found { + return false, nil + } + if err := inspection.WriteExplanation(output, path); err != nil { + return true, err + } + return true, nil +} + +func promptKitEffectiveModelParams(target promptkit.ExecutionTarget) PromptKitEffectiveModelParams { + return PromptKitEffectiveModelParams{ + BackendID: strings.TrimSpace(target.BackendID), + Endpoint: strings.TrimSpace(target.Endpoint), + Model: strings.TrimSpace(target.Model), + Temperature: target.Temperature, + MaxTokens: target.MaxTokens, + TopP: target.TopP, + TimeoutSeconds: target.TimeoutSeconds, + ServiceTier: target.ServiceTier, + ReasoningEffort: target.ReasoningEffort, + APIKeyEnv: strings.TrimSpace(target.APIKeyEnv), + ExtraParams: redactCredentialParameters(target.ExtraParams), + } +} + +func redactCredentialParameters(values map[string]any) map[string]any { + if values == nil { + return nil + } + redacted, _ := redactCredentialParameterValue(values).(map[string]any) + return redacted +} + +func redactCredentialParameterValue(value any) any { + switch typed := value.(type) { + case map[string]any: + copy := make(map[string]any, len(typed)) + for key, nested := range typed { + if isCredentialParameterName(key) { + copy[key] = "[REDACTED]" + continue + } + copy[key] = redactCredentialParameterValue(nested) + } + return copy + case []any: + copy := make([]any, len(typed)) + for index, nested := range typed { + copy[index] = redactCredentialParameterValue(nested) + } + return copy + default: + return typed + } +} + +func isCredentialParameterName(name string) bool { + switch strings.ToLower(name) { + case "api_key", "apikey", "authorization", "credential", "credentials", "password", "secret", "token", "access_token": + return true + default: + return false + } +} diff --git a/internal/framework/llm/promptkit_inspection_test.go b/internal/framework/llm/promptkit_inspection_test.go new file mode 100644 index 00000000..b2fdcecd --- /dev/null +++ b/internal/framework/llm/promptkit_inspection_test.go @@ -0,0 +1,143 @@ +package llm + +import ( + "encoding/json" + "errors" + "strings" + "testing" + + configkit "gitea.maximumdirect.net/eric/ConfigKit" + "gitea.maximumdirect.net/eric/promptkit" +) + +type promptKitInspectionErrorWriter struct{ err error } + +func (w promptKitInspectionErrorWriter) Write([]byte) (int, error) { return 0, w.err } + +func TestPromptKitProfileInspectionCopiesAndRedactsTarget(t *testing.T) { + const ( + profileID = "inspection/profile~case" + keyEnv = "NOT_SET_SYNTHETIC_PROFILE_KEY" + ) + endpoint, initialModel, finalModel := "https://example.invalid/v1", "model-before", "model-after" + params := map[string]any{ + "Authorization": "Bearer authorization-secret", + "api_key": "api-key-secret", + "ApiKey": "api-key-camel-secret", + "apikey": "apikey-secret", + "credential": "credential-secret", + "credentials": "credentials-secret", + "password": "password-secret", + "secret": "secret-secret", + "token": "token-secret", + "access_token": "access-token-secret", + "normal": "visible-value", + "nested": map[string]any{ + "Access_Token": "nested-token-secret", + "items": []any{map[string]any{"PASSWORD": "list-secret", "label": "ordinary"}}, + }, + } + inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{Sources: promptkit.ConfigurationSources{ + Underlays: []configkit.Underlay{ + {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("lower profile", map[string]promptkit.ProfileConfig{ + profileID: {Endpoint: &endpoint, Model: &initialModel, APIKeyEnv: promptKitTestString(keyEnv), ExtraParams: params}, + })}}}, + {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("profile overlay", map[string]promptkit.ProfileConfig{ + profileID: {Model: &finalModel}, + })}}}, + }, + }}) + if err != nil { + t.Fatalf("NewPromptKitProfileInspector() error = %v", err) + } + profile, err := inspector.InspectProfile(t.Context(), profileID) + if err != nil { + t.Fatalf("InspectProfile() error = %v; an unset synthetic credential name must not be resolved", err) + } + if profile.ProfileID != profileID || profile.BackendID != "" || profile.Model != finalModel || profile.CredentialEnvironment != keyEnv || profile.CredentialRequired { + t.Fatalf("profile inspection = %#v", profile) + } + paramsCopy := profile.EffectiveModelParams.ExtraParams + for _, key := range []string{"Authorization", "api_key", "ApiKey", "apikey", "credential", "credentials", "password", "secret", "token", "access_token"} { + if paramsCopy[key] != "[REDACTED]" { + t.Errorf("effective provider parameter %q = %#v, want redacted", key, paramsCopy[key]) + } + } + if paramsCopy["normal"] != "visible-value" { + t.Fatalf("top-level effective provider parameters = %#v", paramsCopy) + } + nested := paramsCopy["nested"].(map[string]any) + if nested["Access_Token"] != "[REDACTED]" { + t.Fatalf("nested effective provider parameters = %#v", nested) + } + items := nested["items"].([]any) + if items[0].(map[string]any)["PASSWORD"] != "[REDACTED]" || items[0].(map[string]any)["label"] != "ordinary" { + t.Fatalf("list provider parameters = %#v", items) + } + encoded, err := json.Marshal(profile.EffectiveModelParams) + if err != nil { + t.Fatal(err) + } + for _, secret := range []string{"authorization-secret", "api-key-secret", "api-key-camel-secret", "apikey-secret", "credential-secret", "credentials-secret", "password-secret", "secret-secret", "token-secret", "access-token-secret", "nested-token-secret", "list-secret"} { + if strings.Contains(string(encoded), secret) { + t.Fatalf("profile reporting projection contains %q", secret) + } + } + nested["ordinary"] = "caller mutation" + paramsCopy["normal"] = "caller mutation" + second, err := inspector.InspectProfile(t.Context(), profileID) + if err != nil { + t.Fatal(err) + } + if second.EffectiveModelParams.ExtraParams["normal"] != "visible-value" || second.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["ordinary"] != nil { + t.Fatalf("profile inspection retained caller mutation: %#v", second.EffectiveModelParams.ExtraParams) + } +} + +func TestPromptKitConfigurationInspectionDistinguishesInvalidAndUnknownPointers(t *testing.T) { + const profileID = "inspection/profile~case" + endpoint, initialModel, finalModel := "https://example.invalid/v1", "model-before", "model-after" + inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{Sources: promptkit.ConfigurationSources{ + Underlays: []configkit.Underlay{ + {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("lower profile", map[string]promptkit.ProfileConfig{ + profileID: {Endpoint: &endpoint, Model: &initialModel}, + })}}}, + {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("profile overlay", map[string]promptkit.ProfileConfig{ + profileID: {Model: &finalModel}, + })}}}, + }, + }}) + if err != nil { + t.Fatalf("NewPromptKitProfileInspector() error = %v", err) + } + + report, err := inspector.EffectiveConfiguration() + if err != nil { + t.Fatalf("EffectiveConfiguration() error = %v", err) + } + if !strings.Contains(report, "model-after") || strings.Contains(report, "model-before") { + t.Fatalf("effective report = %q, want only the effective model", report) + } + const escapedPointer = "/profiles/inspection~1profile~0case/model" + explanation, found, err := inspector.ExplainConfiguration(escapedPointer) + if err != nil || !found || !strings.Contains(explanation, "model-before") || !strings.Contains(explanation, "model-after") { + t.Fatalf("ExplainConfiguration(%q) = (%q, %t, %v), want effective and historical values", escapedPointer, explanation, found, err) + } + if _, found, err := inspector.ExplainConfiguration("/profiles/not-configured"); err != nil || found { + t.Fatalf("unknown configuration pointer = (found %t, error %v), want (false, nil)", found, err) + } + if _, _, err := inspector.ExplainConfiguration("/profiles/~2/model"); !errors.Is(err, ErrInvalidPromptKitConfigurationPointer) { + t.Fatalf("invalid configuration pointer error = %v", err) + } + if _, found, err := inspector.ExplainConfiguration("/profiles/inspection~01profile~00case/model"); err != nil || found { + t.Fatalf("configuration pointer with literal escape sequences = (found %t, error %v), want a valid unknown pointer", found, err) + } + + writerFailure := errors.New("writer failure") + if err := inspector.WriteEffectiveConfiguration(promptKitInspectionErrorWriter{err: writerFailure}); !errors.Is(err, writerFailure) { + t.Fatalf("WriteEffectiveConfiguration() error = %v, want writer cause", err) + } + if _, err := inspector.WriteConfigurationExplanation(promptKitInspectionErrorWriter{err: writerFailure}, escapedPointer); !errors.Is(err, writerFailure) { + t.Fatalf("WriteConfigurationExplanation() error = %v, want writer cause", err) + } +} diff --git a/internal/framework/llm/promptkit_profiles.go b/internal/framework/llm/promptkit_profiles.go index 955e839c..95b0cccd 100644 --- a/internal/framework/llm/promptkit_profiles.go +++ b/internal/framework/llm/promptkit_profiles.go @@ -39,6 +39,7 @@ type PromptKitProfileInspection struct { Model string CredentialEnvironment string CredentialRequired bool + EffectiveModelParams PromptKitEffectiveModelParams `json:"effective_model_params"` } type PromptKitProfileInspectionError struct { @@ -110,6 +111,7 @@ func (i *PromptKitProfileInspector) InspectProfile(ctx context.Context, profileI Model: strings.TrimSpace(inspection.EffectiveModelParams.Model), CredentialEnvironment: strings.TrimSpace(inspection.EffectiveModelParams.APIKeyEnv), CredentialRequired: inspection.APIKeyRequired, + EffectiveModelParams: promptKitEffectiveModelParams(inspection.EffectiveModelParams), }, nil }