Expose provider-neutral PromptKit inspection
This commit is contained in:
@@ -539,6 +539,14 @@ resolution, provider calls, automatic dumps, or mutation of identity.
|
||||
|
||||
Validation: `GOWORK=off go test ./internal/framework/llm`; `git diff --check`.
|
||||
|
||||
> Completion: complete. Added provider-neutral effective-configuration and
|
||||
> explanation string/writer operations, with canonical pointer parsing and
|
||||
> distinct invalid/unknown results. Profile inspection now returns a detached
|
||||
> copy of the effective target and recursively redacts credential-shaped
|
||||
> provider parameters without resolving credentials. Escaped-pointer, history,
|
||||
> writer-error, unset-credential, redaction, and ownership coverage passes with
|
||||
> `go test ./internal/framework/llm`; `git diff --check` passes.
|
||||
|
||||
Prompt size: one facade/DTO, not CLI parsing.
|
||||
|
||||
## Stage 11 — Add Scoped Effective And Explanation Commands
|
||||
|
||||
155
internal/framework/llm/promptkit_inspection.go
Normal file
155
internal/framework/llm/promptkit_inspection.go
Normal file
@@ -0,0 +1,155 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
var ErrInvalidPromptKitConfigurationPointer = errors.New("invalid PromptKit configuration pointer")
|
||||
|
||||
// PromptKitEffectiveModelParams is the provider-neutral JSON projection of an
|
||||
// inspected profile target. Credential-shaped extra-parameter values are
|
||||
// redacted in this reporting copy.
|
||||
type PromptKitEffectiveModelParams struct {
|
||||
BackendID string `json:"backend_id"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
Model string `json:"model"`
|
||||
Temperature float64 `json:"temperature"`
|
||||
MaxTokens int `json:"max_tokens"`
|
||||
TopP float64 `json:"top_p"`
|
||||
TimeoutSeconds int `json:"timeout_seconds"`
|
||||
ServiceTier string `json:"service_tier"`
|
||||
ReasoningEffort string `json:"reasoning_effort"`
|
||||
APIKeyEnv string `json:"api_key_env"`
|
||||
ExtraParams map[string]any `json:"extra_params"`
|
||||
}
|
||||
|
||||
type invalidPromptKitConfigurationPointerError struct {
|
||||
cause error
|
||||
}
|
||||
|
||||
func (e *invalidPromptKitConfigurationPointerError) Error() string {
|
||||
return ErrInvalidPromptKitConfigurationPointer.Error()
|
||||
}
|
||||
|
||||
func (e *invalidPromptKitConfigurationPointerError) Unwrap() error {
|
||||
return e.cause
|
||||
}
|
||||
|
||||
func (e *invalidPromptKitConfigurationPointerError) Is(target error) bool {
|
||||
return target == ErrInvalidPromptKitConfigurationPointer
|
||||
}
|
||||
|
||||
func (i *PromptKitProfileInspector) EffectiveConfiguration() (string, error) {
|
||||
var output bytes.Buffer
|
||||
if err := i.WriteEffectiveConfiguration(&output); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return output.String(), nil
|
||||
}
|
||||
|
||||
func (i *PromptKitProfileInspector) WriteEffectiveConfiguration(output io.Writer) error {
|
||||
if i == nil || i.engine == nil {
|
||||
return fmt.Errorf("PromptKit inspector must not be nil")
|
||||
}
|
||||
inspection, err := i.engine.InspectConfiguration()
|
||||
if err != nil {
|
||||
return fmt.Errorf("inspect PromptKit configuration: %w", err)
|
||||
}
|
||||
return inspection.WriteEffective(output)
|
||||
}
|
||||
|
||||
func (i *PromptKitProfileInspector) ExplainConfiguration(pointer string) (string, bool, error) {
|
||||
var output bytes.Buffer
|
||||
found, err := i.WriteConfigurationExplanation(&output, pointer)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
return output.String(), found, nil
|
||||
}
|
||||
|
||||
func (i *PromptKitProfileInspector) WriteConfigurationExplanation(output io.Writer, pointer string) (bool, error) {
|
||||
if i == nil || i.engine == nil {
|
||||
return false, fmt.Errorf("PromptKit inspector must not be nil")
|
||||
}
|
||||
path, err := configkit.ParsePath(pointer)
|
||||
if err != nil || path.String() != pointer {
|
||||
if err == nil {
|
||||
err = fmt.Errorf("pointer spelling is not canonical")
|
||||
}
|
||||
return false, &invalidPromptKitConfigurationPointerError{cause: err}
|
||||
}
|
||||
inspection, err := i.engine.InspectConfiguration()
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("inspect PromptKit configuration: %w", err)
|
||||
}
|
||||
if _, found := inspection.Explain(path); !found {
|
||||
return false, nil
|
||||
}
|
||||
if err := inspection.WriteExplanation(output, path); err != nil {
|
||||
return true, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func promptKitEffectiveModelParams(target promptkit.ExecutionTarget) PromptKitEffectiveModelParams {
|
||||
return PromptKitEffectiveModelParams{
|
||||
BackendID: strings.TrimSpace(target.BackendID),
|
||||
Endpoint: strings.TrimSpace(target.Endpoint),
|
||||
Model: strings.TrimSpace(target.Model),
|
||||
Temperature: target.Temperature,
|
||||
MaxTokens: target.MaxTokens,
|
||||
TopP: target.TopP,
|
||||
TimeoutSeconds: target.TimeoutSeconds,
|
||||
ServiceTier: target.ServiceTier,
|
||||
ReasoningEffort: target.ReasoningEffort,
|
||||
APIKeyEnv: strings.TrimSpace(target.APIKeyEnv),
|
||||
ExtraParams: redactCredentialParameters(target.ExtraParams),
|
||||
}
|
||||
}
|
||||
|
||||
func redactCredentialParameters(values map[string]any) map[string]any {
|
||||
if values == nil {
|
||||
return nil
|
||||
}
|
||||
redacted, _ := redactCredentialParameterValue(values).(map[string]any)
|
||||
return redacted
|
||||
}
|
||||
|
||||
func redactCredentialParameterValue(value any) any {
|
||||
switch typed := value.(type) {
|
||||
case map[string]any:
|
||||
copy := make(map[string]any, len(typed))
|
||||
for key, nested := range typed {
|
||||
if isCredentialParameterName(key) {
|
||||
copy[key] = "[REDACTED]"
|
||||
continue
|
||||
}
|
||||
copy[key] = redactCredentialParameterValue(nested)
|
||||
}
|
||||
return copy
|
||||
case []any:
|
||||
copy := make([]any, len(typed))
|
||||
for index, nested := range typed {
|
||||
copy[index] = redactCredentialParameterValue(nested)
|
||||
}
|
||||
return copy
|
||||
default:
|
||||
return typed
|
||||
}
|
||||
}
|
||||
|
||||
func isCredentialParameterName(name string) bool {
|
||||
switch strings.ToLower(name) {
|
||||
case "api_key", "apikey", "authorization", "credential", "credentials", "password", "secret", "token", "access_token":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
143
internal/framework/llm/promptkit_inspection_test.go
Normal file
143
internal/framework/llm/promptkit_inspection_test.go
Normal file
@@ -0,0 +1,143 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
type promptKitInspectionErrorWriter struct{ err error }
|
||||
|
||||
func (w promptKitInspectionErrorWriter) Write([]byte) (int, error) { return 0, w.err }
|
||||
|
||||
func TestPromptKitProfileInspectionCopiesAndRedactsTarget(t *testing.T) {
|
||||
const (
|
||||
profileID = "inspection/profile~case"
|
||||
keyEnv = "NOT_SET_SYNTHETIC_PROFILE_KEY"
|
||||
)
|
||||
endpoint, initialModel, finalModel := "https://example.invalid/v1", "model-before", "model-after"
|
||||
params := map[string]any{
|
||||
"Authorization": "Bearer authorization-secret",
|
||||
"api_key": "api-key-secret",
|
||||
"ApiKey": "api-key-camel-secret",
|
||||
"apikey": "apikey-secret",
|
||||
"credential": "credential-secret",
|
||||
"credentials": "credentials-secret",
|
||||
"password": "password-secret",
|
||||
"secret": "secret-secret",
|
||||
"token": "token-secret",
|
||||
"access_token": "access-token-secret",
|
||||
"normal": "visible-value",
|
||||
"nested": map[string]any{
|
||||
"Access_Token": "nested-token-secret",
|
||||
"items": []any{map[string]any{"PASSWORD": "list-secret", "label": "ordinary"}},
|
||||
},
|
||||
}
|
||||
inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{
|
||||
{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("lower profile", map[string]promptkit.ProfileConfig{
|
||||
profileID: {Endpoint: &endpoint, Model: &initialModel, APIKeyEnv: promptKitTestString(keyEnv), ExtraParams: params},
|
||||
})}}},
|
||||
{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("profile overlay", map[string]promptkit.ProfileConfig{
|
||||
profileID: {Model: &finalModel},
|
||||
})}}},
|
||||
},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("NewPromptKitProfileInspector() error = %v", err)
|
||||
}
|
||||
profile, err := inspector.InspectProfile(t.Context(), profileID)
|
||||
if err != nil {
|
||||
t.Fatalf("InspectProfile() error = %v; an unset synthetic credential name must not be resolved", err)
|
||||
}
|
||||
if profile.ProfileID != profileID || profile.BackendID != "" || profile.Model != finalModel || profile.CredentialEnvironment != keyEnv || profile.CredentialRequired {
|
||||
t.Fatalf("profile inspection = %#v", profile)
|
||||
}
|
||||
paramsCopy := profile.EffectiveModelParams.ExtraParams
|
||||
for _, key := range []string{"Authorization", "api_key", "ApiKey", "apikey", "credential", "credentials", "password", "secret", "token", "access_token"} {
|
||||
if paramsCopy[key] != "[REDACTED]" {
|
||||
t.Errorf("effective provider parameter %q = %#v, want redacted", key, paramsCopy[key])
|
||||
}
|
||||
}
|
||||
if paramsCopy["normal"] != "visible-value" {
|
||||
t.Fatalf("top-level effective provider parameters = %#v", paramsCopy)
|
||||
}
|
||||
nested := paramsCopy["nested"].(map[string]any)
|
||||
if nested["Access_Token"] != "[REDACTED]" {
|
||||
t.Fatalf("nested effective provider parameters = %#v", nested)
|
||||
}
|
||||
items := nested["items"].([]any)
|
||||
if items[0].(map[string]any)["PASSWORD"] != "[REDACTED]" || items[0].(map[string]any)["label"] != "ordinary" {
|
||||
t.Fatalf("list provider parameters = %#v", items)
|
||||
}
|
||||
encoded, err := json.Marshal(profile.EffectiveModelParams)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, secret := range []string{"authorization-secret", "api-key-secret", "api-key-camel-secret", "apikey-secret", "credential-secret", "credentials-secret", "password-secret", "secret-secret", "token-secret", "access-token-secret", "nested-token-secret", "list-secret"} {
|
||||
if strings.Contains(string(encoded), secret) {
|
||||
t.Fatalf("profile reporting projection contains %q", secret)
|
||||
}
|
||||
}
|
||||
nested["ordinary"] = "caller mutation"
|
||||
paramsCopy["normal"] = "caller mutation"
|
||||
second, err := inspector.InspectProfile(t.Context(), profileID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if second.EffectiveModelParams.ExtraParams["normal"] != "visible-value" || second.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["ordinary"] != nil {
|
||||
t.Fatalf("profile inspection retained caller mutation: %#v", second.EffectiveModelParams.ExtraParams)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptKitConfigurationInspectionDistinguishesInvalidAndUnknownPointers(t *testing.T) {
|
||||
const profileID = "inspection/profile~case"
|
||||
endpoint, initialModel, finalModel := "https://example.invalid/v1", "model-before", "model-after"
|
||||
inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{
|
||||
{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("lower profile", map[string]promptkit.ProfileConfig{
|
||||
profileID: {Endpoint: &endpoint, Model: &initialModel},
|
||||
})}}},
|
||||
{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("profile overlay", map[string]promptkit.ProfileConfig{
|
||||
profileID: {Model: &finalModel},
|
||||
})}}},
|
||||
},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("NewPromptKitProfileInspector() error = %v", err)
|
||||
}
|
||||
|
||||
report, err := inspector.EffectiveConfiguration()
|
||||
if err != nil {
|
||||
t.Fatalf("EffectiveConfiguration() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(report, "model-after") || strings.Contains(report, "model-before") {
|
||||
t.Fatalf("effective report = %q, want only the effective model", report)
|
||||
}
|
||||
const escapedPointer = "/profiles/inspection~1profile~0case/model"
|
||||
explanation, found, err := inspector.ExplainConfiguration(escapedPointer)
|
||||
if err != nil || !found || !strings.Contains(explanation, "model-before") || !strings.Contains(explanation, "model-after") {
|
||||
t.Fatalf("ExplainConfiguration(%q) = (%q, %t, %v), want effective and historical values", escapedPointer, explanation, found, err)
|
||||
}
|
||||
if _, found, err := inspector.ExplainConfiguration("/profiles/not-configured"); err != nil || found {
|
||||
t.Fatalf("unknown configuration pointer = (found %t, error %v), want (false, nil)", found, err)
|
||||
}
|
||||
if _, _, err := inspector.ExplainConfiguration("/profiles/~2/model"); !errors.Is(err, ErrInvalidPromptKitConfigurationPointer) {
|
||||
t.Fatalf("invalid configuration pointer error = %v", err)
|
||||
}
|
||||
if _, found, err := inspector.ExplainConfiguration("/profiles/inspection~01profile~00case/model"); err != nil || found {
|
||||
t.Fatalf("configuration pointer with literal escape sequences = (found %t, error %v), want a valid unknown pointer", found, err)
|
||||
}
|
||||
|
||||
writerFailure := errors.New("writer failure")
|
||||
if err := inspector.WriteEffectiveConfiguration(promptKitInspectionErrorWriter{err: writerFailure}); !errors.Is(err, writerFailure) {
|
||||
t.Fatalf("WriteEffectiveConfiguration() error = %v, want writer cause", err)
|
||||
}
|
||||
if _, err := inspector.WriteConfigurationExplanation(promptKitInspectionErrorWriter{err: writerFailure}, escapedPointer); !errors.Is(err, writerFailure) {
|
||||
t.Fatalf("WriteConfigurationExplanation() error = %v, want writer cause", err)
|
||||
}
|
||||
}
|
||||
@@ -39,6 +39,7 @@ type PromptKitProfileInspection struct {
|
||||
Model string
|
||||
CredentialEnvironment string
|
||||
CredentialRequired bool
|
||||
EffectiveModelParams PromptKitEffectiveModelParams `json:"effective_model_params"`
|
||||
}
|
||||
|
||||
type PromptKitProfileInspectionError struct {
|
||||
@@ -110,6 +111,7 @@ func (i *PromptKitProfileInspector) InspectProfile(ctx context.Context, profileI
|
||||
Model: strings.TrimSpace(inspection.EffectiveModelParams.Model),
|
||||
CredentialEnvironment: strings.TrimSpace(inspection.EffectiveModelParams.APIKeyEnv),
|
||||
CredentialRequired: inspection.APIKeyRequired,
|
||||
EffectiveModelParams: promptKitEffectiveModelParams(inspection.EffectiveModelParams),
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user