177 lines
7.1 KiB
Go
177 lines
7.1 KiB
Go
package cli
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"path/filepath"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
|
frameworkdebug "gitea.maximumdirect.net/eric/notarius/internal/framework/debug"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
|
"go.maximumdirect.net/configkit"
|
|
"go.maximumdirect.net/promptkit"
|
|
)
|
|
|
|
func TestRunCaptureRoutesRootAndCollectionSensitivityMasks(t *testing.T) {
|
|
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("mixed resources", []byte(`version: 6
|
|
promptkit:
|
|
profiles:
|
|
- id: chosen
|
|
extra_params:
|
|
token: root-profile
|
|
nested/key:
|
|
a~b: [{authorization: root-nested, caption: ordinary-caption}]
|
|
backends:
|
|
- id: chosen
|
|
endpoint: https://example.invalid/v1
|
|
extra_params: {password: root-backend}
|
|
collections:
|
|
fixture:
|
|
profiles:
|
|
- id: chosen
|
|
extra_params: {token: collection-profile}
|
|
backends:
|
|
- id: chosen
|
|
endpoint: https://example.invalid/v1
|
|
extra_params: {password: collection-backend}
|
|
`))}, configkit.LayeredOptions[config.Document]{Definition: config.Definition(t.TempDir())})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
writer := newRunEvidenceRecorder(nil, result).(*runEvidenceRecorder)
|
|
nested := map[string]any{"a~b": []any{map[string]any{"authorization": "root-nested", "caption": "ordinary-caption"}}}
|
|
for _, namespace := range []string{"", "fixture/"} {
|
|
target := promptkit.ExecutionTarget{ExtraParams: map[string]any{"token": "original-token", "password": "original-password", "nested/key": nested, "caption": "ordinary-caption"}}
|
|
masked := writer.target(target, namespace+"chosen", namespace+"chosen")
|
|
if masked.ExtraParams["token"] != "[REDACTED]" || masked.ExtraParams["password"] != "[REDACTED]" || masked.ExtraParams["caption"] != "ordinary-caption" {
|
|
t.Fatalf("misrouted resource masks for %q: %#v", namespace, masked)
|
|
}
|
|
wantNested := nested
|
|
if namespace == "" {
|
|
wantNested = map[string]any{"a~b": []any{map[string]any{"authorization": "[REDACTED]", "caption": "ordinary-caption"}}}
|
|
}
|
|
if !reflect.DeepEqual(masked.ExtraParams["nested/key"], wantNested) {
|
|
t.Fatalf("nested mask crossed resource boundary for %q: %#v", namespace, masked)
|
|
}
|
|
if target.ExtraParams["token"] != "original-token" || nested["a~b"].([]any)[0].(map[string]any)["authorization"] != "root-nested" {
|
|
t.Fatal("native snapshot was mutated")
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRunCaptureUsesFrozenSecretAwareConfiguration(t *testing.T) {
|
|
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("captured application", []byte(`version: 6
|
|
promptkit:
|
|
profiles:
|
|
- id: fixture
|
|
model: sk-ordinary-model-name
|
|
extra_params: {token: captured-secret-value}
|
|
`))}, configkit.LayeredOptions[config.Document]{Definition: config.Definition(t.TempDir())})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// Mutating public typed settings must not change the frozen inspection.
|
|
mutated := "mutated-after-load"
|
|
result.Config.PromptKit.Profiles[0].Model = &mutated
|
|
recorder, err := frameworkdebug.Start(t.TempDir(), "capture", time.Unix(1, 0))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := captureRunConfiguration(recorder, result); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, name := range []string{"application.yml", "promptkit.yml"} {
|
|
content, err := os.ReadFile(filepath.Join(recorder.Path(), "configuration", name))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if strings.Contains(string(content), "captured-secret-value") || strings.Contains(string(content), "mutated-after-load") || !strings.Contains(string(content), "sk-ordinary-model-name") {
|
|
t.Fatalf("configuration capture lost frozen/native secret handling: %s", content)
|
|
}
|
|
}
|
|
if _, err := os.Stat(filepath.Join(recorder.Path(), "configuration", "sources.json")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := captureRunConfiguration(nil, nil); err != nil {
|
|
t.Fatalf("disabled capture inspected configuration: %v", err)
|
|
}
|
|
}
|
|
|
|
type captureClient func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error)
|
|
|
|
func (client captureClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
return client(ctx, request)
|
|
}
|
|
|
|
func TestRunCapturePreservesNativeConfigurationSecretMarksInExecutionSnapshots(t *testing.T) {
|
|
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("application", []byte(`version: 6
|
|
promptkit:
|
|
collections:
|
|
fixture:
|
|
profiles:
|
|
- id: base
|
|
endpoint: https://example.invalid/v1
|
|
model: sk-ordinary-model-name
|
|
api_key_required: false
|
|
extra_params:
|
|
token: configured-secret
|
|
items: [{authorization: nested-secret, caption: ordinary-caption}]
|
|
- id: chosen
|
|
extends: {id: base}
|
|
prompts:
|
|
- id: request
|
|
default_profile: fixture/chosen
|
|
messages: [{role: user, content: sk-ordinary-application-content}]
|
|
output: {format: text, validation_mode: none}
|
|
`))}, configkit.LayeredOptions[config.Document]{Definition: config.Definition(t.TempDir())})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(captureClient(func(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
if request.Target.ExtraParams["token"] != "configured-secret" || request.APIKey != "direct-secret" {
|
|
t.Fatal("capture changed native execution")
|
|
}
|
|
return &promptkit.GenerateResponse{Content: "token and sk-ordinary-response-content"}, nil
|
|
})))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
recorder, err := frameworkdebug.Start(t.TempDir(), "run", time.Unix(1, 0))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
request, capture := pipeline.CaptureNativeRequest(promptkit.RunRequest{PromptID: "fixture/request", APIKey: "direct-secret"}, newRunEvidenceRecorder(recorder, result))
|
|
response, err := engine.Run(context.Background(), request)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := capture.Finish(response, nil, false, false); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, name := range []string{"prepared.json", "result.json"} {
|
|
content, err := os.ReadFile(filepath.Join(recorder.Path(), "native", response.RunID, name))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, secret := range []string{"configured-secret", "nested-secret", "direct-secret"} {
|
|
if strings.Contains(string(content), secret) {
|
|
t.Fatalf("ConfigKit/native secret leaked in %s", name)
|
|
}
|
|
}
|
|
if !strings.Contains(string(content), "sk-ordinary-model-name") || !strings.Contains(string(content), "ordinary-caption") {
|
|
t.Fatalf("ordinary native settings were rewritten: %s", content)
|
|
}
|
|
}
|
|
if response.EffectiveModelParams.ExtraParams["token"] != "configured-secret" {
|
|
t.Fatal("projection mutated the caller's native snapshot")
|
|
}
|
|
content, err := os.ReadFile(filepath.Join(recorder.Path(), "native", response.RunID, "messages.txt"))
|
|
if err != nil || !strings.Contains(string(content), "sk-ordinary-application-content") || !strings.Contains(string(content), "token and sk-ordinary-response-content") {
|
|
t.Fatalf("application content rewritten: %s %v", content, err)
|
|
}
|
|
}
|