package cli import ( "context" "os" "path/filepath" "strings" "testing" "time" "gitea.maximumdirect.net/eric/notarius/internal/core/config" frameworkdebug "gitea.maximumdirect.net/eric/notarius/internal/framework/debug" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "go.maximumdirect.net/configkit" "go.maximumdirect.net/promptkit" ) func TestRunCaptureUsesFrozenSecretAwareConfiguration(t *testing.T) { result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("captured application", []byte(`version: 6 promptkit: profiles: - id: fixture model: sk-ordinary-model-name extra_params: {token: captured-secret-value} `))}, configkit.LayeredOptions[config.Document]{Definition: config.Definition(t.TempDir())}) if err != nil { t.Fatal(err) } // Mutating public typed settings must not change the frozen inspection. mutated := "mutated-after-load" result.Config.PromptKit.Profiles[0].Model = &mutated recorder, err := frameworkdebug.Start(t.TempDir(), "capture", time.Unix(1, 0)) if err != nil { t.Fatal(err) } if err := captureRunConfiguration(recorder, result); err != nil { t.Fatal(err) } for _, name := range []string{"application.yml", "promptkit.yml"} { content, err := os.ReadFile(filepath.Join(recorder.Path(), "configuration", name)) if err != nil { t.Fatal(err) } if strings.Contains(string(content), "captured-secret-value") || strings.Contains(string(content), "mutated-after-load") || !strings.Contains(string(content), "sk-ordinary-model-name") { t.Fatalf("configuration capture lost frozen/native secret handling: %s", content) } } if _, err := os.Stat(filepath.Join(recorder.Path(), "configuration", "sources.json")); err != nil { t.Fatal(err) } if err := captureRunConfiguration(nil, nil); err != nil { t.Fatalf("disabled capture inspected configuration: %v", err) } } type captureClient func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) func (client captureClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { return client(ctx, request) } func TestRunCapturePreservesNativeConfigurationSecretMarksInExecutionSnapshots(t *testing.T) { result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("application", []byte(`version: 6 promptkit: collections: fixture: profiles: - id: base endpoint: https://example.invalid/v1 model: sk-ordinary-model-name api_key_required: false extra_params: token: configured-secret items: [{authorization: nested-secret, caption: ordinary-caption}] - id: chosen extends: {id: base} prompts: - id: request default_profile: fixture/chosen messages: [{role: user, content: sk-ordinary-application-content}] output: {format: text, validation_mode: none} `))}, configkit.LayeredOptions[config.Document]{Definition: config.Definition(t.TempDir())}) if err != nil { t.Fatal(err) } engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(captureClient(func(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { if request.Target.ExtraParams["token"] != "configured-secret" || request.APIKey != "direct-secret" { t.Fatal("capture changed native execution") } return &promptkit.GenerateResponse{Content: "token and sk-ordinary-response-content"}, nil }))) if err != nil { t.Fatal(err) } recorder, err := frameworkdebug.Start(t.TempDir(), "run", time.Unix(1, 0)) if err != nil { t.Fatal(err) } request, capture := pipeline.CaptureNativeRequest(promptkit.RunRequest{PromptID: "fixture/request", APIKey: "direct-secret"}, newRunEvidenceRecorder(recorder, result)) response, err := engine.Run(context.Background(), request) if err != nil { t.Fatal(err) } if err := capture.Finish(response, nil, false, false); err != nil { t.Fatal(err) } for _, name := range []string{"prepared.json", "result.json"} { content, err := os.ReadFile(filepath.Join(recorder.Path(), "native", response.RunID, name)) if err != nil { t.Fatal(err) } for _, secret := range []string{"configured-secret", "nested-secret", "direct-secret"} { if strings.Contains(string(content), secret) { t.Fatalf("ConfigKit/native secret leaked in %s", name) } } if !strings.Contains(string(content), "sk-ordinary-model-name") || !strings.Contains(string(content), "ordinary-caption") { t.Fatalf("ordinary native settings were rewritten: %s", content) } } if response.EffectiveModelParams.ExtraParams["token"] != "configured-secret" { t.Fatal("projection mutated the caller's native snapshot") } content, err := os.ReadFile(filepath.Join(recorder.Path(), "native", response.RunID, "messages.txt")) if err != nil || !strings.Contains(string(content), "sk-ordinary-application-content") || !strings.Contains(string(content), "token and sk-ordinary-response-content") { t.Fatalf("application content rewritten: %s %v", content, err) } }