diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 1026fd86..f43e59cf 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -475,6 +475,15 @@ Prompt size: snapshot/resume regressions, no checkpoint redesign. ## Stage 9 — Verify Source-Relative Content And Capture Lifetime +> Completion: complete. Added integration coverage showing imported content +> resolves from its declaring directory, replacement content resolves from its +> replacement source while retaining the embedded prompt's other fields, and +> constructed engines keep captured messages across file edits while rebuilt +> engines observe changes. Static template and missing-file errors remain +> construction failures; missing request variables remain preparation errors. +> `GOWORK=off go test ./internal/framework/llm ./internal/modules/...` and +> `git diff --check` pass. + Goal: protect resource contexts and static template/content capture. Inspect manifests, source construction, and existing prepared/prompt asset diff --git a/internal/framework/llm/promptkit_sources_integration_test.go b/internal/framework/llm/promptkit_sources_integration_test.go index c240f23a..9c961447 100644 --- a/internal/framework/llm/promptkit_sources_integration_test.go +++ b/internal/framework/llm/promptkit_sources_integration_test.go @@ -8,6 +8,7 @@ import ( "strings" "sync/atomic" "testing" + "testing/fstest" configkit "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" @@ -197,6 +198,197 @@ prompts: } } +func TestPromptKitPromptContentUsesDeclaringAndReplacementSourceContexts(t *testing.T) { + rootDirectory := t.TempDir() + declaringDirectory := filepath.Join(rootDirectory, "child") + replacementDirectory := t.TempDir() + writeFile := func(path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + } + rootConfig := filepath.Join(rootDirectory, "operator.yaml") + declaringConfig := filepath.Join(declaringDirectory, "imported.yaml") + replacementConfig := filepath.Join(replacementDirectory, "replacement.yaml") + writeFile(rootConfig, `configkit: + imports: + - path: child/imported.yaml +`) + writeFile(filepath.Join(rootDirectory, "messages", "shared.tmpl"), "root decoy") + writeFile(declaringConfig, `prompts: + imported.capture: + v1: + messages: + - role: user + content_file: messages/shared.tmpl + output: + format: text + validation_mode: none +`) + writeFile(filepath.Join(declaringDirectory, "messages", "shared.tmpl"), "declared import") + writeFile(replacementConfig, `prompts: + adapter.test: + v1: + messages: + - role: user + content_file: messages/shared.tmpl +`) + writeFile(filepath.Join(replacementDirectory, "messages", "shared.tmpl"), "replacement original") + + endpoint, model := "https://example.invalid/v1", "content-context-model" + newClient := func() *PromptKitClient { + t.Helper() + client, err := NewPromptKitClient(PromptKitClientConfig{ + Assets: newTestPromptKitAssets(t), + Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{ + {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("content context profile", map[string]promptkit.ProfileConfig{ + "content-context-profile": {Endpoint: &endpoint, Model: &model}, + })}}}, + {Inputs: []configkit.Input{{Source: configkit.File(rootConfig), Imports: true}}}, + {Inputs: []configkit.Input{{Source: configkit.File(replacementConfig), Imports: true}}}, + }}, + }) + if err != nil { + t.Fatalf("NewPromptKitClient() error = %v", err) + } + return client + } + prepareMessage := func(client *PromptKitClient, promptID string) string { + t.Helper() + request := promptkit.RunRequest{PromptID: promptID, ProfileID: "content-context-profile", SessionID: "content-context-session"} + if promptID == "adapter.test" { + request.Inputs = map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline(`{"source":true}`), + } + } + prepared, err := client.engine.Prepare(context.Background(), request) + if err != nil { + t.Fatalf("Prepare(%q) error = %v", promptID, err) + } + if len(prepared.Messages) != 1 { + t.Fatalf("Prepare(%q) messages = %#v, want one", promptID, prepared.Messages) + } + return prepared.Messages[0].Content + } + + first := newClient() + if got := prepareMessage(first, "imported.capture"); got != "declared import" { + t.Fatalf("imported prompt content = %q, want declaring-source content", got) + } + if got := prepareMessage(first, "adapter.test"); got != "replacement original" { + t.Fatalf("partially overlaid application prompt content = %q, want replacement-source content", got) + } + writeFile(filepath.Join(declaringDirectory, "messages", "shared.tmpl"), "declared import updated") + writeFile(filepath.Join(replacementDirectory, "messages", "shared.tmpl"), "replacement updated") + if got := prepareMessage(first, "imported.capture"); got != "declared import" { + t.Fatalf("existing engine imported prompt content after edit = %q", got) + } + if got := prepareMessage(first, "adapter.test"); got != "replacement original" { + t.Fatalf("existing engine replacement content after edit = %q", got) + } + second := newClient() + if got := prepareMessage(second, "imported.capture"); got != "declared import updated" { + t.Fatalf("new engine imported prompt content = %q", got) + } + if got := prepareMessage(second, "adapter.test"); got != "replacement updated" { + t.Fatalf("new engine replacement content = %q", got) + } +} + +func TestPromptKitStaticMessageFailuresOccurAtConstruction(t *testing.T) { + for _, test := range []struct { + name string + prompt string + files fstest.MapFS + }{ + { + name: "missing message file", + prompt: `prompts: + construction.failure: + v1: + messages: + - role: user + content_file: messages/missing.tmpl + output: + format: text + validation_mode: none +`, + }, + { + name: "invalid static template", + prompt: `prompts: + construction.failure: + v1: + messages: + - role: user + content: '{{ .request_value' + output: + format: text + validation_mode: none +`, + }, + } { + t.Run(test.name, func(t *testing.T) { + assets := NewAssetRegistry() + files := test.files + if files == nil { + files = fstest.MapFS{} + } + files["prompt.yaml"] = &fstest.MapFile{Data: []byte(test.prompt)} + if err := assets.RegisterPromptFS(files, "."); err != nil { + t.Fatal(err) + } + _, err := NewPromptKitClient(PromptKitClientConfig{ + Assets: assets, + Sources: promptKitTestSources(nil, map[string]promptkit.ProfileConfig{"construction-profile": { + Endpoint: promptKitTestString("https://example.invalid/v1"), + Model: promptKitTestString("construction-model"), + }}), + }) + if err == nil || !errors.Is(err, promptkit.ErrInvalidConfig) { + t.Fatalf("NewPromptKitClient() error = %v, want static construction failure", err) + } + }) + } +} + +func TestPromptKitMissingRequestVariableFailsDuringPreparation(t *testing.T) { + assets := NewAssetRegistry() + if err := assets.RegisterPromptFS(fstest.MapFS{ + "prompt.yaml": {Data: []byte(`prompts: + request.variable: + v1: + messages: + - role: user + content: 'Value: {{ .request_value }}' + output: + format: text + validation_mode: none +`)}, + }, "."); err != nil { + t.Fatal(err) + } + client, err := NewPromptKitClient(PromptKitClientConfig{ + Assets: assets, + Sources: promptKitTestSources(nil, map[string]promptkit.ProfileConfig{"request-profile": { + Endpoint: promptKitTestString("https://example.invalid/v1"), + Model: promptKitTestString("request-model"), + }}), + }) + if err != nil { + t.Fatalf("NewPromptKitClient() error = %v, want valid static template", err) + } + if _, err := client.engine.Prepare(context.Background(), promptkit.RunRequest{ + PromptID: "request.variable", ProfileID: "request-profile", + }); !errors.Is(err, promptkit.ErrPromptRender) { + t.Fatalf("Prepare() error = %v, want missing request variable render failure", err) + } +} + func TestPromptKitProfileSourceGroupsPreserveNativeLayerRules(t *testing.T) { t.Run("directory group conflicts", func(t *testing.T) { directory := t.TempDir()