diff --git a/application_handoff_integration_test.go b/application_handoff_integration_test.go new file mode 100644 index 0000000..f9e9ae7 --- /dev/null +++ b/application_handoff_integration_test.go @@ -0,0 +1,325 @@ +package promptkit_test + +import ( + "bytes" + "context" + "errors" + "fmt" + "reflect" + "strings" + "sync" + "sync/atomic" + "testing" + "testing/fstest" + + "go.maximumdirect.net/configkit" + "go.maximumdirect.net/promptkit" +) + +type integrationComponent struct { + ID string `yaml:"id"` +} + +type integrationApplication struct { + Version string `yaml:"version"` + Sibling string `yaml:"sibling"` + Components []integrationComponent `yaml:"components"` + Prompts []string `yaml:"prompts"` + PromptKit promptkit.Settings `yaml:"promptkit"` +} + +func integrationDefinition(resolves, validates *atomic.Int64) configkit.Definition[integrationApplication] { + definition := configkit.Definition[integrationApplication]{ + Defaults: func() integrationApplication { return integrationApplication{Sibling: "default"} }, + Version: &configkit.VersionRule{Path: configkit.Keys("version"), Check: func(version configkit.Version, validation *configkit.Validation) error { + if version.Present && version.Text != "app-v1" { + validation.Error(configkit.Keys("version"), "invalid application version") + } + return nil + }}, + SequenceIdentities: []configkit.SequenceIdentity{configkit.SequenceIdentityAt(configkit.Keys("components"), "id")}, + Sensitive: func(path configkit.Path) bool { return path.String() == "/sibling" }, + Resolve: func(_ *integrationApplication, _ *configkit.Resolution) error { resolves.Add(1); return nil }, + Validate: func(_ *integrationApplication, validation *configkit.Validation) { + validates.Add(1) + validation.Warning(configkit.Keys("sibling"), "application warning") + }, + } + configured := promptkit.Configure(definition, func(value *integrationApplication) *promptkit.Settings { return &value.PromptKit }) + previous := configured.Validate + configured.Validate = func(value *integrationApplication, validation *configkit.Validation) { + previous(value, validation) + if len(value.PromptKit.Backends) != 1 || value.PromptKit.Backends[0].Endpoint == nil || strings.Contains(*value.PromptKit.Backends[0].Endpoint, " ") { + validation.Error(configkit.Keys("promptkit"), "backend normalization missing") + } + if value.PromptKit.Application["section_extra"] != "section" || value.PromptKit.Backends[0].Application["backend_extra"] != "backend" || value.PromptKit.Profiles[0].Application["profile_extra"] != "profile" || value.PromptKit.Prompts[0].Application["prompt_extra"] != "prompt" || value.PromptKit.Collections["group"].Application["collection_extra"] != "collection" { + validation.Error(configkit.Keys("promptkit"), "application fields missing") + } + } + return configured +} + +const integrationDocument = `version: app-v1 +prompts: [application-only] +components: [{id: second-component}] +promptkit: + section_extra: section + backends: + - id: local + endpoint: ' https://example.test/v1 ' + extra_params: {provider: {mode: original}} + backend_extra: backend + profiles: + - id: profile + backend: local + model: original-model + profile_extra: profile + prompts: + - id: example + default_profile: profile + inputs: [{name: document, required: false}] + messages: [{role: user, content: hello}] + output: + format: json + validation_mode: json_schema + schema: {type: object, required: [ok], properties: {ok: {type: boolean}}} + prompt_extra: prompt + collections: + group: {collection_extra: collection} +` + +func TestApplicationOwnedLoadAndCapture(t *testing.T) { + var resolves, validates atomic.Int64 + definition := integrationDefinition(&resolves, &validates) + result, err := configkit.Load(configkit.Bytes("application", []byte(integrationDocument)), configkit.Options[integrationApplication]{Definition: definition}) + if err != nil { + t.Fatal(err) + } + if result.Config.Sibling != "default" || result.Config.Prompts[0] != "application-only" || len(result.Diagnostics()) != 1 || result.Config.PromptKit.Prompts[0].Output.RepairAttempts == nil || *result.Config.PromptKit.Prompts[0].Output.RepairAttempts != 1 { + t.Fatalf("loaded application = %+v, diagnostics = %+v", result.Config, result.Diagnostics()) + } + client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: `{"ok":true}`}} + engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client)) + if err != nil { + t.Fatal(err) + } + if resolves.Load() != 1 || validates.Load() != 1 { + t.Fatalf("callbacks rerun during handoff: %d %d", resolves.Load(), validates.Load()) + } + before, err := engine.InspectPrompt(context.Background(), "example") + if err != nil { + t.Fatal(err) + } + app := &result.Config.PromptKit + *app.Backends[0].Endpoint = "https://changed.test/v1" + app.Backends[0].ExtraParams["provider"].(map[string]any)["mode"] = "changed" + *app.Profiles[0].Model = "changed-model" + *app.Prompts[0].Messages[0].Content = "changed" + app.Prompts[0].Inputs[0].Name = "changed" + app.Prompts[0].Output.Schema.(map[string]any)["required"].([]any)[0] = "changed" + app.Prompts = nil + app.Collections = nil + after, err := engine.InspectPrompt(context.Background(), "example") + if err != nil || !reflect.DeepEqual(before, after) { + t.Fatalf("captured prompt changed: %+v, %+v, %v", before, after, err) + } + run, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "example"}) + if err != nil || run.Validation.Status != promptkit.ValidationPassed || len(client.requests) != 1 || client.requests[0].Prompt.Messages[0].Content != "hello" || client.requests[0].Target.Model != "original-model" { + t.Fatalf("run = %+v, requests = %+v, error = %v", run, client.requests, err) + } + if strings.Contains(fmt.Sprintf("%+v", client.requests[0].Target.ExtraParams), "backend_extra") || strings.Contains(fmt.Sprintf("%+v", after), "prompt_extra") { + t.Fatal("application fields leaked into resources or provider parameters") + } + view, err := engine.InspectConfiguration() + if err != nil { + t.Fatal(err) + } + var report bytes.Buffer + if err := view.WriteEffective(&report); err != nil || !strings.Contains(report.String(), "section_extra: section") || !strings.Contains(report.String(), "prompt_extra: prompt") { + t.Fatalf("inspection = %q, %v", report.String(), err) + } + if len(view.Diagnostics()) != 1 { + t.Fatalf("diagnostics = %+v", view.Diagnostics()) + } + var wg sync.WaitGroup + for range 4 { + wg.Go(func() { + _, err := configkit.Load(configkit.Bytes("repeat", []byte(integrationDocument)), configkit.Options[integrationApplication]{Definition: definition}) + if err != nil { + t.Error(err) + } + }) + } + wg.Wait() + if resolves.Load() != 5 || validates.Load() != 5 { + t.Fatalf("reused callbacks = %d %d", resolves.Load(), validates.Load()) + } +} + +func TestHandoffMountedSourcesAndExecution(t *testing.T) { + baseFS := &countingMountFS{FS: fstest.MapFS{ + "base/collection.yaml": {Data: []byte("prompts:\n - id: parent\n default_profile: profile\n messages: [{role: user, content_file: original.txt}]\n output: {format: json, validation_mode: json_schema, schema_file: schema.json}\n")}, + "base/original.txt": {Data: []byte("original")}, "base/schema.json": {Data: []byte(`{"type":"object"}`)}, + }} + operatorFS := &countingMountFS{FS: fstest.MapFS{ + "operator/item.yaml": {Data: []byte("id: child\nextends: {id: parent}\nmessages: [{role: user, content_file: replacement.txt}]\n")}, + "operator/replacement.txt": {Data: []byte("replacement")}, + }} + base := configkit.NewFilesystem("embedded", baseFS) + operator := configkit.NewFilesystem("operator", operatorFS) + collection, err := promptkit.CollectionInput("team", base.File("base/collection.yaml")) + if err != nil { + t.Fatal(err) + } + collection.Filesystem = &configkit.FilesystemContext{FS: &base, Root: "."} + item := configkit.Input{Source: operator.File("operator/item.yaml"), Destination: configkit.Keys("promptkit", "collections", "team", "prompts"), PlacementMode: configkit.PlacementModeSequenceItem, Filesystem: &configkit.FilesystemContext{FS: &operator, Root: "."}} + profile := promptkit.SettingsInput(configkit.Mapping("profile", map[string]any{"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.test/v1", "model": "model"}}})) + type root struct { + PromptKit promptkit.Settings `yaml:"promptkit"` + } + definition := promptkit.Configure(configkit.Definition[root]{}, func(value *root) *promptkit.Settings { return &value.PromptKit }) + layers := []configkit.Underlay{{Inputs: []configkit.Input{profile, collection}}, {Inputs: []configkit.Input{item}}} + result, err := configkit.LoadLayered(configkit.Input{Source: configkit.NoSource()}, configkit.LayeredOptions[root]{Definition: definition, Underlays: layers}) + if err != nil { + t.Fatal(err) + } + beforeBase, beforeOperator := baseFS.opens, operatorFS.opens + client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: `{}`}} + engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client)) + if err != nil { + t.Fatal(err) + } + if baseFS.opens-beforeBase != 1 || operatorFS.opens-beforeOperator != 1 { + t.Fatalf("unexpected construction reads: base=%d operator=%d", baseFS.opens-beforeBase, operatorFS.opens-beforeOperator) + } + standalone, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: layers}}, promptkit.WithBundledOpenRouter(false), promptkit.WithLLMClient(client)) + if err != nil { + t.Fatal(err) + } + wantPrompt, err := standalone.InspectPrompt(context.Background(), "team/child") + if err != nil { + t.Fatal(err) + } + gotPrompt, err := engine.InspectPrompt(context.Background(), "team/child") + if err != nil || !reflect.DeepEqual(gotPrompt, wantPrompt) { + t.Fatalf("resource reports = %+v, %+v, %v", gotPrompt, wantPrompt, err) + } + wantInventory, err := standalone.ListPrompts(context.Background()) + if err != nil { + t.Fatal(err) + } + gotInventory, err := engine.ListPrompts(context.Background()) + if err != nil || !reflect.DeepEqual(gotInventory, wantInventory) { + t.Fatalf("inventories = %+v, %+v, %v", gotInventory, wantInventory, err) + } + collectionSettings := result.Config.PromptKit.Collections["team"] + *collectionSettings.Prompts[1].Output.SchemaFile = "changed.json" + prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "team/child"}) + if err != nil || prepared.Messages[0].Content != "replacement" { + t.Fatalf("prepared = %+v, %v", prepared, err) + } + standalonePrepared, err := standalone.Prepare(context.Background(), promptkit.RunRequest{PromptID: "team/child"}) + if err != nil || !reflect.DeepEqual(prepared.StructuredOutput, standalonePrepared.StructuredOutput) || !reflect.DeepEqual(prepared.OutputContract, standalonePrepared.OutputContract) || !reflect.DeepEqual(prepared.Messages, standalonePrepared.Messages) { + t.Fatalf("prepared details differ: %+v, %+v, %v", prepared, standalonePrepared, err) + } + run, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "team/child"}) + if err != nil || run.Validation.Status != promptkit.ValidationPassed { + t.Fatalf("run = %+v, %v", run, err) + } +} + +func TestHandoffUsesOnlyBundlesSelectedBeforeLoad(t *testing.T) { + type root struct { + PromptKit promptkit.Settings `yaml:"promptkit"` + } + definition := promptkit.Configure(configkit.Definition[root]{}, func(value *root) *promptkit.Settings { return &value.PromptKit }) + result, err := configkit.LoadLayered(configkit.Input{Source: configkit.NoSource()}, configkit.LayeredOptions[root]{Definition: definition, Underlays: []configkit.Underlay{promptkit.BundledOpenRouterUnderlay()}}) + if err != nil { + t.Fatal(err) + } + engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithBundledOpenRouter(false)) + if err != nil { + t.Fatal(err) + } + backends, err := engine.ListBackends(context.Background()) + if err != nil || len(backends) != 1 || backends[0] != "openrouter" { + t.Fatalf("bundles = %+v, %v", backends, err) + } +} + +func TestApplicationFieldsComposeAndInherit(t *testing.T) { + type root struct { + PromptKit promptkit.Settings `yaml:"promptkit"` + } + definition := promptkit.Configure(configkit.Definition[root]{}, func(value *root) *promptkit.Settings { return &value.PromptKit }) + underlay := promptkit.SettingsInput(configkit.Mapping("defaults", map[string]any{ + "section_data": map[string]any{"first": "section"}, + "backends": []any{map[string]any{"id": "base", "endpoint": "https://example.test/v1", "app_data": map[string]any{"first": "backend"}}}, + "profiles": []any{map[string]any{"id": "base", "model": "model", "app_data": map[string]any{"first": "profile"}}}, + "prompts": []any{map[string]any{"id": "base", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}, "app_data": map[string]any{"first": "prompt"}}}, + "collections": map[string]any{"team": map[string]any{"app_data": map[string]any{"first": "collection"}}}, + })) + main := promptkit.SettingsInput(configkit.Mapping("operator", map[string]any{ + "section_data": map[string]any{"second": "section"}, + "backends": []any{map[string]any{"id": "child", "extends": map[string]any{"id": "base"}, "app_data": map[string]any{"second": "backend"}}}, + "profiles": []any{map[string]any{"id": "child", "extends": map[string]any{"id": "base"}, "app_data": map[string]any{"second": "profile"}}}, + "prompts": []any{map[string]any{"id": "child", "extends": map[string]any{"id": "base"}, "app_data": map[string]any{"second": "prompt"}}}, + "collections": map[string]any{"team": map[string]any{"app_data": map[string]any{"second": "collection"}}}, + })) + result, err := configkit.LoadLayered(main, configkit.LayeredOptions[root]{Definition: definition, Underlays: []configkit.Underlay{{Inputs: []configkit.Input{underlay}}}}) + if err != nil { + t.Fatal(err) + } + settings := result.Config.PromptKit + for name, fields := range map[string]map[string]any{"section": settings.Application, "backend": settings.Backends[1].Application, "profile": settings.Profiles[1].Application, "prompt": settings.Prompts[1].Application, "collection": settings.Collections["team"].Application} { + key := "app_data" + if name == "section" { + key = "section_data" + } + data, ok := fields[key].(map[string]any) + if !ok || data["first"] != name || data["second"] != name { + t.Errorf("%s extension = %#v", name, fields[key]) + } + } +} + +func TestHandoffMissingAndNullSections(t *testing.T) { + type noSection struct { + Prompts []string `yaml:"prompts"` + } + missing, err := configkit.Load[noSection](configkit.Bytes("application-only", []byte("prompts: [application]\n"))) + if err != nil { + t.Fatal(err) + } + empty, err := promptkit.NewFromConfig(missing, promptkit.RuntimeConfig{}) + if err != nil { + t.Fatal(err) + } + if prompts, err := empty.ListPrompts(context.Background()); err != nil || len(prompts) != 0 { + t.Fatalf("application root resources were consumed: %+v, %v", prompts, err) + } + for _, source := range []string{"{}", "promptkit: null", "promptkit: {}"} { + t.Run(source, func(t *testing.T) { + type root struct { + PromptKit *promptkit.Settings `yaml:"promptkit"` + Prompts []string `yaml:"prompts"` + } + definition := promptkit.Configure(configkit.Definition[root]{}, func(value *root) *promptkit.Settings { return value.PromptKit }) + result, err := configkit.Load(configkit.Bytes("missing", []byte(source)), configkit.Options[root]{Definition: definition}) + if err != nil { + t.Fatal(err) + } + engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}) + if err != nil { + t.Fatal(err) + } + prompts, err := engine.ListPrompts(context.Background()) + if err != nil || len(prompts) != 0 { + t.Fatalf("prompts = %+v, %v", prompts, err) + } + }) + } + if _, err := promptkit.NewFromConfig(nil, promptkit.RuntimeConfig{}); !errors.Is(err, promptkit.ErrInvalidConfig) { + t.Fatalf("nil result = %v", err) + } +} diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index a90f637..3ece068 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -553,6 +553,9 @@ and configuration contracts; expected test call-site migrations are complete. ## Stage 13 — Add Application-Owned Handoff Integration Coverage +Status: complete. External handoff integration tests passed under the race +detector, including concurrent loads, source contexts, and ownership checks. + Focus: a small external-package root integration test group, with real ConfigKit and PromptKit collaborators and an offline model client.