diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 495ed52..6a98471 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -380,6 +380,25 @@ Verify app overrides are available and framework overrides are not emulated. ## Stage 4: Install Native PromptKit Resources, Requirements, And Preview +**Status: Complete.** Composed all twelve native callbacks, empty-default +namespace views, and native convenience declarations. Request builders remain +nil so native seeds pass through unchanged. Every callback carries application +acquisition evidence separately, preserving framework reports and typed errors. + +Verified on Go 1.27.4 with focused real-library tests, race checks, and vet: +exact required and undeclared input names, optional/conditional dependencies, +explicit-empty variable presence, complete versus partial reports and structured +issue codes, qualified prompt/profile identities, namespace selection, selection +failures, and discovery without artifacts, schemas, or generation credentials. +Also checked five inventories and native default/filter behavior, three named +inspections, content-bearing prepared reports in automatic/text/YAML/JSON modes, +and concrete validation/preparation without generation. A loopback-only provider +verified explicit smoke execution and native unsuccessful findings with status +zero, without disclosing provider payloads. Narratio's shared-context discovery, +validation, and preview coverage is present; execution consistency is checked +with positional generation in Stage 5 and the assembled workflow in Stage 9. +Legacy-file compilation cleanup remains deferred to Stage 8. + **Primary files:** CLI PromptKit-suite composition, engine-loading adapters, and behavior tests replacing custom definition/prepared inspection tests. diff --git a/internal/adapter/cli/app.go b/internal/adapter/cli/app.go index 5296ef1..57a2948 100644 --- a/internal/adapter/cli/app.go +++ b/internal/adapter/cli/app.go @@ -20,7 +20,7 @@ func Execute(ctx context.Context, identity commandkit.Identity, args []string, s } func newApp(identity commandkit.Identity) (*commandkit.App, error) { - definitions := ckprompt.FromEngine(loadEngine) + definitions := promptSuite() declarations := []commandkit.Declaration{ commandkit.NewCommand(commandkit.CommandSpec[struct{}]{Name: "run", Summary: "Run a stored prompt.", Run: commandNotImplemented}), commandkit.NewCommand(commandkit.CommandSpec[serveInput]{Name: "serve", Summary: "Serve the HTTP API.", Flags: serveFlags(), Run: func(commandkit.Invocation, serveInput) commandkit.Outcome { diff --git a/internal/adapter/cli/native_prompt.go b/internal/adapter/cli/native_prompt.go new file mode 100644 index 0000000..b2b069b --- /dev/null +++ b/internal/adapter/cli/native_prompt.go @@ -0,0 +1,40 @@ +package cli + +import ( + "context" + + "go.maximumdirect.net/ckprompt" + "go.maximumdirect.net/commandkit" + "go.maximumdirect.net/promptkit" +) + +func promptSuite() ckprompt.Suite[applicationInput, *acquisitionState] { + suite := ckprompt.FromEngine(loadEngine) + suite.NewInput = func() applicationInput { return applicationInput{} } + suite.NewState = func() *acquisitionState { return &acquisitionState{} } + suite = ckprompt.WithRequestValidation(suite, loadEngine, nil) + suite = ckprompt.WithRequestPreparation(suite, loadEngine, nil) + suite = ckprompt.WithSmokeTesting(suite, loadEngine, nil) + suite = ckprompt.WithNamespaces(suite, promptkit.Namespaces{}) + suite.ListPrompts = withApplicationEvidence(suite.ListPrompts) + suite.ListProfiles = withApplicationEvidence(suite.ListProfiles) + suite.ListBackends = withApplicationEvidence(suite.ListBackends) + suite.ListNamespaces = withApplicationEvidence(suite.ListNamespaces) + suite.ListCollections = withApplicationEvidence(suite.ListCollections) + suite.InspectPrompt = withApplicationEvidence(suite.InspectPrompt) + suite.InspectProfile = withApplicationEvidence(suite.InspectProfile) + suite.InspectBackend = withApplicationEvidence(suite.InspectBackend) + suite.InspectRequirements = withApplicationEvidence(suite.InspectRequirements) + suite.ValidateRequest = withApplicationEvidence(suite.ValidateRequest) + suite.PrepareRequest = withApplicationEvidence(suite.PrepareRequest) + suite.SmokeTest = withApplicationEvidence(suite.SmokeTest) + return suite +} + +func withApplicationEvidence[R, V any](callback func(context.Context, R, applicationInput, *acquisitionState) (commandkit.Acquired[V], error)) func(context.Context, R, applicationInput, *acquisitionState) (commandkit.Acquired[V], error) { + return func(ctx context.Context, request R, input applicationInput, state *acquisitionState) (commandkit.Acquired[V], error) { + start := state.mark() + acquired, err := callback(ctx, request, input, state) + return attachApplicationEvidence(acquired, err, state, start, false) + } +} diff --git a/internal/adapter/cli/native_prompt_test.go b/internal/adapter/cli/native_prompt_test.go new file mode 100644 index 0000000..3ac69d0 --- /dev/null +++ b/internal/adapter/cli/native_prompt_test.go @@ -0,0 +1,361 @@ +package cli + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "sync/atomic" + "testing" + + "go.maximumdirect.net/ckconfig" + "go.maximumdirect.net/commandkit" + "go.maximumdirect.net/promptkit" +) + +func nativePromptFixture(t *testing.T, endpoint string) { + t.Helper() + t.Chdir(t.TempDir()) + writeNativeFile(t, "app.yml", "promptkit:\n configuration: framework.yml\n schema_dir: nonexistent-schemas\ntransient_retries: 0\n") + writeNativeFile(t, "framework.yml", fmt.Sprintf(`collections: + support: + release_version: '1' + backends: + - id: local + endpoint: %q + profiles: + - id: writer + backend: support/local + model: controlled-model + - id: protected + backend: support/local + model: controlled-model + api_key_required: true + prompts: + - id: summary + default_profile: support/protected + session_id: '{{.session_variable}}' + inputs: + - name: transcript + required: true + content_type: text/plain + description: Exact transcript binding + - name: glossary + required: false + description: '' + messages: + - role: user + content: 'template-canary {{input "transcript"}} {{input "unlisted"}} {{if .mode}}{{input "glossary"}}{{end}} {{.title}}' + output: + format: text + validation_mode: none + - id: dynamic + default_profile: support/protected + messages: + - role: user + content: '{{input .source}}' + output: + format: text + validation_mode: none + - id: unsupported + default_profile: support/protected + messages: + - role: user + content: '{{define "child"}}{{input "called"}}{{end}}{{template "child" .}}' + output: + format: text + validation_mode: none + - id: schema-needed + default_profile: support/protected + messages: + - role: user + content: 'Return JSON' + output: + format: json + validation_mode: json_schema + schema_path: missing.json + review: + release_version: '1' + profiles: + - id: writer + backend: support/local + model: alternate-model + prompts: + - id: summary + default_profile: review/writer + inputs: + - name: memo + required: true + messages: + - role: user + content: '{{input "memo"}}' + output: + format: text + validation_mode: none +`, endpoint)) +} + +type consumerRequirements struct { + PromptID string `json:"prompt_id"` + ProfileID string `json:"profile_id"` + Inputs []struct { + Name string `json:"name"` + Kind promptkit.RequirementKind `json:"kind"` + Declared bool `json:"declared"` + ContentType *string `json:"content_type"` + Description *string `json:"description"` + } `json:"inputs"` + Variables []struct { + Name string `json:"name"` + Supplied bool `json:"supplied"` + } `json:"variables"` + Complete bool `json:"complete"` + Issues []struct { + Code promptkit.RequirementIssueCode `json:"code"` + } `json:"issues"` +} + +func discoverNative(t *testing.T, prompt string, flags ...string) consumerRequirements { + t.Helper() + args := []string{"promptkit", "requirements", "show", prompt, "--config=app.yml", "--format=json"} + args = append(args, flags...) + outcome, output, diagnostics := executeNative(t, args...) + if outcome.Status != 0 { + t.Fatalf("discovery: %+v %q %q", outcome, output, diagnostics) + } + var report consumerRequirements + if err := json.Unmarshal([]byte(output), &report); err != nil { + t.Fatal(err) + } + for _, excluded := range []string{"template-canary", "unrelated-variable-value", "schema_version"} { + if strings.Contains(output, excluded) { + t.Fatalf("unexpected discovery content %q: %s", excluded, output) + } + } + return report +} + +func TestRequirementsExposeExactBindingsBeforeArtifactsExist(t *testing.T) { + nativePromptFixture(t, "http://127.0.0.1:1/v1") + report := discoverNative(t, "support/summary", "--var=mode=", "--var=unrelated=unrelated-variable-value", "--session-id=session") + if !report.Complete || report.PromptID != "support/summary" || report.ProfileID != "support/protected" { + t.Fatalf("discovery identity/completeness: %+v", report) + } + var names []string + for _, input := range report.Inputs { + names = append(names, input.Name) + switch input.Name { + case "transcript": + if input.Kind != promptkit.RequirementRequired || !input.Declared || input.ContentType == nil || *input.ContentType != "text/plain" || input.Description == nil { + t.Fatalf("required metadata: %+v", input) + } + case "unlisted": + if input.Kind != promptkit.RequirementRequired || input.Declared { + t.Fatalf("undeclared reference: %+v", input) + } + case "glossary": + if input.Kind != promptkit.RequirementOptional || input.Description == nil || *input.Description != "" { + t.Fatalf("optional metadata: %+v", input) + } + } + } + if !reflect.DeepEqual(names, []string{"glossary", "transcript", "unlisted"}) { + t.Fatalf("exact input names: %v", names) + } + var emptySupplied, absent bool + for _, variable := range report.Variables { + if variable.Name == "mode" { + emptySupplied = variable.Supplied + } + if variable.Name == "title" { + absent = !variable.Supplied + } + } + if !emptySupplied || !absent { + t.Fatalf("variable presence: %+v", report.Variables) + } + schemaIndependent := discoverNative(t, "support/schema-needed") + if !schemaIndependent.Complete || len(schemaIndependent.Inputs) != 0 { + t.Fatalf("schema-free discovery: %+v", schemaIndependent) + } + // Complete discovery does not assert that these required bindings exist. + outcome, _, diagnostics := executeNative(t, "promptkit", "requirements", "validate", "support/summary", "--config=app.yml", "--profile=support/writer", "--var=mode=", "--var=title=Title", "--session-id=session") + if outcome.Status == 0 || diagnostics == "" { + t.Fatalf("unbound request validated: %+v %q", outcome, diagnostics) + } + outcome, _, diagnostics = executeNative(t, "promptkit", "show", "prepared", "support/schema-needed", "--config=app.yml", "--profile=support/writer") + if outcome.Status == 0 || diagnostics == "" { + t.Fatalf("missing preparation schema accepted: %+v %q", outcome, diagnostics) + } +} + +func TestRequirementsPreserveConditionalAndPartialDiscovery(t *testing.T) { + nativePromptFixture(t, "http://127.0.0.1:1/v1") + unknown := discoverNative(t, "support/summary", "--session-id=session") + if unknown.Complete || len(unknown.Issues) == 0 { + t.Fatalf("unknown condition claimed complete: %+v", unknown) + } + for _, input := range unknown.Inputs { + if input.Name == "glossary" && input.Kind != promptkit.RequirementConditional { + t.Fatalf("unknown branch: %+v", input) + } + } + known := discoverNative(t, "support/summary", "--var=mode=include", "--session-id=session") + if !known.Complete { + t.Fatalf("known branch incomplete: %+v", known) + } + for _, input := range known.Inputs { + if input.Name == "glossary" && input.Kind != promptkit.RequirementRequired { + t.Fatalf("known branch: %+v", input) + } + } + dynamic := discoverNative(t, "support/dynamic") + if dynamic.Complete || len(dynamic.Issues) == 0 { + t.Fatalf("dynamic name not partial: %+v", dynamic) + } + resolved := discoverNative(t, "support/dynamic", "--var=source=exact,name") + if !resolved.Complete || len(resolved.Inputs) != 1 || resolved.Inputs[0].Name != "exact,name" || resolved.Inputs[0].Declared { + t.Fatalf("resolved identifier: %+v", resolved) + } + unsupported := discoverNative(t, "support/unsupported") + var found bool + for _, issue := range unsupported.Issues { + found = found || issue.Code == promptkit.RequirementIssueUnsupportedConstruct + } + if unsupported.Complete || !found { + t.Fatalf("unsupported analysis not preserved: %+v", unsupported) + } +} + +func TestRequirementsSelectionUsesNativeNamespacesAndFailures(t *testing.T) { + nativePromptFixture(t, "http://127.0.0.1:1/v1") + before := discoverNative(t, "summary", "--prompt-namespace=support", "--var=mode=", "--session-id=session") + after := discoverNative(t, "support/summary", "--prompt-namespace=review", "--profile=writer", "--profile-namespace=review", "--var=mode=", "--session-id=session") + if after.PromptID != "support/summary" || after.ProfileID != "review/writer" || !reflect.DeepEqual(before.Inputs, after.Inputs) { + t.Fatalf("identity/definition changed: before=%+v after=%+v", before, after) + } + other := discoverNative(t, "review/summary") + if other.PromptID != "review/summary" || other.ProfileID != "review/writer" || len(other.Inputs) != 1 || other.Inputs[0].Name != "memo" { + t.Fatalf("other definition: %+v", other) + } + for _, args := range [][]string{ + {"promptkit", "requirements", "show", "missing", "--config=app.yml"}, + {"promptkit", "requirements", "show", "support/summary", "--config=app.yml", "--profile=missing"}, + } { + outcome, output, diagnostics := executeNative(t, args...) + if outcome.Status == 0 || output != "" || diagnostics == "" { + t.Fatalf("selection failure: %+v %q %q", outcome, output, diagnostics) + } + } +} + +func TestNativeInventoriesInspectionAndPreparationDoNotGenerate(t *testing.T) { + var calls atomic.Int64 + var fail atomic.Bool + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + w.Header().Set("Content-Type", "application/json") + if fail.Load() { + w.WriteHeader(http.StatusServiceUnavailable) + fmt.Fprint(w, `{"error":{"message":"provider-private-canary"}}`) + return + } + fmt.Fprint(w, `{"choices":[{"message":{"role":"assistant","content":"controlled"}}]}`) + })) + defer server.Close() + nativePromptFixture(t, server.URL+"/v1") + for _, args := range [][]string{ + {"promptkit", "list", "prompts", "--all-namespaces"}, + {"promptkit", "list", "profiles", "--namespace=support"}, + {"promptkit", "list", "backends", "--namespace=support"}, + {"promptkit", "list", "namespaces"}, + {"promptkit", "list", "collections"}, + {"promptkit", "show", "prompt", "support/summary"}, + {"promptkit", "show", "profile", "support/writer"}, + {"promptkit", "show", "backend", "support/local"}, + } { + args = append(args, "--config=app.yml", "--format=json") + outcome, output, diagnostics := executeNative(t, args...) + if outcome.Status != 0 || output == "" { + t.Fatalf("%v: %+v %q %q", args, outcome, output, diagnostics) + } + var value any + if err := json.Unmarshal([]byte(output), &value); err != nil { + t.Fatal(err) + } + if args[1] == "show" && args[2] == "prompt" && !strings.Contains(output, "template-canary") { + t.Fatalf("native prompt content omitted: %s", output) + } + } + engine, err := loadEngine(context.Background(), ckconfig.Selection{Config: commandkit.Supplied[string]{Value: "app.yml", Set: true}}, applicationInput{}, nil) + if err != nil { + t.Fatal(err) + } + nativePrompts, err := engine.ListPrompts(context.Background()) + if err != nil { + t.Fatal(err) + } + outcome, output, diagnostics := executeNative(t, "promptkit", "list", "prompts", "--config=app.yml", "--format=json") + var listed []promptkit.PromptIdentity + if err := json.Unmarshal([]byte(output), &listed); err != nil { + t.Fatal(err) + } + if outcome.Status != 0 || !reflect.DeepEqual(listed, nativePrompts) { + t.Fatalf("native default listing changed: %+v %q %q", outcome, output, diagnostics) + } + outcome, output, diagnostics = executeNative(t, "promptkit", "list", "prompts", "--namespace=review", "--config=app.yml", "--format=json") + if outcome.Status != 0 || strings.Contains(output, "support/summary") || !strings.Contains(output, "review/summary") { + t.Fatalf("namespace filter lost: %+v %q %q", outcome, output, diagnostics) + } + writeNativeFile(t, "transcript.md", "artifact-payload-canary") + flags := []string{"summary", "--config=app.yml", "--prompt-namespace=support", "--profile=writer", "--profile-namespace=support", "--session-id=session", "--var=mode=", "--var=title=Title", "--input-file=transcript=transcript.md", "--input=unlisted=literal"} + for _, command := range [][]string{{"promptkit", "requirements", "validate"}, {"promptkit", "show", "prepared"}} { + args := append(append([]string{}, command...), flags...) + outcome, output, diagnostics := executeNative(t, append(args, "--format=json")...) + if outcome.Status != 0 { + t.Fatalf("%v: %+v %q %q", args, outcome, output, diagnostics) + } + if command[1] == "show" && (!strings.Contains(output, "artifact-payload-canary") || !strings.Contains(output, "support/writer") || !strings.Contains(output, "support/summary")) { + t.Fatalf("native prepared content/context lost: %s", output) + } + } + for _, format := range []string{"", "text", "yaml"} { + args := append([]string{"promptkit", "show", "prepared"}, flags...) + if format != "" { + args = append(args, "--format="+format) + } + outcome, output, diagnostics := executeNative(t, args...) + if outcome.Status != 0 || !strings.Contains(output, "artifact-payload-canary") { + t.Fatalf("native preview %q: %+v %q %q", format, outcome, output, diagnostics) + } + } + if calls.Load() != 0 { + t.Fatal("inspection/preparation contacted provider") + } + // Smoke tests are the explicit provider-effect boundary. Native findings, + // not an application readiness policy, determine what the report says. + outcome, output, diagnostics = executeNative(t, "promptkit", "test", "text_inference", "--config=app.yml", "--profile=writer", "--profile-namespace=support", "--format=json") + if outcome.Status != 0 || calls.Load() != 1 || !strings.Contains(output, "text_inference") { + t.Fatalf("controlled smoke: %+v %q %q calls=%d", outcome, output, diagnostics, calls.Load()) + } + fail.Store(true) + outcome, output, diagnostics = executeNative(t, "promptkit", "test", "text_inference", "--config=app.yml", "--profile=support/writer", "--format=json") + var findings struct { + Completed bool `json:"completed"` + Failure string `json:"failure"` + } + if err := json.Unmarshal([]byte(output), &findings); err != nil { + t.Fatal(err) + } + if outcome.Status != 0 || !findings.Completed || findings.Failure == "" || strings.Contains(output+diagnostics, "provider-private-canary") { + t.Fatalf("native unsuccessful finding changed: %+v %q %q", outcome, output, diagnostics) + } + writeNativeFile(t, "invalid-app.yml", "server:\n max_request_bytes: -1\n") + outcome, output, diagnostics = executeNative(t, "promptkit", "requirements", "show", "support/summary", "--config=invalid-app.yml", "--quiet") + if outcome.Kind != commandkit.OutcomeUsage || output != "" || !strings.Contains(diagnostics, "Application configuration") { + t.Fatalf("acquisition evidence: %+v %q %q", outcome, output, diagnostics) + } +}