Migrate PromptKit integration to shared inspection
This commit is contained in:
@@ -183,7 +183,18 @@ available evidence survives adaptation. Exported GoDoc describes this handoff.
|
||||
|
||||
## Stage 4: Migrate ckprompt adapters, fixtures, and boundary coverage
|
||||
|
||||
Status: pending.
|
||||
Status: complete.
|
||||
|
||||
Evidence: Migrated discovered settings fragments to `/promptkit`, exposed native
|
||||
inspection paths, and added shared-load, requirements, schema, prepared-hash,
|
||||
and request-replacement coverage. `GOTOOLCHAIN=local go test
|
||||
./integrations/ckprompt` passed in the aligned workspace with Go 1.27.4. The
|
||||
missing-main-file constructor retains PromptKit's invalid-config identity and
|
||||
produces an operational outcome with native load diagnostics. ConfigKit's
|
||||
`LoadError` explicitly does not unwrap underlying filesystem errors, so this
|
||||
native error cannot also match `os.ErrNotExist` as originally anticipated;
|
||||
the separate joined-error case verifies forwarding of that identity when supplied.
|
||||
The composed example migration remains in Stage 5.
|
||||
|
||||
Primary files: [adapters.go](../../integrations/ckprompt/adapters.go), the
|
||||
integration's existing configuration/comparison/preparation/requirements tests,
|
||||
|
||||
@@ -295,6 +295,7 @@ func ConfigScope[I, S any](load func(context.Context, ckconfig.Selection, I, S)
|
||||
Effective: *inspection,
|
||||
Sources: *inspection,
|
||||
Explanation: *inspection,
|
||||
Paths: *inspection,
|
||||
Metadata: &metadata,
|
||||
Findings: &ckconfig.Findings{Records: inspection.Diagnostics(), Messages: messages.diagnostics},
|
||||
},
|
||||
|
||||
@@ -98,6 +98,9 @@ func adapterEngineFrom(t *testing.T, label, configuration string, options ...pro
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for index := range inputs {
|
||||
inputs[index].Destination = configkit.Keys("promptkit")
|
||||
}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: inputs}},
|
||||
}}, append([]promptkit.Option{promptkit.WithBundledOpenRouter(false)}, options...)...)
|
||||
|
||||
@@ -247,6 +247,9 @@ profiles:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for index := range inputs {
|
||||
inputs[index].Destination = configkit.Keys("promptkit")
|
||||
}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: inputs}}}}, promptkit.WithBundledOpenRouter(false))
|
||||
if err != nil {
|
||||
t.Fatalf("construct comparison engine: %v", err)
|
||||
|
||||
@@ -163,6 +163,9 @@ profiles:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for index := range inputs {
|
||||
inputs[index].Destination = configkit.Keys("promptkit")
|
||||
}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: inputs}}}}, promptkit.WithBundledOpenRouter(false))
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
|
||||
@@ -81,8 +81,8 @@ func TestPromptConstructorFailurePrecedence(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, invalid := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{Source: configkit.File(malformed)}}}, promptkit.WithBundledOpenRouter(false))
|
||||
_, missing := promptkit.NewEngine(promptkit.Config{}, promptkit.WithBundledOpenRouter(false), promptkit.WithSchemaFile(filepath.Join(dir, "missing.json")))
|
||||
if invalid == nil || missing == nil || !errors.Is(invalid, promptkit.ErrInvalidConfig) || !errors.Is(missing, promptkit.ErrInvalidConfig) || !errors.Is(missing, os.ErrNotExist) {
|
||||
_, missing := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{Source: configkit.File(filepath.Join(dir, "missing.yml"))}}}, promptkit.WithBundledOpenRouter(false))
|
||||
if invalid == nil || missing == nil || !errors.Is(invalid, promptkit.ErrInvalidConfig) || !errors.Is(missing, promptkit.ErrInvalidConfig) {
|
||||
t.Fatalf("constructor fixtures invalid=%v missing=%v", invalid, missing)
|
||||
}
|
||||
operational := errors.New("independent failure")
|
||||
@@ -91,11 +91,13 @@ func TestPromptConstructorFailurePrecedence(t *testing.T) {
|
||||
cause error
|
||||
extra error
|
||||
canceled bool
|
||||
kind commandkit.OutcomeKind
|
||||
}{
|
||||
{"missing schema", missing, os.ErrNotExist, false},
|
||||
{"joined operational", errors.Join(invalid, operational), operational, false},
|
||||
{"joined cancellation", errors.Join(invalid, context.Canceled), context.Canceled, false},
|
||||
{"already canceled", nil, context.Canceled, true},
|
||||
{"missing configuration", missing, promptkit.ErrInvalidConfig, false, commandkit.OutcomeOperational},
|
||||
{"joined missing error", errors.Join(missing, os.ErrNotExist), os.ErrNotExist, false, commandkit.OutcomeOperational},
|
||||
{"joined operational", errors.Join(invalid, operational), operational, false, commandkit.OutcomeOperational},
|
||||
{"joined cancellation", errors.Join(invalid, context.Canceled), context.Canceled, false, commandkit.OutcomeOperational},
|
||||
{"already canceled", nil, context.Canceled, true, commandkit.OutcomeOperational},
|
||||
} {
|
||||
for _, args := range [][]string{{"promptkit", "list", "prompts"}, {"config", "show", "--scope=promptkit"}} {
|
||||
t.Run(tc.name+"/"+strings.Join(args, "_"), func(t *testing.T) {
|
||||
@@ -115,7 +117,7 @@ func TestPromptConstructorFailurePrecedence(t *testing.T) {
|
||||
}
|
||||
var out bytes.Buffer
|
||||
result := app.Execute(ctx, args, commandkit.Streams{Out: &out})
|
||||
if result.Kind != commandkit.OutcomeOperational || result.Status != 1 || !errors.Is(result.Cause, tc.extra) {
|
||||
if result.Kind != tc.kind || !errors.Is(result.Cause, tc.extra) {
|
||||
t.Fatalf("outcome=%#v output=%q", result, out.String())
|
||||
}
|
||||
if tc.canceled {
|
||||
|
||||
109
integrations/ckprompt/schema_reporting_test.go
Normal file
109
integrations/ckprompt/schema_reporting_test.go
Normal file
@@ -0,0 +1,109 @@
|
||||
package ckprompt
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/ckconfig"
|
||||
commandkit "go.maximumdirect.net/commandkit"
|
||||
promptkit "go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestNativeSchemaAndPreparedEvidenceReports(t *testing.T) {
|
||||
const fixture = `profiles:
|
||||
- id: default
|
||||
endpoint: https://example.invalid/v1
|
||||
model: example
|
||||
prompts:
|
||||
- id: inline
|
||||
default_profile: default
|
||||
messages: [{role: user, content: hello}]
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema: {type: object}
|
||||
repair_attempts: 0
|
||||
- id: referenced
|
||||
default_profile: default
|
||||
messages: [{role: user, content: hello}]
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_file: schema.json
|
||||
repair_attempts: 0
|
||||
`
|
||||
engine := adapterEngineFrom(t, "schemas", fixture)
|
||||
loader := func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) {
|
||||
return engine, nil
|
||||
}
|
||||
suite := WithRequestPreparation(FromEngine(loader), loader, nil)
|
||||
app := newPromptTestApp(t, Command(suite))
|
||||
for _, id := range []string{"inline", "referenced"} {
|
||||
native, err := engine.InspectPrompt(t.Context(), id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if native.OutputContract.RepairAttempts != 0 || (id == "inline" && native.OutputContract.Schema == nil) || (id == "referenced" && native.OutputContract.SchemaFile == nil) {
|
||||
t.Fatalf("native output = %#v", native.OutputContract)
|
||||
}
|
||||
var output bytes.Buffer
|
||||
result := app.Execute(t.Context(), []string{"promptkit", "show", "prompt", id, "--format=json"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
||||
var actual any
|
||||
decoder := json.NewDecoder(&output)
|
||||
decoder.UseNumber()
|
||||
err = decoder.Decode(&actual)
|
||||
if result.Kind != commandkit.OutcomeSuccess || err != nil || !reflect.DeepEqual(actual, jsonValue(t, native)) {
|
||||
t.Fatalf("report = %#v, %v, %#v", result, err, actual)
|
||||
}
|
||||
}
|
||||
native, err := engine.Prepare(t.Context(), promptkit.RunRequest{PromptID: "inline"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if native.MessageContentHash == "" || native.OutputContract.RepairAttempts != 0 || native.OutputContract.Schema == nil {
|
||||
t.Fatalf("native prepared = %#v", native)
|
||||
}
|
||||
var output bytes.Buffer
|
||||
outcome := app.Execute(t.Context(), []string{"promptkit", "show", "prepared", "inline", "--format=json"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
||||
var actual any
|
||||
decoder := json.NewDecoder(&output)
|
||||
decoder.UseNumber()
|
||||
err = decoder.Decode(&actual)
|
||||
if outcome.Kind != commandkit.OutcomeSuccess || err != nil {
|
||||
t.Fatalf("prepared report = %#v, %v", outcome, err)
|
||||
}
|
||||
gotReport := actual.(map[string]any)
|
||||
expected := jsonValue(t, native).(map[string]any)
|
||||
for _, key := range []string{"message_content_hash", "output_contract", "structured_output"} {
|
||||
if !reflect.DeepEqual(gotReport[key], expected[key]) {
|
||||
t.Errorf("%s = %#v, want %#v", key, gotReport[key], expected[key])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparationBuilderForwardsReplacementRepairBudget(t *testing.T) {
|
||||
engine := adapterEngine(t)
|
||||
loader := func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) {
|
||||
return engine, nil
|
||||
}
|
||||
zero := 0
|
||||
for _, budget := range []*int{nil, &zero} {
|
||||
replacement := &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: budget}
|
||||
request := promptkit.RunRequest{PromptID: "empty", Validation: replacement}
|
||||
want, err := engine.Prepare(t.Context(), request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
suite := WithRequestPreparation(Suite[struct{}, struct{}]{}, loader, func(context.Context, PreparationRequest, struct{}, struct{}) (promptkit.RunRequest, error) {
|
||||
return request, nil
|
||||
})
|
||||
got, err := suite.PrepareRequest(t.Context(), PreparationRequest{}, struct{}{}, struct{}{})
|
||||
if err != nil || !reflect.DeepEqual(got.Value.OutputContract, want.OutputContract) || got.Value.MessageContentHash != want.MessageContentHash || (budget == nil && got.Value.OutputContract.RepairAttempts != 1) || (budget != nil && got.Value.OutputContract.RepairAttempts != 0) {
|
||||
t.Fatalf("replacement = %#v, %v; native = %#v", got.Value, err, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
118
integrations/ckprompt/shared_load_test.go
Normal file
118
integrations/ckprompt/shared_load_test.go
Normal file
@@ -0,0 +1,118 @@
|
||||
package ckprompt
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/ckconfig"
|
||||
commandkit "go.maximumdirect.net/commandkit"
|
||||
configkit "go.maximumdirect.net/configkit"
|
||||
promptkit "go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestConfigScopeUsesNativeSharedLoadPaths(t *testing.T) {
|
||||
type application struct {
|
||||
Name string `yaml:"app_name"`
|
||||
PromptKit promptkit.Settings `yaml:"promptkit"`
|
||||
}
|
||||
definition := promptkit.Configure(configkit.Definition[application]{Validate: func(_ *application, v *configkit.Validation) {
|
||||
v.Warning(configkit.Keys("app_name"), "application warning")
|
||||
}}, func(a *application) *promptkit.Settings { return &a.PromptKit })
|
||||
source := "app_name: example\npromptkit:\n " + strings.ReplaceAll(strings.TrimSpace(adapterPrompt), "\n", "\n ") + "\n"
|
||||
result, err := configkit.Load(configkit.Bytes("shared application", []byte(source)), configkit.Options[application]{Definition: definition})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
native, err := engine.InspectConfiguration()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
scope := ConfigScope(func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) {
|
||||
return engine, nil
|
||||
})
|
||||
app := newPromptTestApp(t, ckconfig.Command(ckconfig.Suite[struct{}, struct{}]{Scopes: []ckconfig.Scope[struct{}, struct{}]{scope}}))
|
||||
run := func(args ...string) ([]string, string, string) {
|
||||
t.Helper()
|
||||
var out, diag bytes.Buffer
|
||||
argv := append([]string{"config", "explain", "--scope=promptkit", "--format=json", "--verbose"}, args...)
|
||||
outcome := app.Execute(context.Background(), argv, commandkit.Streams{Out: &out, Err: &diag})
|
||||
if outcome.Kind != commandkit.OutcomeSuccess {
|
||||
t.Fatalf("%v: %#v %s", args, outcome, diag.String())
|
||||
}
|
||||
var report struct {
|
||||
Paths []struct {
|
||||
Path string `json:"path"`
|
||||
} `json:"paths"`
|
||||
}
|
||||
if args[0] == "--list" {
|
||||
if err := json.Unmarshal(out.Bytes(), &report); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
var paths []string
|
||||
for _, entry := range report.Paths {
|
||||
paths = append(paths, entry.Path)
|
||||
}
|
||||
return paths, out.String(), diag.String()
|
||||
}
|
||||
metadata := native.Metadata()
|
||||
expected := func(prefix string, explicit *bool) []string {
|
||||
var paths []string
|
||||
for _, path := range native.HistoricalPaths() {
|
||||
info, present := metadata.Lookup(path)
|
||||
if !present || (prefix != "" && !strings.HasPrefix(path.String(), prefix)) || (explicit != nil && info.Explicit != *explicit) {
|
||||
continue
|
||||
}
|
||||
paths = append(paths, path.String())
|
||||
}
|
||||
return paths
|
||||
}
|
||||
yes := true
|
||||
for _, tc := range []struct {
|
||||
args []string
|
||||
prefix string
|
||||
explicit *bool
|
||||
}{
|
||||
{[]string{"--list"}, "", nil}, {[]string{"--list", "--historical"}, "", nil}, {[]string{"--list", "--under=/promptkit/prompts"}, "/promptkit/prompts", nil}, {[]string{"--list", "--explicit=true"}, "", &yes},
|
||||
} {
|
||||
paths, _, diag := run(tc.args...)
|
||||
if !reflect.DeepEqual(paths, expected(tc.prefix, tc.explicit)) || !strings.Contains(diag, "application warning") {
|
||||
t.Fatalf("paths=%v want=%v diagnostics=%q", paths, expected(tc.prefix, tc.explicit), diag)
|
||||
}
|
||||
}
|
||||
_, output, diag := run("--path=/app_name")
|
||||
if !strings.Contains(output, "app_name") || !strings.Contains(output, "shared application") || !strings.Contains(diag, "application warning") {
|
||||
t.Fatalf("output=%q diagnostics=%q", output, diag)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequirementsReportsApplicationInputDefaults(t *testing.T) {
|
||||
type application struct {
|
||||
PromptKit promptkit.Settings `yaml:"promptkit"`
|
||||
}
|
||||
settings := promptkit.Settings{
|
||||
Profiles: []promptkit.ProfileSettings{{ID: "default", Endpoint: new("https://example.invalid/v1"), Model: new("example")}},
|
||||
Prompts: []promptkit.PromptSettings{{ID: "briefing", DefaultProfile: new("default"), Inputs: []promptkit.PromptInputSettings{{Name: "doc", Required: true, ContentType: new("text/plain"), Description: new("application default")}}, Messages: []promptkit.PromptMessageSettings{{Role: new("user"), Content: new("hello")}}, Output: &promptkit.PromptOutputSettings{Format: new("text"), ValidationMode: new("none")}}},
|
||||
}
|
||||
definition := promptkit.Configure(configkit.Definition[application]{Defaults: func() application { return application{PromptKit: settings} }}, func(value *application) *promptkit.Settings { return &value.PromptKit })
|
||||
result, err := configkit.Load(configkit.NoSource(), configkit.Options[application]{Definition: definition})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
native, _, report, _ := executeRequirementsAgainstNative(t, engine, promptkit.RequirementInspectionRequest{PromptID: "briefing"}, []string{"promptkit", "requirements", "show", "briefing"})
|
||||
if len(native.Inputs) != 1 || native.Inputs[0].ContentType == nil || *native.Inputs[0].ContentType != "text/plain" || native.Inputs[0].Description == nil || *native.Inputs[0].Description != "application default" || len(report) == 0 {
|
||||
t.Fatalf("native requirements = %#v, report = %#v", native, report)
|
||||
}
|
||||
}
|
||||
@@ -37,6 +37,9 @@ func validationTestEngine(t *testing.T, configuration string, options ...promptk
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for index := range inputs {
|
||||
inputs[index].Destination = configkit.Keys("promptkit")
|
||||
}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: inputs}},
|
||||
}}, append([]promptkit.Option{promptkit.WithBundledOpenRouter(false)}, options...)...)
|
||||
|
||||
Reference in New Issue
Block a user