Migrate PromptKit integration to shared inspection
This commit is contained in:
109
integrations/ckprompt/schema_reporting_test.go
Normal file
109
integrations/ckprompt/schema_reporting_test.go
Normal file
@@ -0,0 +1,109 @@
|
||||
package ckprompt
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/ckconfig"
|
||||
commandkit "go.maximumdirect.net/commandkit"
|
||||
promptkit "go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestNativeSchemaAndPreparedEvidenceReports(t *testing.T) {
|
||||
const fixture = `profiles:
|
||||
- id: default
|
||||
endpoint: https://example.invalid/v1
|
||||
model: example
|
||||
prompts:
|
||||
- id: inline
|
||||
default_profile: default
|
||||
messages: [{role: user, content: hello}]
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema: {type: object}
|
||||
repair_attempts: 0
|
||||
- id: referenced
|
||||
default_profile: default
|
||||
messages: [{role: user, content: hello}]
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_file: schema.json
|
||||
repair_attempts: 0
|
||||
`
|
||||
engine := adapterEngineFrom(t, "schemas", fixture)
|
||||
loader := func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) {
|
||||
return engine, nil
|
||||
}
|
||||
suite := WithRequestPreparation(FromEngine(loader), loader, nil)
|
||||
app := newPromptTestApp(t, Command(suite))
|
||||
for _, id := range []string{"inline", "referenced"} {
|
||||
native, err := engine.InspectPrompt(t.Context(), id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if native.OutputContract.RepairAttempts != 0 || (id == "inline" && native.OutputContract.Schema == nil) || (id == "referenced" && native.OutputContract.SchemaFile == nil) {
|
||||
t.Fatalf("native output = %#v", native.OutputContract)
|
||||
}
|
||||
var output bytes.Buffer
|
||||
result := app.Execute(t.Context(), []string{"promptkit", "show", "prompt", id, "--format=json"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
||||
var actual any
|
||||
decoder := json.NewDecoder(&output)
|
||||
decoder.UseNumber()
|
||||
err = decoder.Decode(&actual)
|
||||
if result.Kind != commandkit.OutcomeSuccess || err != nil || !reflect.DeepEqual(actual, jsonValue(t, native)) {
|
||||
t.Fatalf("report = %#v, %v, %#v", result, err, actual)
|
||||
}
|
||||
}
|
||||
native, err := engine.Prepare(t.Context(), promptkit.RunRequest{PromptID: "inline"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if native.MessageContentHash == "" || native.OutputContract.RepairAttempts != 0 || native.OutputContract.Schema == nil {
|
||||
t.Fatalf("native prepared = %#v", native)
|
||||
}
|
||||
var output bytes.Buffer
|
||||
outcome := app.Execute(t.Context(), []string{"promptkit", "show", "prepared", "inline", "--format=json"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
||||
var actual any
|
||||
decoder := json.NewDecoder(&output)
|
||||
decoder.UseNumber()
|
||||
err = decoder.Decode(&actual)
|
||||
if outcome.Kind != commandkit.OutcomeSuccess || err != nil {
|
||||
t.Fatalf("prepared report = %#v, %v", outcome, err)
|
||||
}
|
||||
gotReport := actual.(map[string]any)
|
||||
expected := jsonValue(t, native).(map[string]any)
|
||||
for _, key := range []string{"message_content_hash", "output_contract", "structured_output"} {
|
||||
if !reflect.DeepEqual(gotReport[key], expected[key]) {
|
||||
t.Errorf("%s = %#v, want %#v", key, gotReport[key], expected[key])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparationBuilderForwardsReplacementRepairBudget(t *testing.T) {
|
||||
engine := adapterEngine(t)
|
||||
loader := func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) {
|
||||
return engine, nil
|
||||
}
|
||||
zero := 0
|
||||
for _, budget := range []*int{nil, &zero} {
|
||||
replacement := &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: budget}
|
||||
request := promptkit.RunRequest{PromptID: "empty", Validation: replacement}
|
||||
want, err := engine.Prepare(t.Context(), request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
suite := WithRequestPreparation(Suite[struct{}, struct{}]{}, loader, func(context.Context, PreparationRequest, struct{}, struct{}) (promptkit.RunRequest, error) {
|
||||
return request, nil
|
||||
})
|
||||
got, err := suite.PrepareRequest(t.Context(), PreparationRequest{}, struct{}{}, struct{}{})
|
||||
if err != nil || !reflect.DeepEqual(got.Value.OutputContract, want.OutputContract) || got.Value.MessageContentHash != want.MessageContentHash || (budget == nil && got.Value.OutputContract.RepairAttempts != 1) || (budget != nil && got.Value.OutputContract.RepairAttempts != 0) {
|
||||
t.Fatalf("replacement = %#v, %v; native = %#v", got.Value, err, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user