Expose native configuration scopes and independent comparisons
This commit is contained in:
59
internal/adapter/cli/acquisition.go
Normal file
59
internal/adapter/cli/acquisition.go
Normal file
@@ -0,0 +1,59 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"go.maximumdirect.net/ckconfig"
|
||||
"go.maximumdirect.net/commandkit"
|
||||
"go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
type applicationAcquisition struct {
|
||||
sources []configkit.SourceInfo
|
||||
records []configkit.Diagnostic
|
||||
err error
|
||||
}
|
||||
|
||||
// acquisitionState holds only presentation evidence for this invocation, never
|
||||
// configuration or engines. Each reached loader still performs a fresh load.
|
||||
type acquisitionState struct{ applications []applicationAcquisition }
|
||||
|
||||
// Native PromptKit adapters interpret a ConfigKit LoadError as framework
|
||||
// evidence. Transport application acquisition failure separately so it cannot
|
||||
// become a finding in the framework view. The callback decorator restores the
|
||||
// original typed cause before the native command classifies the outcome.
|
||||
var errApplicationAcquisition = errors.New("application configuration acquisition failed")
|
||||
|
||||
func (state *acquisitionState) mark() int {
|
||||
if state == nil {
|
||||
return 0
|
||||
}
|
||||
return len(state.applications)
|
||||
}
|
||||
|
||||
func attachApplicationEvidence[T any](acquired commandkit.Acquired[T], err error, state *acquisitionState, start int, comparison bool) (commandkit.Acquired[T], error) {
|
||||
if state == nil {
|
||||
return acquired, err
|
||||
}
|
||||
for index, application := range state.applications[start:] {
|
||||
context := "Application configuration"
|
||||
if comparison {
|
||||
side := "left"
|
||||
if index > 0 {
|
||||
side = "right"
|
||||
}
|
||||
context += " (" + side + ")"
|
||||
}
|
||||
diagnostics, explanation := ckconfig.AdaptDiagnostics(context+":\n", application.sources, application.records)
|
||||
acquired.Diagnostics = append(acquired.Diagnostics, diagnostics...)
|
||||
if acquired.Explanation.Message == "" {
|
||||
acquired.Explanation.Message = explanation.Message
|
||||
}
|
||||
acquired.Explanation.Details = append(acquired.Explanation.Details, explanation.Details...)
|
||||
if application.err != nil && errors.Is(err, errApplicationAcquisition) {
|
||||
err = fmt.Errorf("%s: %w", context, application.err)
|
||||
}
|
||||
}
|
||||
return acquired, err
|
||||
}
|
||||
@@ -20,19 +20,13 @@ func Execute(ctx context.Context, identity commandkit.Identity, args []string, s
|
||||
}
|
||||
|
||||
func newApp(identity commandkit.Identity) (*commandkit.App, error) {
|
||||
application := ckconfig.FromConfigKit("app", "Scriptorium application configuration.", loadApplication)
|
||||
framework := ckprompt.ConfigScope(loadEngine)
|
||||
configuration := ckconfig.Command(ckconfig.Suite[applicationInput, struct{}]{
|
||||
NewInput: func() applicationInput { return applicationInput{} },
|
||||
Scopes: []ckconfig.Scope[applicationInput, struct{}]{application, framework},
|
||||
})
|
||||
definitions := ckprompt.FromEngine(loadEngine)
|
||||
declarations := []commandkit.Declaration{
|
||||
commandkit.NewCommand(commandkit.CommandSpec[struct{}]{Name: "run", Summary: "Run a stored prompt.", Run: commandNotImplemented}),
|
||||
commandkit.NewCommand(commandkit.CommandSpec[serveInput]{Name: "serve", Summary: "Serve the HTTP API.", Flags: serveFlags(), Run: func(commandkit.Invocation, serveInput) commandkit.Outcome {
|
||||
return commandkit.Failure("Server startup is not implemented.")
|
||||
}}),
|
||||
configuration,
|
||||
configurationCommand(),
|
||||
commandkit.Completion("scriptorium"),
|
||||
}
|
||||
declarations = append(declarations, ckprompt.Declarations(definitions)...)
|
||||
|
||||
41
internal/adapter/cli/native_config.go
Normal file
41
internal/adapter/cli/native_config.go
Normal file
@@ -0,0 +1,41 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config"
|
||||
"go.maximumdirect.net/ckconfig"
|
||||
"go.maximumdirect.net/ckprompt"
|
||||
"go.maximumdirect.net/commandkit"
|
||||
"go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
func configurationCommand() commandkit.Command {
|
||||
application := ckconfig.FromConfigKit("app", "Scriptorium application configuration.",
|
||||
func(ctx context.Context, selection ckconfig.Selection, input applicationInput, _ *acquisitionState) (*configkit.Result[appconfig.Document], error) {
|
||||
return loadApplication(ctx, selection, input, struct{}{})
|
||||
})
|
||||
application.Features.Overrides = true
|
||||
framework := frameworkConfigurationScope()
|
||||
return ckconfig.Command(ckconfig.Suite[applicationInput, *acquisitionState]{
|
||||
NewInput: func() applicationInput { return applicationInput{} },
|
||||
NewState: func() *acquisitionState { return &acquisitionState{} },
|
||||
Scopes: []ckconfig.Scope[applicationInput, *acquisitionState]{application, framework},
|
||||
})
|
||||
}
|
||||
|
||||
func frameworkConfigurationScope() ckconfig.Scope[applicationInput, *acquisitionState] {
|
||||
scope := ckprompt.ConfigScope(loadEngine)
|
||||
acquire, compare := scope.Acquire, scope.Compare
|
||||
scope.Acquire = func(ctx context.Context, request ckconfig.Request, input applicationInput, state *acquisitionState) (commandkit.Acquired[ckconfig.View], error) {
|
||||
start := state.mark()
|
||||
acquired, err := acquire(ctx, request, input, state)
|
||||
return attachApplicationEvidence(acquired, err, state, start, false)
|
||||
}
|
||||
scope.Compare = func(ctx context.Context, request ckconfig.ComparisonRequest[applicationInput], state *acquisitionState) (commandkit.Acquired[[]configkit.Difference], error) {
|
||||
start := state.mark()
|
||||
acquired, err := compare(ctx, request, state)
|
||||
return attachApplicationEvidence(acquired, err, state, start, true)
|
||||
}
|
||||
return scope
|
||||
}
|
||||
247
internal/adapter/cli/native_config_test.go
Normal file
247
internal/adapter/cli/native_config_test.go
Normal file
@@ -0,0 +1,247 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/ckconfig"
|
||||
"go.maximumdirect.net/commandkit"
|
||||
"go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
func executeNative(t *testing.T, args ...string) (commandkit.Outcome, string, string) {
|
||||
t.Helper()
|
||||
var output, diagnostics bytes.Buffer
|
||||
outcome := Execute(context.Background(), commandkit.Identity{Name: "scriptorium", Version: "test"}, args, &output, &diagnostics)
|
||||
return outcome, output.String(), diagnostics.String()
|
||||
}
|
||||
|
||||
func writeNativeFile(t *testing.T, path, content string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(content), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func nativeConfigurationFixture(t *testing.T) {
|
||||
t.Helper()
|
||||
t.Chdir(t.TempDir())
|
||||
writeNativeFile(t, "left.yml", "promptkit:\n configuration: left-framework.yml\n schema_dir: ' ./a/../schemas '\nserver:\n addr: :1111\n")
|
||||
writeNativeFile(t, "right.yml", "promptkit:\n configuration: right-framework.yml\nserver:\n addr: :2222\n")
|
||||
writeNativeFile(t, "scriptorium.yml", "promptkit:\n configuration: right-framework.yml\nserver:\n addr: :3333\n")
|
||||
writeNativeFile(t, "overlays/local.yml", "promptkit:\n configuration: right-framework.yml\nserver:\n addr: :4444\n")
|
||||
for _, side := range []string{"left", "right"} {
|
||||
writeNativeFile(t, side+"-framework.yml", "collections:\n custom:\n release_version: '1'\n profiles:\n - id: selected\n model: "+side+"-model\n extra_params:\n 'key/with~mark': visible\n password: "+side+"-password-canary\n")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeApplicationScopeDoesNotLoadFramework(t *testing.T) {
|
||||
nativeConfigurationFixture(t)
|
||||
writeNativeFile(t, "unavailable.yml", "promptkit:\n configuration: missing.yml\n")
|
||||
for _, operation := range []string{"validate", "show", "sources", "explain"} {
|
||||
args := []string{"config", operation, "--scope=app", "--config=unavailable.yml", "--format=json"}
|
||||
if operation == "explain" {
|
||||
args = append(args, "--path=/promptkit/configuration")
|
||||
}
|
||||
outcome, output, diagnostics := executeNative(t, args...)
|
||||
if outcome.Status != 0 || output == "" {
|
||||
t.Fatalf("%v: %+v %q %q", args, outcome, output, diagnostics)
|
||||
}
|
||||
}
|
||||
// The native default scope remains application configuration.
|
||||
outcome, output, diagnostics := executeNative(t, "config", "show", "--config=left.yml", "--path=/promptkit/schema_dir", "--format=json")
|
||||
if outcome.Status != 0 || strings.TrimSpace(output) != `"schemas"` {
|
||||
t.Fatalf("show: %+v %q %q", outcome, output, diagnostics)
|
||||
}
|
||||
outcome, output, diagnostics = executeNative(t, "config", "explain", "--scope=app", "--config=left.yml", "--path=/promptkit/schema_dir", "--format=json")
|
||||
if outcome.Status != 0 || !strings.Contains(output, "schemas") || !strings.Contains(output, "normalization") {
|
||||
t.Fatalf("explain: %+v %q %q", outcome, output, diagnostics)
|
||||
}
|
||||
outcome, output, diagnostics = executeNative(t, "config", "show", "--scope=promptkit", "--config=unavailable.yml")
|
||||
if outcome.Status != 1 || output != "" || diagnostics == "" {
|
||||
t.Fatalf("framework load: %+v %q %q", outcome, output, diagnostics)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeConfigurationPathsAndRedaction(t *testing.T) {
|
||||
nativeConfigurationFixture(t)
|
||||
for _, test := range []struct{ operation, path, want string }{
|
||||
{"show", "", "left-model"},
|
||||
{"show", "/collections/custom/profiles/0/model", "left-model"},
|
||||
{"show", "/collections/custom/profiles/0/extra_params/key~1with~0mark", "visible"},
|
||||
{"explain", "/collections/custom/profiles/0/extra_params/key~1with~0mark", "visible"},
|
||||
{"explain", "/collections/custom/profiles/0/extra_params/password", "redacted"},
|
||||
} {
|
||||
outcome, output, diagnostics := executeNative(t, "config", test.operation, "--scope=promptkit", "--config=left.yml", "--path="+test.path, "--format=json")
|
||||
if outcome.Status != 0 || !strings.Contains(output, test.want) || strings.Contains(output+diagnostics, "left-password-canary") {
|
||||
t.Fatalf("%+v: %+v %q %q", test, outcome, output, diagnostics)
|
||||
}
|
||||
var value any
|
||||
if err := json.Unmarshal([]byte(output), &value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
outcome, output, diagnostics := executeNative(t, "config", "explain", "--scope=promptkit", "--config=left.yml", "--list", "--under=/collections/custom/profiles", "--format=json")
|
||||
if outcome.Status != 0 || !strings.Contains(output, "/collections/custom/profiles/0/model") {
|
||||
t.Fatalf("paths: %+v %q %q", outcome, output, diagnostics)
|
||||
}
|
||||
outcome, output, diagnostics = executeNative(t, "config", "show", "--scope=promptkit", "--config=left.yml", "--overlay=local", "--path=/collections/custom/profiles/0/model", "--format=json")
|
||||
if outcome.Status != 0 || !strings.Contains(output, "right-model") {
|
||||
t.Fatalf("overlay selection: %+v %q %q", outcome, output, diagnostics)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeConfigurationComparisonSelectionsAreIndependent(t *testing.T) {
|
||||
nativeConfigurationFixture(t)
|
||||
for _, test := range []struct {
|
||||
scope string
|
||||
right []string
|
||||
before, after string
|
||||
}{
|
||||
{"app", []string{"--against-config=right.yml"}, ":1111", ":2222"},
|
||||
{"app", nil, ":1111", ":3333"},
|
||||
{"app", []string{"--against-config=left.yml", "--against-overlay=local"}, ":1111", ":4444"},
|
||||
{"app", []string{"--against-config=right.yml", `--set=/server/addr="native-left"`, `--against-set=/server/addr="native-right"`}, "native-left", "native-right"},
|
||||
{"promptkit", []string{"--against-config=right.yml"}, "left-model", "right-model"},
|
||||
{"promptkit", nil, "left-model", "right-model"},
|
||||
{"promptkit", []string{"--against-config=left.yml", "--against-overlay=local"}, "left-model", "right-model"},
|
||||
} {
|
||||
args := []string{"config", "diff", "--scope=" + test.scope, "--config=left.yml", "--details", "--check", "--format=json"}
|
||||
args = append(args, test.right...)
|
||||
outcome, output, diagnostics := executeNative(t, args...)
|
||||
if outcome.Status != 3 || outcome.Kind != commandkit.OutcomeDomain || !strings.Contains(output, test.before) || !strings.Contains(output, test.after) {
|
||||
t.Fatalf("%v: %+v %q %q", args, outcome, output, diagnostics)
|
||||
}
|
||||
if strings.Contains(output+diagnostics, "password-canary") {
|
||||
t.Fatal("comparison disclosed classified value")
|
||||
}
|
||||
}
|
||||
for _, scope := range []string{"app", "promptkit"} {
|
||||
outcome, _, diagnostics := executeNative(t, "config", "diff", "--scope="+scope, "--config=left.yml", "--against-config=left.yml", "--overlay=local", "--check")
|
||||
if outcome.Status != 3 {
|
||||
t.Fatalf("left overlay inherited by right %s: %+v %q", scope, outcome, diagnostics)
|
||||
}
|
||||
outcome, _, diagnostics = executeNative(t, "config", "diff", "--scope="+scope, "--config=left.yml", "--against-config=left.yml", "--check")
|
||||
if outcome.Status != 0 {
|
||||
t.Fatalf("equal selections %s: %+v %q", scope, outcome, diagnostics)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeConfigurationOverridesAndCapabilityChecks(t *testing.T) {
|
||||
nativeConfigurationFixture(t)
|
||||
outcome, output, diagnostics := executeNative(t, "config", "show", "--scope=app", "--config=left.yml", `--set=/server/addr="native"`, "--path=/server/addr", "--format=json")
|
||||
if outcome.Status != 0 || strings.TrimSpace(output) != `"native"` {
|
||||
t.Fatalf("override: %+v %q %q", outcome, output, diagnostics)
|
||||
}
|
||||
for _, args := range [][]string{
|
||||
{"config", "show", "--scope=promptkit", `--set=/server/addr="native"`},
|
||||
{"config", "show", "--scope=all", `--set=/server/addr="native"`},
|
||||
{"config", "diff", "--scope=promptkit", `--against-set=/server/addr="native"`},
|
||||
{"config", "show", "--addr=:9999"}, {"config", "show", "--max-request-bytes=0"},
|
||||
{"config", "diff", "--against-addr=:9999"}, {"config", "diff", "--against-schema-dir=schemas"},
|
||||
} {
|
||||
outcome, _, diagnostics := executeNative(t, args...)
|
||||
if outcome.Status != 2 || outcome.Kind != commandkit.OutcomeUsage || diagnostics == "" {
|
||||
t.Fatalf("%v: %+v %q", args, outcome, diagnostics)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplicationFailureEvidenceIsNotFrameworkEvidence(t *testing.T) {
|
||||
nativeConfigurationFixture(t)
|
||||
writeNativeFile(t, "invalid.yml", "server:\n max_request_bytes: -1\n")
|
||||
scope := frameworkConfigurationScope()
|
||||
state := &acquisitionState{}
|
||||
selection := ckconfig.Selection{Config: commandkit.Supplied[string]{Set: true, Value: "invalid.yml"}}
|
||||
acquired, err := scope.Acquire(context.Background(), ckconfig.Request{Operation: ckconfig.Validate, Selection: selection}, applicationInput{}, state)
|
||||
var native *configkit.LoadError
|
||||
if !errors.As(err, &native) || errors.Is(err, errApplicationAcquisition) || ckconfig.ClassifyConfigurationError(err) != commandkit.OutcomeUsage {
|
||||
t.Fatalf("typed application error lost: %v", err)
|
||||
}
|
||||
if acquired.Value.Findings != nil || acquired.Value.Sources != nil || acquired.Value.Effective != nil {
|
||||
t.Fatalf("application evidence inserted into framework view: %+v", acquired.Value)
|
||||
}
|
||||
var details bytes.Buffer
|
||||
for _, detail := range acquired.Explanation.Details {
|
||||
if err := detail(&details); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(details.String(), "Application configuration") || !strings.Contains(details.String(), "validate.error") || strings.Contains(details.String(), `scope "promptkit"`) {
|
||||
t.Fatal(details.String())
|
||||
}
|
||||
outcome, output, diagnostics := executeNative(t, "config", "validate", "--scope=promptkit", "--config=invalid.yml", "--quiet")
|
||||
if outcome.Status != 2 || output != "" || !strings.Contains(diagnostics, "Application configuration") || !strings.Contains(diagnostics, "validate.error") {
|
||||
t.Fatalf("terminal evidence: %+v %q %q", outcome, output, diagnostics)
|
||||
}
|
||||
for _, side := range []string{"left", "right"} {
|
||||
args := []string{"config", "diff", "--scope=promptkit", "--config=left.yml", "--against-config=right.yml"}
|
||||
if side == "left" {
|
||||
args[3] = "--config=invalid.yml"
|
||||
} else {
|
||||
args[4] = "--against-config=invalid.yml"
|
||||
}
|
||||
outcome, _, diagnostics := executeNative(t, args...)
|
||||
if outcome.Status != 2 || !errors.As(outcome.Cause, &native) || !strings.Contains(diagnostics, "Application configuration ("+side+")") {
|
||||
t.Fatalf("%s evidence: %+v %q", side, outcome, diagnostics)
|
||||
}
|
||||
}
|
||||
// In contrast, a framework load failure keeps its native findings and sources.
|
||||
writeNativeFile(t, "bad-framework.yml", "profiles: [\n")
|
||||
writeNativeFile(t, "framework-app.yml", "promptkit:\n configuration: bad-framework.yml\n")
|
||||
selection.Config.Value = "framework-app.yml"
|
||||
acquired, err = scope.Acquire(context.Background(), ckconfig.Request{Operation: ckconfig.Validate, Selection: selection}, applicationInput{}, &acquisitionState{})
|
||||
if err == nil || acquired.Value.Findings == nil || acquired.Value.Sources == nil || len(acquired.Value.Findings.Records) == 0 {
|
||||
t.Fatalf("native framework evidence lost: %+v %v", acquired, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcquisitionDiagnosticsRetainBothComparisonSides(t *testing.T) {
|
||||
state := &acquisitionState{}
|
||||
for _, label := range []string{"left", "right"} {
|
||||
result, err := configkit.Load(configkit.Mapping(label, map[string]any{"value": label}), configkit.Options[map[string]any]{
|
||||
Definition: configkit.Definition[map[string]any]{Validate: func(_ *map[string]any, validation *configkit.Validation) {
|
||||
validation.Warning(configkit.Keys("value"), "application finding")
|
||||
}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
state.applications = append(state.applications, applicationAcquisition{sources: result.Sources(), records: result.Diagnostics()})
|
||||
}
|
||||
native := commandkit.Acquired[int]{Value: 42, Diagnostics: []commandkit.Diagnostic{{Severity: commandkit.SeverityWarning, Write: func(writer io.Writer) error {
|
||||
_, err := io.WriteString(writer, "framework finding\n")
|
||||
return err
|
||||
}}}}
|
||||
acquired, err := attachApplicationEvidence(native, nil, state, 0, true)
|
||||
if err != nil || acquired.Value != native.Value {
|
||||
t.Fatalf("native value changed: %+v %v", acquired, err)
|
||||
}
|
||||
var output bytes.Buffer
|
||||
for _, diagnostic := range acquired.Diagnostics {
|
||||
if err := diagnostic.Write(&output); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, fragment := range []string{"framework finding", "Application configuration (left)", "Application configuration (right)", "validate.warning"} {
|
||||
if !strings.Contains(output.String(), fragment) {
|
||||
t.Fatalf("lost %q: %s", fragment, output.String())
|
||||
}
|
||||
}
|
||||
// A later callback sharing invocation state must not replay earlier evidence.
|
||||
acquired, err = attachApplicationEvidence(native, nil, state, state.mark(), false)
|
||||
if err != nil || len(acquired.Diagnostics) != len(native.Diagnostics) {
|
||||
t.Fatalf("evidence replayed: %+v %v", acquired, err)
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strconv"
|
||||
|
||||
appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config"
|
||||
@@ -87,8 +88,22 @@ func loadApplication(ctx context.Context, selection ckconfig.Selection, input ap
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func loadEngine(ctx context.Context, selection ckconfig.Selection, input applicationInput, state struct{}) (*promptkit.Engine, error) {
|
||||
result, err := loadApplication(ctx, selection, input, state)
|
||||
func loadEngine(ctx context.Context, selection ckconfig.Selection, input applicationInput, state *acquisitionState) (*promptkit.Engine, error) {
|
||||
result, err := loadApplication(ctx, selection, input, struct{}{})
|
||||
if state != nil {
|
||||
evidence := applicationAcquisition{err: err}
|
||||
if result != nil {
|
||||
evidence.sources, evidence.records = result.Sources(), result.Diagnostics()
|
||||
}
|
||||
var native *configkit.LoadError
|
||||
if errors.As(err, &native) && native != nil {
|
||||
evidence.sources, evidence.records = native.Sources(), native.Diagnostics()
|
||||
}
|
||||
state.applications = append(state.applications, evidence)
|
||||
if err != nil {
|
||||
return nil, errApplicationAcquisition
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user