From f2f807e87b346aed7b3ffa00c5b3fb0ae47838a2 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Fri, 20 Nov 2026 04:32:01 +0000 Subject: [PATCH] Expose native configuration scopes and independent comparisons --- docs/roadmap/implementation.md | 16 ++ internal/adapter/cli/acquisition.go | 59 +++++ internal/adapter/cli/app.go | 8 +- internal/adapter/cli/native_config.go | 41 ++++ internal/adapter/cli/native_config_test.go | 247 +++++++++++++++++++++ internal/adapter/cli/selection.go | 19 +- 6 files changed, 381 insertions(+), 9 deletions(-) create mode 100644 internal/adapter/cli/acquisition.go create mode 100644 internal/adapter/cli/native_config.go create mode 100644 internal/adapter/cli/native_config_test.go diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index a95355c..495ed52 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -327,6 +327,22 @@ composition matrix. ## Stage 3: Install Native Configuration Scopes And Independent Comparison +**Status: Complete.** Installed native `app` and `promptkit` scopes with all five +operations, path/metadata capabilities, and native-only selectors. Only `app` +accepts temporary overrides. The PromptKit acquisition decorator carries +application diagnostics separately and restores typed application failures before +classification; it does not place application evidence in the framework view, +rewrite effective values, or cache configuration/engines. + +Verified on Go 1.27.4 with focused real-library integration tests, race checks, +and vet: application-only inspection with unavailable framework sources, +normalized show/explain agreement, root/index/escaped-key paths, redaction, +independent explicit and implicit comparison selections, independent overlays and +native overrides, diff-check status 3, disabled framework/all-scope overrides, +absent custom configuration flags, and diagnostic/cause preservation for both +comparison sides. Full-package checks remain deferred while the unreachable +legacy CLI files/tests await Stage 8 removal. + **Primary files:** CLI configuration-suite composition and loaders; replace `internal/adapter/cli/config.go` and `framework_config.go` responsibilities. diff --git a/internal/adapter/cli/acquisition.go b/internal/adapter/cli/acquisition.go new file mode 100644 index 0000000..1a714cc --- /dev/null +++ b/internal/adapter/cli/acquisition.go @@ -0,0 +1,59 @@ +package cli + +import ( + "errors" + "fmt" + + "go.maximumdirect.net/ckconfig" + "go.maximumdirect.net/commandkit" + "go.maximumdirect.net/configkit" +) + +type applicationAcquisition struct { + sources []configkit.SourceInfo + records []configkit.Diagnostic + err error +} + +// acquisitionState holds only presentation evidence for this invocation, never +// configuration or engines. Each reached loader still performs a fresh load. +type acquisitionState struct{ applications []applicationAcquisition } + +// Native PromptKit adapters interpret a ConfigKit LoadError as framework +// evidence. Transport application acquisition failure separately so it cannot +// become a finding in the framework view. The callback decorator restores the +// original typed cause before the native command classifies the outcome. +var errApplicationAcquisition = errors.New("application configuration acquisition failed") + +func (state *acquisitionState) mark() int { + if state == nil { + return 0 + } + return len(state.applications) +} + +func attachApplicationEvidence[T any](acquired commandkit.Acquired[T], err error, state *acquisitionState, start int, comparison bool) (commandkit.Acquired[T], error) { + if state == nil { + return acquired, err + } + for index, application := range state.applications[start:] { + context := "Application configuration" + if comparison { + side := "left" + if index > 0 { + side = "right" + } + context += " (" + side + ")" + } + diagnostics, explanation := ckconfig.AdaptDiagnostics(context+":\n", application.sources, application.records) + acquired.Diagnostics = append(acquired.Diagnostics, diagnostics...) + if acquired.Explanation.Message == "" { + acquired.Explanation.Message = explanation.Message + } + acquired.Explanation.Details = append(acquired.Explanation.Details, explanation.Details...) + if application.err != nil && errors.Is(err, errApplicationAcquisition) { + err = fmt.Errorf("%s: %w", context, application.err) + } + } + return acquired, err +} diff --git a/internal/adapter/cli/app.go b/internal/adapter/cli/app.go index ca2a83c..5296ef1 100644 --- a/internal/adapter/cli/app.go +++ b/internal/adapter/cli/app.go @@ -20,19 +20,13 @@ func Execute(ctx context.Context, identity commandkit.Identity, args []string, s } func newApp(identity commandkit.Identity) (*commandkit.App, error) { - application := ckconfig.FromConfigKit("app", "Scriptorium application configuration.", loadApplication) - framework := ckprompt.ConfigScope(loadEngine) - configuration := ckconfig.Command(ckconfig.Suite[applicationInput, struct{}]{ - NewInput: func() applicationInput { return applicationInput{} }, - Scopes: []ckconfig.Scope[applicationInput, struct{}]{application, framework}, - }) definitions := ckprompt.FromEngine(loadEngine) declarations := []commandkit.Declaration{ commandkit.NewCommand(commandkit.CommandSpec[struct{}]{Name: "run", Summary: "Run a stored prompt.", Run: commandNotImplemented}), commandkit.NewCommand(commandkit.CommandSpec[serveInput]{Name: "serve", Summary: "Serve the HTTP API.", Flags: serveFlags(), Run: func(commandkit.Invocation, serveInput) commandkit.Outcome { return commandkit.Failure("Server startup is not implemented.") }}), - configuration, + configurationCommand(), commandkit.Completion("scriptorium"), } declarations = append(declarations, ckprompt.Declarations(definitions)...) diff --git a/internal/adapter/cli/native_config.go b/internal/adapter/cli/native_config.go new file mode 100644 index 0000000..b504492 --- /dev/null +++ b/internal/adapter/cli/native_config.go @@ -0,0 +1,41 @@ +package cli + +import ( + "context" + + appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config" + "go.maximumdirect.net/ckconfig" + "go.maximumdirect.net/ckprompt" + "go.maximumdirect.net/commandkit" + "go.maximumdirect.net/configkit" +) + +func configurationCommand() commandkit.Command { + application := ckconfig.FromConfigKit("app", "Scriptorium application configuration.", + func(ctx context.Context, selection ckconfig.Selection, input applicationInput, _ *acquisitionState) (*configkit.Result[appconfig.Document], error) { + return loadApplication(ctx, selection, input, struct{}{}) + }) + application.Features.Overrides = true + framework := frameworkConfigurationScope() + return ckconfig.Command(ckconfig.Suite[applicationInput, *acquisitionState]{ + NewInput: func() applicationInput { return applicationInput{} }, + NewState: func() *acquisitionState { return &acquisitionState{} }, + Scopes: []ckconfig.Scope[applicationInput, *acquisitionState]{application, framework}, + }) +} + +func frameworkConfigurationScope() ckconfig.Scope[applicationInput, *acquisitionState] { + scope := ckprompt.ConfigScope(loadEngine) + acquire, compare := scope.Acquire, scope.Compare + scope.Acquire = func(ctx context.Context, request ckconfig.Request, input applicationInput, state *acquisitionState) (commandkit.Acquired[ckconfig.View], error) { + start := state.mark() + acquired, err := acquire(ctx, request, input, state) + return attachApplicationEvidence(acquired, err, state, start, false) + } + scope.Compare = func(ctx context.Context, request ckconfig.ComparisonRequest[applicationInput], state *acquisitionState) (commandkit.Acquired[[]configkit.Difference], error) { + start := state.mark() + acquired, err := compare(ctx, request, state) + return attachApplicationEvidence(acquired, err, state, start, true) + } + return scope +} diff --git a/internal/adapter/cli/native_config_test.go b/internal/adapter/cli/native_config_test.go new file mode 100644 index 0000000..fec96f5 --- /dev/null +++ b/internal/adapter/cli/native_config_test.go @@ -0,0 +1,247 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "go.maximumdirect.net/ckconfig" + "go.maximumdirect.net/commandkit" + "go.maximumdirect.net/configkit" +) + +func executeNative(t *testing.T, args ...string) (commandkit.Outcome, string, string) { + t.Helper() + var output, diagnostics bytes.Buffer + outcome := Execute(context.Background(), commandkit.Identity{Name: "scriptorium", Version: "test"}, args, &output, &diagnostics) + return outcome, output.String(), diagnostics.String() +} + +func writeNativeFile(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0600); err != nil { + t.Fatal(err) + } +} + +func nativeConfigurationFixture(t *testing.T) { + t.Helper() + t.Chdir(t.TempDir()) + writeNativeFile(t, "left.yml", "promptkit:\n configuration: left-framework.yml\n schema_dir: ' ./a/../schemas '\nserver:\n addr: :1111\n") + writeNativeFile(t, "right.yml", "promptkit:\n configuration: right-framework.yml\nserver:\n addr: :2222\n") + writeNativeFile(t, "scriptorium.yml", "promptkit:\n configuration: right-framework.yml\nserver:\n addr: :3333\n") + writeNativeFile(t, "overlays/local.yml", "promptkit:\n configuration: right-framework.yml\nserver:\n addr: :4444\n") + for _, side := range []string{"left", "right"} { + writeNativeFile(t, side+"-framework.yml", "collections:\n custom:\n release_version: '1'\n profiles:\n - id: selected\n model: "+side+"-model\n extra_params:\n 'key/with~mark': visible\n password: "+side+"-password-canary\n") + } +} + +func TestNativeApplicationScopeDoesNotLoadFramework(t *testing.T) { + nativeConfigurationFixture(t) + writeNativeFile(t, "unavailable.yml", "promptkit:\n configuration: missing.yml\n") + for _, operation := range []string{"validate", "show", "sources", "explain"} { + args := []string{"config", operation, "--scope=app", "--config=unavailable.yml", "--format=json"} + if operation == "explain" { + args = append(args, "--path=/promptkit/configuration") + } + outcome, output, diagnostics := executeNative(t, args...) + if outcome.Status != 0 || output == "" { + t.Fatalf("%v: %+v %q %q", args, outcome, output, diagnostics) + } + } + // The native default scope remains application configuration. + outcome, output, diagnostics := executeNative(t, "config", "show", "--config=left.yml", "--path=/promptkit/schema_dir", "--format=json") + if outcome.Status != 0 || strings.TrimSpace(output) != `"schemas"` { + t.Fatalf("show: %+v %q %q", outcome, output, diagnostics) + } + outcome, output, diagnostics = executeNative(t, "config", "explain", "--scope=app", "--config=left.yml", "--path=/promptkit/schema_dir", "--format=json") + if outcome.Status != 0 || !strings.Contains(output, "schemas") || !strings.Contains(output, "normalization") { + t.Fatalf("explain: %+v %q %q", outcome, output, diagnostics) + } + outcome, output, diagnostics = executeNative(t, "config", "show", "--scope=promptkit", "--config=unavailable.yml") + if outcome.Status != 1 || output != "" || diagnostics == "" { + t.Fatalf("framework load: %+v %q %q", outcome, output, diagnostics) + } +} + +func TestNativeConfigurationPathsAndRedaction(t *testing.T) { + nativeConfigurationFixture(t) + for _, test := range []struct{ operation, path, want string }{ + {"show", "", "left-model"}, + {"show", "/collections/custom/profiles/0/model", "left-model"}, + {"show", "/collections/custom/profiles/0/extra_params/key~1with~0mark", "visible"}, + {"explain", "/collections/custom/profiles/0/extra_params/key~1with~0mark", "visible"}, + {"explain", "/collections/custom/profiles/0/extra_params/password", "redacted"}, + } { + outcome, output, diagnostics := executeNative(t, "config", test.operation, "--scope=promptkit", "--config=left.yml", "--path="+test.path, "--format=json") + if outcome.Status != 0 || !strings.Contains(output, test.want) || strings.Contains(output+diagnostics, "left-password-canary") { + t.Fatalf("%+v: %+v %q %q", test, outcome, output, diagnostics) + } + var value any + if err := json.Unmarshal([]byte(output), &value); err != nil { + t.Fatal(err) + } + } + outcome, output, diagnostics := executeNative(t, "config", "explain", "--scope=promptkit", "--config=left.yml", "--list", "--under=/collections/custom/profiles", "--format=json") + if outcome.Status != 0 || !strings.Contains(output, "/collections/custom/profiles/0/model") { + t.Fatalf("paths: %+v %q %q", outcome, output, diagnostics) + } + outcome, output, diagnostics = executeNative(t, "config", "show", "--scope=promptkit", "--config=left.yml", "--overlay=local", "--path=/collections/custom/profiles/0/model", "--format=json") + if outcome.Status != 0 || !strings.Contains(output, "right-model") { + t.Fatalf("overlay selection: %+v %q %q", outcome, output, diagnostics) + } +} + +func TestNativeConfigurationComparisonSelectionsAreIndependent(t *testing.T) { + nativeConfigurationFixture(t) + for _, test := range []struct { + scope string + right []string + before, after string + }{ + {"app", []string{"--against-config=right.yml"}, ":1111", ":2222"}, + {"app", nil, ":1111", ":3333"}, + {"app", []string{"--against-config=left.yml", "--against-overlay=local"}, ":1111", ":4444"}, + {"app", []string{"--against-config=right.yml", `--set=/server/addr="native-left"`, `--against-set=/server/addr="native-right"`}, "native-left", "native-right"}, + {"promptkit", []string{"--against-config=right.yml"}, "left-model", "right-model"}, + {"promptkit", nil, "left-model", "right-model"}, + {"promptkit", []string{"--against-config=left.yml", "--against-overlay=local"}, "left-model", "right-model"}, + } { + args := []string{"config", "diff", "--scope=" + test.scope, "--config=left.yml", "--details", "--check", "--format=json"} + args = append(args, test.right...) + outcome, output, diagnostics := executeNative(t, args...) + if outcome.Status != 3 || outcome.Kind != commandkit.OutcomeDomain || !strings.Contains(output, test.before) || !strings.Contains(output, test.after) { + t.Fatalf("%v: %+v %q %q", args, outcome, output, diagnostics) + } + if strings.Contains(output+diagnostics, "password-canary") { + t.Fatal("comparison disclosed classified value") + } + } + for _, scope := range []string{"app", "promptkit"} { + outcome, _, diagnostics := executeNative(t, "config", "diff", "--scope="+scope, "--config=left.yml", "--against-config=left.yml", "--overlay=local", "--check") + if outcome.Status != 3 { + t.Fatalf("left overlay inherited by right %s: %+v %q", scope, outcome, diagnostics) + } + outcome, _, diagnostics = executeNative(t, "config", "diff", "--scope="+scope, "--config=left.yml", "--against-config=left.yml", "--check") + if outcome.Status != 0 { + t.Fatalf("equal selections %s: %+v %q", scope, outcome, diagnostics) + } + } +} + +func TestNativeConfigurationOverridesAndCapabilityChecks(t *testing.T) { + nativeConfigurationFixture(t) + outcome, output, diagnostics := executeNative(t, "config", "show", "--scope=app", "--config=left.yml", `--set=/server/addr="native"`, "--path=/server/addr", "--format=json") + if outcome.Status != 0 || strings.TrimSpace(output) != `"native"` { + t.Fatalf("override: %+v %q %q", outcome, output, diagnostics) + } + for _, args := range [][]string{ + {"config", "show", "--scope=promptkit", `--set=/server/addr="native"`}, + {"config", "show", "--scope=all", `--set=/server/addr="native"`}, + {"config", "diff", "--scope=promptkit", `--against-set=/server/addr="native"`}, + {"config", "show", "--addr=:9999"}, {"config", "show", "--max-request-bytes=0"}, + {"config", "diff", "--against-addr=:9999"}, {"config", "diff", "--against-schema-dir=schemas"}, + } { + outcome, _, diagnostics := executeNative(t, args...) + if outcome.Status != 2 || outcome.Kind != commandkit.OutcomeUsage || diagnostics == "" { + t.Fatalf("%v: %+v %q", args, outcome, diagnostics) + } + } +} + +func TestApplicationFailureEvidenceIsNotFrameworkEvidence(t *testing.T) { + nativeConfigurationFixture(t) + writeNativeFile(t, "invalid.yml", "server:\n max_request_bytes: -1\n") + scope := frameworkConfigurationScope() + state := &acquisitionState{} + selection := ckconfig.Selection{Config: commandkit.Supplied[string]{Set: true, Value: "invalid.yml"}} + acquired, err := scope.Acquire(context.Background(), ckconfig.Request{Operation: ckconfig.Validate, Selection: selection}, applicationInput{}, state) + var native *configkit.LoadError + if !errors.As(err, &native) || errors.Is(err, errApplicationAcquisition) || ckconfig.ClassifyConfigurationError(err) != commandkit.OutcomeUsage { + t.Fatalf("typed application error lost: %v", err) + } + if acquired.Value.Findings != nil || acquired.Value.Sources != nil || acquired.Value.Effective != nil { + t.Fatalf("application evidence inserted into framework view: %+v", acquired.Value) + } + var details bytes.Buffer + for _, detail := range acquired.Explanation.Details { + if err := detail(&details); err != nil { + t.Fatal(err) + } + } + if !strings.Contains(details.String(), "Application configuration") || !strings.Contains(details.String(), "validate.error") || strings.Contains(details.String(), `scope "promptkit"`) { + t.Fatal(details.String()) + } + outcome, output, diagnostics := executeNative(t, "config", "validate", "--scope=promptkit", "--config=invalid.yml", "--quiet") + if outcome.Status != 2 || output != "" || !strings.Contains(diagnostics, "Application configuration") || !strings.Contains(diagnostics, "validate.error") { + t.Fatalf("terminal evidence: %+v %q %q", outcome, output, diagnostics) + } + for _, side := range []string{"left", "right"} { + args := []string{"config", "diff", "--scope=promptkit", "--config=left.yml", "--against-config=right.yml"} + if side == "left" { + args[3] = "--config=invalid.yml" + } else { + args[4] = "--against-config=invalid.yml" + } + outcome, _, diagnostics := executeNative(t, args...) + if outcome.Status != 2 || !errors.As(outcome.Cause, &native) || !strings.Contains(diagnostics, "Application configuration ("+side+")") { + t.Fatalf("%s evidence: %+v %q", side, outcome, diagnostics) + } + } + // In contrast, a framework load failure keeps its native findings and sources. + writeNativeFile(t, "bad-framework.yml", "profiles: [\n") + writeNativeFile(t, "framework-app.yml", "promptkit:\n configuration: bad-framework.yml\n") + selection.Config.Value = "framework-app.yml" + acquired, err = scope.Acquire(context.Background(), ckconfig.Request{Operation: ckconfig.Validate, Selection: selection}, applicationInput{}, &acquisitionState{}) + if err == nil || acquired.Value.Findings == nil || acquired.Value.Sources == nil || len(acquired.Value.Findings.Records) == 0 { + t.Fatalf("native framework evidence lost: %+v %v", acquired, err) + } +} + +func TestAcquisitionDiagnosticsRetainBothComparisonSides(t *testing.T) { + state := &acquisitionState{} + for _, label := range []string{"left", "right"} { + result, err := configkit.Load(configkit.Mapping(label, map[string]any{"value": label}), configkit.Options[map[string]any]{ + Definition: configkit.Definition[map[string]any]{Validate: func(_ *map[string]any, validation *configkit.Validation) { + validation.Warning(configkit.Keys("value"), "application finding") + }}, + }) + if err != nil { + t.Fatal(err) + } + state.applications = append(state.applications, applicationAcquisition{sources: result.Sources(), records: result.Diagnostics()}) + } + native := commandkit.Acquired[int]{Value: 42, Diagnostics: []commandkit.Diagnostic{{Severity: commandkit.SeverityWarning, Write: func(writer io.Writer) error { + _, err := io.WriteString(writer, "framework finding\n") + return err + }}}} + acquired, err := attachApplicationEvidence(native, nil, state, 0, true) + if err != nil || acquired.Value != native.Value { + t.Fatalf("native value changed: %+v %v", acquired, err) + } + var output bytes.Buffer + for _, diagnostic := range acquired.Diagnostics { + if err := diagnostic.Write(&output); err != nil { + t.Fatal(err) + } + } + for _, fragment := range []string{"framework finding", "Application configuration (left)", "Application configuration (right)", "validate.warning"} { + if !strings.Contains(output.String(), fragment) { + t.Fatalf("lost %q: %s", fragment, output.String()) + } + } + // A later callback sharing invocation state must not replay earlier evidence. + acquired, err = attachApplicationEvidence(native, nil, state, state.mark(), false) + if err != nil || len(acquired.Diagnostics) != len(native.Diagnostics) { + t.Fatalf("evidence replayed: %+v %v", acquired, err) + } +} diff --git a/internal/adapter/cli/selection.go b/internal/adapter/cli/selection.go index 92a8ffe..95a0fd2 100644 --- a/internal/adapter/cli/selection.go +++ b/internal/adapter/cli/selection.go @@ -2,6 +2,7 @@ package cli import ( "context" + "errors" "strconv" appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config" @@ -87,8 +88,22 @@ func loadApplication(ctx context.Context, selection ckconfig.Selection, input ap return result, nil } -func loadEngine(ctx context.Context, selection ckconfig.Selection, input applicationInput, state struct{}) (*promptkit.Engine, error) { - result, err := loadApplication(ctx, selection, input, state) +func loadEngine(ctx context.Context, selection ckconfig.Selection, input applicationInput, state *acquisitionState) (*promptkit.Engine, error) { + result, err := loadApplication(ctx, selection, input, struct{}{}) + if state != nil { + evidence := applicationAcquisition{err: err} + if result != nil { + evidence.sources, evidence.records = result.Sources(), result.Diagnostics() + } + var native *configkit.LoadError + if errors.As(err, &native) && native != nil { + evidence.sources, evidence.records = native.Sources(), native.Diagnostics() + } + state.applications = append(state.applications, evidence) + if err != nil { + return nil, errApplicationAcquisition + } + } if err != nil { return nil, err }