package cli import ( "bytes" "context" "encoding/json" "errors" "io" "os" "path/filepath" "strings" "testing" "go.maximumdirect.net/ckconfig" "go.maximumdirect.net/commandkit" "go.maximumdirect.net/configkit" ) func executeNative(t *testing.T, args ...string) (commandkit.Outcome, string, string) { t.Helper() var output, diagnostics bytes.Buffer outcome := Execute(context.Background(), commandkit.Identity{Name: "scriptorium", Version: "test"}, args, &output, &diagnostics) return outcome, output.String(), diagnostics.String() } func writeNativeFile(t *testing.T, path, content string) { t.Helper() if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { t.Fatal(err) } if err := os.WriteFile(path, []byte(content), 0600); err != nil { t.Fatal(err) } } func nativeConfigurationFixture(t *testing.T) { t.Helper() t.Chdir(t.TempDir()) writeNativeFile(t, "left.yml", "promptkit:\n configuration: left-framework.yml\n schema_dir: ' ./a/../schemas '\nserver:\n addr: :1111\n") writeNativeFile(t, "right.yml", "promptkit:\n configuration: right-framework.yml\nserver:\n addr: :2222\n") writeNativeFile(t, "scriptorium.yml", "promptkit:\n configuration: right-framework.yml\nserver:\n addr: :3333\n") writeNativeFile(t, "overlays/local.yml", "promptkit:\n configuration: right-framework.yml\nserver:\n addr: :4444\n") for _, side := range []string{"left", "right"} { writeNativeFile(t, side+"-framework.yml", "collections:\n custom:\n release_version: '1'\n profiles:\n - id: selected\n model: "+side+"-model\n extra_params:\n 'key/with~mark': visible\n password: "+side+"-password-canary\n") } } func TestNativeApplicationScopeDoesNotLoadFramework(t *testing.T) { nativeConfigurationFixture(t) writeNativeFile(t, "unavailable.yml", "promptkit:\n configuration: missing.yml\n") for _, operation := range []string{"validate", "show", "sources", "explain"} { args := []string{"config", operation, "--scope=app", "--config=unavailable.yml", "--format=json"} if operation == "explain" { args = append(args, "--path=/promptkit/configuration") } outcome, output, diagnostics := executeNative(t, args...) if outcome.Status != 0 || output == "" { t.Fatalf("%v: %+v %q %q", args, outcome, output, diagnostics) } } // The native default scope remains application configuration. outcome, output, diagnostics := executeNative(t, "config", "show", "--config=left.yml", "--path=/promptkit/schema_dir", "--format=json") if outcome.Status != 0 || strings.TrimSpace(output) != `"schemas"` { t.Fatalf("show: %+v %q %q", outcome, output, diagnostics) } outcome, output, diagnostics = executeNative(t, "config", "explain", "--scope=app", "--config=left.yml", "--path=/promptkit/schema_dir", "--format=json") if outcome.Status != 0 || !strings.Contains(output, "schemas") || !strings.Contains(output, "normalization") { t.Fatalf("explain: %+v %q %q", outcome, output, diagnostics) } outcome, output, diagnostics = executeNative(t, "config", "show", "--scope=promptkit", "--config=unavailable.yml") if outcome.Status != 1 || output != "" || diagnostics == "" { t.Fatalf("framework load: %+v %q %q", outcome, output, diagnostics) } } func TestNativeConfigurationPathsAndRedaction(t *testing.T) { nativeConfigurationFixture(t) for _, test := range []struct{ operation, path, want string }{ {"show", "", "left-model"}, {"show", "/collections/custom/profiles/0/model", "left-model"}, {"show", "/collections/custom/profiles/0/extra_params/key~1with~0mark", "visible"}, {"explain", "/collections/custom/profiles/0/extra_params/key~1with~0mark", "visible"}, {"explain", "/collections/custom/profiles/0/extra_params/password", "redacted"}, } { outcome, output, diagnostics := executeNative(t, "config", test.operation, "--scope=promptkit", "--config=left.yml", "--path="+test.path, "--format=json") if outcome.Status != 0 || !strings.Contains(output, test.want) || strings.Contains(output+diagnostics, "left-password-canary") { t.Fatalf("%+v: %+v %q %q", test, outcome, output, diagnostics) } var value any if err := json.Unmarshal([]byte(output), &value); err != nil { t.Fatal(err) } } outcome, output, diagnostics := executeNative(t, "config", "explain", "--scope=promptkit", "--config=left.yml", "--list", "--under=/collections/custom/profiles", "--format=json") if outcome.Status != 0 || !strings.Contains(output, "/collections/custom/profiles/0/model") { t.Fatalf("paths: %+v %q %q", outcome, output, diagnostics) } outcome, output, diagnostics = executeNative(t, "config", "show", "--scope=promptkit", "--config=left.yml", "--overlay=local", "--path=/collections/custom/profiles/0/model", "--format=json") if outcome.Status != 0 || !strings.Contains(output, "right-model") { t.Fatalf("overlay selection: %+v %q %q", outcome, output, diagnostics) } } func TestNativeConfigurationComparisonSelectionsAreIndependent(t *testing.T) { nativeConfigurationFixture(t) for _, test := range []struct { scope string right []string before, after string }{ {"app", []string{"--against-config=right.yml"}, ":1111", ":2222"}, {"app", nil, ":1111", ":3333"}, {"app", []string{"--against-config=left.yml", "--against-overlay=local"}, ":1111", ":4444"}, {"app", []string{"--against-config=right.yml", `--set=/server/addr="native-left"`, `--against-set=/server/addr="native-right"`}, "native-left", "native-right"}, {"promptkit", []string{"--against-config=right.yml"}, "left-model", "right-model"}, {"promptkit", nil, "left-model", "right-model"}, {"promptkit", []string{"--against-config=left.yml", "--against-overlay=local"}, "left-model", "right-model"}, } { args := []string{"config", "diff", "--scope=" + test.scope, "--config=left.yml", "--details", "--check", "--format=json"} args = append(args, test.right...) outcome, output, diagnostics := executeNative(t, args...) if outcome.Status != 3 || outcome.Kind != commandkit.OutcomeDomain || !strings.Contains(output, test.before) || !strings.Contains(output, test.after) { t.Fatalf("%v: %+v %q %q", args, outcome, output, diagnostics) } if strings.Contains(output+diagnostics, "password-canary") { t.Fatal("comparison disclosed classified value") } } for _, scope := range []string{"app", "promptkit"} { outcome, _, diagnostics := executeNative(t, "config", "diff", "--scope="+scope, "--config=left.yml", "--against-config=left.yml", "--overlay=local", "--check") if outcome.Status != 3 { t.Fatalf("left overlay inherited by right %s: %+v %q", scope, outcome, diagnostics) } outcome, _, diagnostics = executeNative(t, "config", "diff", "--scope="+scope, "--config=left.yml", "--against-config=left.yml", "--check") if outcome.Status != 0 { t.Fatalf("equal selections %s: %+v %q", scope, outcome, diagnostics) } } } func TestNativeConfigurationOverridesAndCapabilityChecks(t *testing.T) { nativeConfigurationFixture(t) outcome, output, diagnostics := executeNative(t, "config", "show", "--scope=app", "--config=left.yml", `--set=/server/addr="native"`, "--path=/server/addr", "--format=json") if outcome.Status != 0 || strings.TrimSpace(output) != `"native"` { t.Fatalf("override: %+v %q %q", outcome, output, diagnostics) } for _, args := range [][]string{ {"config", "show", "--scope=promptkit", `--set=/server/addr="native"`}, {"config", "show", "--scope=all", `--set=/server/addr="native"`}, {"config", "diff", "--scope=promptkit", `--against-set=/server/addr="native"`}, {"config", "show", "--addr=:9999"}, {"config", "show", "--max-request-bytes=0"}, {"config", "diff", "--against-addr=:9999"}, {"config", "diff", "--against-schema-dir=schemas"}, } { outcome, _, diagnostics := executeNative(t, args...) if outcome.Status != 2 || outcome.Kind != commandkit.OutcomeUsage || diagnostics == "" { t.Fatalf("%v: %+v %q", args, outcome, diagnostics) } } } func TestApplicationFailureEvidenceIsNotFrameworkEvidence(t *testing.T) { nativeConfigurationFixture(t) writeNativeFile(t, "invalid.yml", "server:\n max_request_bytes: -1\n") scope := frameworkConfigurationScope() state := &acquisitionState{} selection := ckconfig.Selection{Config: commandkit.Supplied[string]{Set: true, Value: "invalid.yml"}} acquired, err := scope.Acquire(context.Background(), ckconfig.Request{Operation: ckconfig.Validate, Selection: selection}, applicationInput{}, state) var native *configkit.LoadError if !errors.As(err, &native) || errors.Is(err, errApplicationAcquisition) || ckconfig.ClassifyConfigurationError(err) != commandkit.OutcomeUsage { t.Fatalf("typed application error lost: %v", err) } if acquired.Value.Findings != nil || acquired.Value.Sources != nil || acquired.Value.Effective != nil { t.Fatalf("application evidence inserted into framework view: %+v", acquired.Value) } var details bytes.Buffer for _, detail := range acquired.Explanation.Details { if err := detail(&details); err != nil { t.Fatal(err) } } if !strings.Contains(details.String(), "Application configuration") || !strings.Contains(details.String(), "validate.error") || strings.Contains(details.String(), `scope "promptkit"`) { t.Fatal(details.String()) } outcome, output, diagnostics := executeNative(t, "config", "validate", "--scope=promptkit", "--config=invalid.yml", "--quiet") if outcome.Status != 2 || output != "" || !strings.Contains(diagnostics, "Application configuration") || !strings.Contains(diagnostics, "validate.error") { t.Fatalf("terminal evidence: %+v %q %q", outcome, output, diagnostics) } for _, side := range []string{"left", "right"} { args := []string{"config", "diff", "--scope=promptkit", "--config=left.yml", "--against-config=right.yml"} if side == "left" { args[3] = "--config=invalid.yml" } else { args[4] = "--against-config=invalid.yml" } outcome, _, diagnostics := executeNative(t, args...) if outcome.Status != 2 || !errors.As(outcome.Cause, &native) || !strings.Contains(diagnostics, "Application configuration ("+side+")") { t.Fatalf("%s evidence: %+v %q", side, outcome, diagnostics) } } // In contrast, a framework load failure keeps its native findings and sources. writeNativeFile(t, "bad-framework.yml", "profiles: [\n") writeNativeFile(t, "framework-app.yml", "promptkit:\n configuration: bad-framework.yml\n") selection.Config.Value = "framework-app.yml" acquired, err = scope.Acquire(context.Background(), ckconfig.Request{Operation: ckconfig.Validate, Selection: selection}, applicationInput{}, &acquisitionState{}) if err == nil || acquired.Value.Findings == nil || acquired.Value.Sources == nil || len(acquired.Value.Findings.Records) == 0 { t.Fatalf("native framework evidence lost: %+v %v", acquired, err) } } func TestAcquisitionDiagnosticsRetainBothComparisonSides(t *testing.T) { state := &acquisitionState{} for _, label := range []string{"left", "right"} { result, err := configkit.Load(configkit.Mapping(label, map[string]any{"value": label}), configkit.Options[map[string]any]{ Definition: configkit.Definition[map[string]any]{Validate: func(_ *map[string]any, validation *configkit.Validation) { validation.Warning(configkit.Keys("value"), "application finding") }}, }) if err != nil { t.Fatal(err) } state.applications = append(state.applications, applicationAcquisition{sources: result.Sources(), records: result.Diagnostics()}) } native := commandkit.Acquired[int]{Value: 42, Diagnostics: []commandkit.Diagnostic{{Severity: commandkit.SeverityWarning, Write: func(writer io.Writer) error { _, err := io.WriteString(writer, "framework finding\n") return err }}}} acquired, err := attachApplicationEvidence(native, nil, state, 0, true) if err != nil || acquired.Value != native.Value { t.Fatalf("native value changed: %+v %v", acquired, err) } var output bytes.Buffer for _, diagnostic := range acquired.Diagnostics { if err := diagnostic.Write(&output); err != nil { t.Fatal(err) } } for _, fragment := range []string{"framework finding", "Application configuration (left)", "Application configuration (right)", "validate.warning"} { if !strings.Contains(output.String(), fragment) { t.Fatalf("lost %q: %s", fragment, output.String()) } } // A later callback sharing invocation state must not replay earlier evidence. acquired, err = attachApplicationEvidence(native, nil, state, state.mark(), false) if err != nil || len(acquired.Diagnostics) != len(native.Diagnostics) { t.Fatalf("evidence replayed: %+v %v", acquired, err) } }