From c3023cb609bbd3fe9fc52db91fbafa29db25491d Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Tue, 17 Nov 2026 22:09:58 +0000 Subject: [PATCH] Resolve pipeline resources from their source locations --- docs/roadmap/implementation.md | 2 +- internal/config/effective_digest.go | 26 +---- internal/config/effective_pipeline.go | 4 +- internal/config/effective_projection.go | 8 +- internal/config/load.go | 41 +------ internal/config/notarius_test.go | 10 +- internal/config/pipeline_configkit.go | 28 ++++- internal/config/pipeline_configkit_test.go | 6 +- internal/config/resource_base.go | 45 ++++++++ internal/config/resource_base_test.go | 127 +++++++++++++++++++++ 10 files changed, 209 insertions(+), 88 deletions(-) create mode 100644 internal/config/resource_base.go create mode 100644 internal/config/resource_base_test.go diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 8b39a1b..14d6d76 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -127,7 +127,7 @@ directories and temporary native search descriptions. Test selected invalid input, no source, bootstrap override/default behavior, and diff bootstrap once. Keep tests isolated from the operator's system configuration and secrets. -## Stage 3: Resolve pipeline resources from native origin context +## Stage 3: Resolve pipeline resources from native origin context [x] Read `resolvePipelineConfig`, `resolveNotariusPaths`, `finalizeLoadedPipeline`, `effective_digest.go`, and the upstream source-aware diff --git a/internal/config/effective_digest.go b/internal/config/effective_digest.go index 7cf1476..5f35c97 100644 --- a/internal/config/effective_digest.go +++ b/internal/config/effective_digest.go @@ -4,8 +4,6 @@ import ( "crypto/sha256" "encoding/hex" "fmt" - "path/filepath" - "strings" ) const ( @@ -53,7 +51,7 @@ func recomputePipelineEffectiveDigest(cfg *PipelineConfig) error { if cfg == nil || cfg.resolution == nil { return fmt.Errorf("pipeline resolution metadata is required") } - canonical, err := projectionJSON(projectEffectivePipeline(cfg, true)) + canonical, err := projectionJSON(projectEffectivePipeline(cfg)) if err != nil { return err } @@ -62,25 +60,3 @@ func recomputePipelineEffectiveDigest(cfg *PipelineConfig) error { cfg.resolution.effectiveDigest = hex.EncodeToString(digest[:]) return nil } - -// captureLogicalNotariusPaths retains normalized user-facing path semantics -// before runtime resolution makes relative paths depend on the checkout or -// installation directory. Runtime paths remain absolute; provenance does not. -func captureLogicalNotariusPaths(cfg *PipelineConfig) { - if cfg == nil || cfg.resolution == nil || cfg.Notarius == nil { - return - } - configPath := normalizeLogicalFilesystemPath(cfg.Notarius.ConfigPath) - workingDirectory := normalizeLogicalFilesystemPath(cfg.Notarius.WorkingDirectory) - cfg.resolution.logicalNotariusConfig = configPath - cfg.resolution.logicalNotariusWorking = workingDirectory - cfg.resolution.logicalNotariusCaptured = true -} - -func normalizeLogicalFilesystemPath(value string) string { - value = strings.TrimSpace(value) - if value == "" { - return "" - } - return filepath.ToSlash(filepath.Clean(value)) -} diff --git a/internal/config/effective_pipeline.go b/internal/config/effective_pipeline.go index 6b86457..734fd5c 100644 --- a/internal/config/effective_pipeline.go +++ b/internal/config/effective_pipeline.go @@ -63,7 +63,7 @@ func EffectivePipelineValues(cfg *PipelineConfig) ([]EffectivePipelineValueRecor if cfg == nil { return nil, fmt.Errorf("pipeline config is required") } - return projectionValueRecords(projectEffectivePipeline(cfg, true)) + return projectionValueRecords(projectEffectivePipeline(cfg)) } // EffectivePipelineSources projects pipeline ownership after defaults and @@ -157,7 +157,7 @@ func MarshalEffectivePipeline(cfg *PipelineConfig) ([]byte, error) { return nil, fmt.Errorf("pipeline config is required") } - value, err := projectionValueTree(projectEffectivePipeline(cfg, true)) + value, err := projectionValueTree(projectEffectivePipeline(cfg)) if err != nil { return nil, err } diff --git a/internal/config/effective_projection.go b/internal/config/effective_projection.go index 28b401f..b37e134 100644 --- a/internal/config/effective_projection.go +++ b/internal/config/effective_projection.go @@ -43,7 +43,7 @@ type projectedPipelineValue struct { segments []string } -func projectEffectivePipeline(cfg *PipelineConfig, logicalNotarius bool) effectivePipelineProjection { +func projectEffectivePipeline(cfg *PipelineConfig) effectivePipelineProjection { projection := effectivePipelineProjection{ Workspace: cfg.Workspace, Campaigns: cfg.Campaigns, Storage: cfg.Storage, Spool: cfg.Spool, Cache: cfg.Cache, Publish: cfg.Publish, @@ -59,12 +59,6 @@ func projectEffectivePipeline(cfg *PipelineConfig, logicalNotarius bool) effecti Artifacts: cfg.Scriptorium.Artifacts, } } - if logicalNotarius && cfg.Notarius != nil && cfg.resolution != nil && cfg.resolution.logicalNotariusCaptured { - notarius := *cfg.Notarius - notarius.ConfigPath = cfg.resolution.logicalNotariusConfig - notarius.WorkingDirectory = cfg.resolution.logicalNotariusWorking - projection.Notarius = ¬arius - } return projection } diff --git a/internal/config/load.go b/internal/config/load.go index 5800cfc..d115807 100644 --- a/internal/config/load.go +++ b/internal/config/load.go @@ -30,7 +30,7 @@ func LoadPipelineWithOptions(path string, opts PipelineLoadOptions) (*PipelineCo if err != nil { return nil, err } - return finalizeLoadedPipeline(EffectivePipelineRootPath(cfg), cfg) + return finalizeLoadedPipeline(cfg) } // LoadPipelineOverlayPair resolves two overlays as independent ConfigKit loads. @@ -51,11 +51,7 @@ func LoadPipelineOverlayPair(path, leftOverlay, rightOverlay string) (*PipelineC return left, right, nil } -func finalizeLoadedPipeline(path string, cfg *PipelineConfig) (*PipelineConfig, error) { - captureLogicalNotariusPaths(cfg) - if err := resolveNotariusPaths(cfg, path); err != nil { - return nil, fmt.Errorf("load pipeline config: %w", err) - } +func finalizeLoadedPipeline(cfg *PipelineConfig) (*PipelineConfig, error) { finalizeArtifactDependencies(cfg) if err := finalizePipelineResolution(cfg); err != nil { return nil, fmt.Errorf("load pipeline config: %w", err) @@ -427,36 +423,3 @@ func shortName(path, fallback string) string { } return base } - -func resolveNotariusPaths(cfg *PipelineConfig, pipelinePath string) error { - if cfg == nil || cfg.Notarius == nil || !cfg.Notarius.Enabled { - return nil - } - pipelineAbs, err := filepath.Abs(pipelinePath) - if err != nil { - return fmt.Errorf("resolve pipeline config path %q: %w", pipelinePath, err) - } - baseDir := filepath.Dir(pipelineAbs) - resolve := func(value string) (string, error) { - trimmed := strings.TrimSpace(value) - if trimmed == "" { - return "", nil - } - if !filepath.IsAbs(trimmed) { - trimmed = filepath.Join(baseDir, trimmed) - } - return filepath.Abs(trimmed) - } - - resolvedConfigPath, err := resolve(cfg.Notarius.ConfigPath) - if err != nil { - return fmt.Errorf("resolve pipeline.notarius.config_path: %w", err) - } - cfg.Notarius.ConfigPath = resolvedConfigPath - workingDirectory, err := resolve(cfg.Notarius.WorkingDirectory) - if err != nil { - return fmt.Errorf("resolve pipeline.notarius.working_directory: %w", err) - } - cfg.Notarius.WorkingDirectory = workingDirectory - return nil -} diff --git a/internal/config/notarius_test.go b/internal/config/notarius_test.go index d0abd15..925fe25 100644 --- a/internal/config/notarius_test.go +++ b/internal/config/notarius_test.go @@ -51,8 +51,8 @@ notarius: if disabled.Notarius.Resume || disabled.Notarius.ChunkCache != "" { t.Fatalf("disabled cache-control defaults = %#v", disabled.Notarius) } - if disabled.Notarius.ConfigPath != "relative/notarius.yml" || disabled.Notarius.WorkingDirectory != "" { - t.Fatalf("disabled paths were resolved unexpectedly: %#v", disabled.Notarius) + if disabled.Notarius.ConfigPath != filepath.Join(dir, "relative/notarius.yml") || disabled.Notarius.WorkingDirectory != "" { + t.Fatalf("disabled resource paths = %#v", disabled.Notarius) } } @@ -294,7 +294,7 @@ func TestNotariusCacheControlsSafeSerialization(t *testing.T) { } } -func TestNotariusRelativePathsDoNotMakeEffectiveDigestLocationDependent(t *testing.T) { +func TestNotariusRelativePathsAffectEffectiveDigestWhenResourcesDiffer(t *testing.T) { pipelineYAML := testPipelineBaseYAML + ` notarius: enabled: true @@ -325,8 +325,8 @@ notarius: configs[0].Notarius.WorkingDirectory == configs[1].Notarius.WorkingDirectory { t.Fatalf("runtime Notarius paths should remain location-specific: %#v / %#v", configs[0].Notarius, configs[1].Notarius) } - if first, second := EffectivePipelineDigest(configs[0]), EffectivePipelineDigest(configs[1]); first == "" || first != second { - t.Fatalf("relocated logical configuration digests = %q / %q, want equal non-empty values", first, second) + if first, second := EffectivePipelineDigest(configs[0]), EffectivePipelineDigest(configs[1]); first == "" || first == second { + t.Fatalf("different resource digests = %q / %q, want distinct non-empty values", first, second) } } diff --git a/internal/config/pipeline_configkit.go b/internal/config/pipeline_configkit.go index c0f633e..72beea1 100644 --- a/internal/config/pipeline_configkit.go +++ b/internal/config/pipeline_configkit.go @@ -4,6 +4,7 @@ import ( "bytes" "errors" "fmt" + "os" "path/filepath" "reflect" "strings" @@ -17,9 +18,6 @@ type pipelineResolutionMetadata struct { sources []string selectedOverlay *pipelineOverlaySelection effectiveDigest string - logicalNotariusConfig string - logicalNotariusWorking string - logicalNotariusCaptured bool artifactFamilies map[string]ScriptoriumArtifactFamilyConfig artifactFamiliesExpanded bool configkitMetadata configkit.Metadata @@ -52,12 +50,18 @@ func loadPipelineWithConfigKit(path string, opts PipelineLoadOptions) (*Pipeline } func loadPipelineConfigKitResult(source configkit.Source, overlay string) (*configkit.Result[PipelineConfig], error) { + invocationDir, err := os.Getwd() + if err != nil { + return nil, fmt.Errorf("capture invocation directory: %w", err) + } return configkit.Load(source, configkit.Options[PipelineConfig]{ Imports: true, Overlay: overlay, Definition: configkit.Definition[PipelineConfig]{ Defaults: pipelineConfigDefaults, - Resolve: resolvePipelineConfig, + Resolve: func(cfg *PipelineConfig, resolution *configkit.Resolution) error { + return resolvePipelineConfig(cfg, resolution, invocationDir) + }, Validate: validatePipelineConfigKit, }, }) @@ -154,7 +158,19 @@ func defaultPublishOutputs() []PublishOutputRule { return outputs } -func resolvePipelineConfig(cfg *PipelineConfig, resolution *configkit.Resolution) error { +func resolvePipelineConfig(cfg *PipelineConfig, resolution *configkit.Resolution, invocationDir string) error { + resolveHostResource(resolution, configkit.Keys("workspace", "root"), &cfg.Workspace.Root, invocationDir) + resolveHostResource(resolution, configkit.Keys("campaigns", "root"), &cfg.Campaigns.Root, invocationDir) + resolveHostResource(resolution, configkit.Keys("spool", "root"), &cfg.Spool.Root, invocationDir) + resolveHostResource(resolution, configkit.Keys("cache", "root"), &cfg.Cache.Root, invocationDir) + resolveHostResource(resolution, configkit.Keys("audita", "config_path"), &cfg.Audita.ConfigPath, invocationDir) + if cfg.Scriptorium != nil { + resolveHostResource(resolution, configkit.Keys("scriptorium", "config_path"), &cfg.Scriptorium.ConfigPath, invocationDir) + } + if cfg.Notarius != nil { + resolveHostResource(resolution, configkit.Keys("notarius", "config_path"), &cfg.Notarius.ConfigPath, invocationDir) + resolveHostResource(resolution, configkit.Keys("notarius", "working_directory"), &cfg.Notarius.WorkingDirectory, invocationDir) + } backend := strings.ToLower(strings.TrimSpace(cfg.Storage.Backend)) if backend != cfg.Storage.Backend { configkit.Normalize(resolution, configkit.Keys("storage", "backend"), &cfg.Storage.Backend, backend, "canonical storage backend") @@ -279,7 +295,7 @@ func pipelineMetadataFromConfigKit(result *configkit.Result[PipelineConfig], opt } func pipelineMetadataPaths(cfg *PipelineConfig) ([]projectedPipelineValue, error) { - effective, err := projectionDetailedRecords(projectEffectivePipeline(cfg, false)) + effective, err := projectionDetailedRecords(projectEffectivePipeline(cfg)) if err != nil { return nil, err } diff --git a/internal/config/pipeline_configkit_test.go b/internal/config/pipeline_configkit_test.go index abdd473..55fab2b 100644 --- a/internal/config/pipeline_configkit_test.go +++ b/internal/config/pipeline_configkit_test.go @@ -338,7 +338,7 @@ whisperx: } } -func TestInheritedNotariusPathsKeepPipelineRootBase(t *testing.T) { +func TestInheritedNotariusPathsFollowSupplyingSource(t *testing.T) { dir := t.TempDir() pipelinePath := writePipelineFixture(t, dir, "pipeline.yml", `config: imports: @@ -364,9 +364,9 @@ notarius: if err != nil { t.Fatalf("LoadPipeline() error = %v", err) } - wantConfigPath := filepath.Join(dir, "notarius", "config.yml") + wantConfigPath := filepath.Join(dir, "profiles", "notarius", "config.yml") if cfg.Notarius.ConfigPath != wantConfigPath { - t.Fatalf("imported Notarius config_path = %q, want pipeline-root path %q", cfg.Notarius.ConfigPath, wantConfigPath) + t.Fatalf("imported Notarius config_path = %q, want source-relative path %q", cfg.Notarius.ConfigPath, wantConfigPath) } if got, want := cfg.Notarius.WorkingDirectory, filepath.Dir(wantConfigPath); got != want { t.Fatalf("derived Notarius working_directory = %q, want pipeline-root directory %q", got, want) diff --git a/internal/config/resource_base.go b/internal/config/resource_base.go new file mode 100644 index 0000000..e3c8ee5 --- /dev/null +++ b/internal/config/resource_base.go @@ -0,0 +1,45 @@ +package config + +import ( + "path/filepath" + "strings" + + configkit "go.maximumdirect.net/configkit" +) + +// resourceBase chooses the effective host source, falling back to the main +// source and then the captured invocation directory for contextless inputs. +func resourceBase(resolution *configkit.Resolution, path configkit.Path, invocationDir string) string { + info, found := resolution.EntryInfo(path) + if found && info.Origin.Layer == configkit.LayerCLI { + return invocationDir + } + if found && len(info.Origin.Locations) == 1 { + if base, ok := hostSourceBase(resolution, info.Origin.Locations[0].SourceID); ok { + return base + } + } + if base, ok := hostSourceBase(resolution, "source-1"); ok { + return base + } + return invocationDir +} + +func hostSourceBase(resolution *configkit.Resolution, sourceID string) (string, bool) { + source, ok := resolution.Source(sourceID) + if !ok || source.Kind != configkit.SourceKindFile || !filepath.IsAbs(source.Path) { + return "", false + } + if _, ok := resolution.SourceContext(sourceID); !ok { + return "", false + } + return filepath.Dir(source.Path), true +} + +func resolveHostResource(resolution *configkit.Resolution, path configkit.Path, value *string, invocationDir string) { + trimmed := strings.TrimSpace(*value) + if trimmed == "" || filepath.IsAbs(trimmed) { + return + } + configkit.Normalize(resolution, path, value, filepath.Join(resourceBase(resolution, path, invocationDir), trimmed), "resolved relative resource from its source") +} diff --git a/internal/config/resource_base_test.go b/internal/config/resource_base_test.go new file mode 100644 index 0000000..8ebd1c4 --- /dev/null +++ b/internal/config/resource_base_test.go @@ -0,0 +1,127 @@ +package config + +import ( + "os" + "path/filepath" + "testing" + + configkit "go.maximumdirect.net/configkit" +) + +func TestPipelineResourcePathsFollowEffectiveSources(t *testing.T) { + dir := t.TempDir() + root := writePipelineFixture(t, dir, "pipeline.yml", `config: + imports: + - path: fragments/shared.yml +workspace: {root: root-work} +`) + writePipelineFixture(t, dir, "fragments/shared.yml", `campaigns: {root: campaigns} +spool: {root: /absolute/spool} +cache: {root: cache} +audita: {config_path: config/tool.yml} +scriptorium: {extends: audita} +notarius: + extends: audita + enabled: true + pipeline_id: dnd-session +`) + writePipelineFixture(t, dir, "overlays/testing.yml", "audita:\n config_path: custom.yml\n") + base, err := LoadPipeline(root) + if err != nil { + t.Fatal(err) + } + wantConfig := filepath.Join(dir, "fragments", "config", "tool.yml") + if base.Audita.ConfigPath != wantConfig || base.Scriptorium.ConfigPath != wantConfig || base.Notarius.ConfigPath != wantConfig { + t.Fatalf("resource paths: audita=%q scriptorium=%q notarius=%q", base.Audita.ConfigPath, base.Scriptorium.ConfigPath, base.Notarius.ConfigPath) + } + if base.Notarius.WorkingDirectory != filepath.Dir(wantConfig) || base.Workspace.Root != filepath.Join(dir, "root-work") || base.Campaigns.Root != filepath.Join(dir, "fragments", "campaigns") || base.Cache.Root != filepath.Join(dir, "fragments", "cache") || base.Spool.Root != "/absolute/spool" { + t.Fatalf("resolved paths: %#v", base) + } + overlay := "testing" + updated, err := LoadPipelineWithOptions(root, PipelineLoadOptions{Overlay: &overlay}) + if err != nil { + t.Fatal(err) + } + wantOverlay := filepath.Join(dir, "overlays", "custom.yml") + if updated.Audita.ConfigPath != wantOverlay || updated.Scriptorium.ConfigPath != wantOverlay || updated.Notarius.ConfigPath != wantOverlay { + t.Fatalf("overlay resource paths: audita=%q scriptorium=%q notarius=%q", updated.Audita.ConfigPath, updated.Scriptorium.ConfigPath, updated.Notarius.ConfigPath) + } +} + +func TestResourceBaseUsesInvocationDirectoryForContextlessAndCLIValues(t *testing.T) { + invocation := t.TempDir() + type document struct { + Path string `yaml:"path"` + } + for _, overrides := range [][]configkit.Override{nil, {{Path: configkit.Keys("path"), Value: "cli.txt", Flag: "--path"}}} { + result, err := configkit.Load(configkit.Bytes("remote snapshot", []byte("path: input.txt\n")), configkit.Options[document]{ + Overrides: overrides, + Definition: configkit.Definition[document]{Resolve: func(doc *document, resolution *configkit.Resolution) error { + resolveHostResource(resolution, configkit.Keys("path"), &doc.Path, invocation) + return nil + }}, + }) + if err != nil { + t.Fatal(err) + } + name := "input.txt" + if len(overrides) > 0 { + name = "cli.txt" + } + if got, want := result.Config.Path, filepath.Join(invocation, name); got != want { + t.Fatalf("resolved %q, want %q", got, want) + } + } +} + +func TestEquivalentPipelineResourceLayoutsHaveEqualDigests(t *testing.T) { + dir := t.TempDir() + first := writePipelineFixture(t, dir, "first.yml", "audita: {config_path: shared/tool.yml}\n") + second := writePipelineFixture(t, dir, "second.yml", "config:\n imports:\n - path: shared/settings.yml\n") + writePipelineFixture(t, dir, "shared/settings.yml", "audita: {config_path: tool.yml}\n") + a, err := LoadPipeline(first) + if err != nil { + t.Fatal(err) + } + b, err := LoadPipeline(second) + if err != nil { + t.Fatal(err) + } + if a.Audita.ConfigPath != b.Audita.ConfigPath || EffectivePipelineDigest(a) != EffectivePipelineDigest(b) { + t.Fatalf("equivalent resources differ: %q / %q", a.Audita.ConfigPath, b.Audita.ConfigPath) + } + if _, err := os.Stat(filepath.Join(dir, "shared", "tool.yml")); !os.IsNotExist(err) { + t.Fatalf("fixture unexpectedly exists: %v", err) + } +} + +func TestResourceBaseFallsBackToMainForCompositeAndDefaultOrigins(t *testing.T) { + dir := t.TempDir() + root := writePipelineFixture(t, dir, "pipeline.yml", "config:\n imports:\n - path: fragment/part.yml\nvalues: {first: one}\n") + writePipelineFixture(t, dir, "fragment/part.yml", "values: {second: two}\n") + type document struct { + Values map[string]string `yaml:"values"` + Default string `yaml:"default"` + } + var compositeBase, defaultBase string + var contributors int + _, err := configkit.Load(configkit.File(root), configkit.Options[document]{ + Imports: true, + Definition: configkit.Definition[document]{ + Defaults: func() document { return document{Default: "default.txt"} }, + Resolve: func(_ *document, resolution *configkit.Resolution) error { + info, _ := resolution.EntryInfo(configkit.Keys("values")) + contributors = len(info.Origin.Locations) + compositeBase = resourceBase(resolution, configkit.Keys("values"), t.TempDir()) + defaultBase = resourceBase(resolution, configkit.Keys("default"), t.TempDir()) + return nil + }, + }, + }) + if err != nil { + t.Fatal(err) + } + if contributors < 2 || compositeBase != dir || defaultBase != dir { + t.Fatalf("fallback bases = %q, %q; want %q", compositeBase, defaultBase, dir) + } +}