From 61ceab786c8c4c747e3f7ef5d00b31063f6514ab Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Tue, 17 Nov 2026 20:48:05 +0000 Subject: [PATCH] Compose pipeline steps using native identities --- docs/roadmap/implementation.md | 6 + .../config/collection_composition_test.go | 180 ++++++++++++++++++ internal/core/config/document_normalizer.go | 16 ++ internal/core/config/document_paths.go | 28 +-- internal/core/config/document_paths_test.go | 6 +- internal/core/config/load.go | 7 + internal/core/config/operational_document.go | 2 +- internal/core/config/pipeline_document.go | 31 +-- 8 files changed, 249 insertions(+), 27 deletions(-) create mode 100644 internal/core/config/collection_composition_test.go create mode 100644 internal/core/config/document_normalizer.go diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 0ac0a5ae..f1af2bcb 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -228,6 +228,12 @@ Prompt size: one resolution/provenance boundary. ## Stage 4 — Register Steps And Exercise Native Collection Semantics +**Complete.** Registered step IDs and verified real-file peer composition, +upsert, replacement, append, clearing validation, inherited records, CLI list +assignment, source-relative references, duplicate peers, and unregistered lists. +Focused config and framework resolution tests pass. Identity-changing +normalization records the affected list, while unchanged identities retain leaf history. + Goal: implement the accepted step identity and composition policy. Inspect documentDefinition, pipeline step validation/conversion, ConfigKit's diff --git a/internal/core/config/collection_composition_test.go b/internal/core/config/collection_composition_test.go new file mode 100644 index 00000000..d9d8f72d --- /dev/null +++ b/internal/core/config/collection_composition_test.go @@ -0,0 +1,180 @@ +package config + +import ( + "os" + "path/filepath" + "testing" + + configkit "go.maximumdirect.net/configkit" +) + +func TestStepCompositionAcrossSourcesAndOverlays(t *testing.T) { + root := t.TempDir() + write := func(name, data string) { + t.Helper() + path := filepath.Join(root, name) + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(data), 0600); err != nil { + t.Fatal(err) + } + } + write("notarius.yml", `version: 5 +config: + imports: + - path: one/pipeline.yml + required: true + - path: two/pipeline.yml + required: true +`) + write("one/pipeline.yml", `pipelines: + session: + input: {module: input} + steps: + - id: first + references: {source: {path: ./first.txt}} + artifacts: {first: {extract: {module: extract, retries: 4}}} +`) + write("two/pipeline.yml", `pipelines: + session: + steps: + - id: second + references: {source: {path: ./second.txt}} + artifacts: {second: {extract: {module: extract}}} +`) + write("overlays/upsert.yml", `pipelines: + session: + steps: + - id: first + artifacts: {replacement: {extract: {module: extract}}} +`) + write("overlays/replace.yml", `config: + sequences: + - destination: [pipelines, session, steps] + mode: replace +pipelines: + session: + steps: + - id: third + artifacts: {third: {extract: {module: extract}}} +`) + write("overlays/append.yml", `config: + sequences: + - destination: [pipelines, session, steps] + mode: append +pipelines: + session: + steps: + - id: third + artifacts: {third: {extract: {module: extract}}} +`) + write("overlays/clear.yml", `config: + sequences: + - destination: [pipelines, session, steps] + mode: replace +pipelines: + session: {steps: []} +`) + main := filepath.Join(root, "notarius.yml") + loaded, err := Load(main, LoadOptions{}) + if err != nil { + t.Fatal(err) + } + steps := loaded.Config.Pipelines["session"].Steps + if len(steps) != 2 || steps[0].ID != "first" || steps[1].ID != "second" || steps[0].References["source"].Path != filepath.Join(root, "one", "first.txt") || steps[1].References["source"].Path != filepath.Join(root, "two", "second.txt") { + t.Fatalf("composed steps = %#v", steps) + } + for _, tc := range []struct { + overlay string + ids []string + }{{"upsert", []string{"first", "second"}}, {"replace", []string{"third"}}, {"append", []string{"first", "second", "third"}}} { + t.Run(tc.overlay, func(t *testing.T) { + got, err := Load(main, LoadOptions{Overlay: tc.overlay}) + if err != nil { + t.Fatal(err) + } + steps := got.Config.Pipelines["session"].Steps + if len(steps) != len(tc.ids) { + t.Fatalf("steps = %#v", steps) + } + for i, id := range tc.ids { + if steps[i].ID != id { + t.Fatalf("steps = %#v", steps) + } + } + if tc.overlay == "upsert" && (len(steps[0].References) != 0 || steps[0].Artifacts["replacement"].Extract.Module != "extract") { + t.Fatalf("upsert did not replace record: %#v", steps[0]) + } + }) + } + if _, err := Load(main, LoadOptions{Overlay: "clear"}); err == nil { + t.Fatal("domain validation accepted empty steps") + } + overrides := []configkit.Override{{Path: configkit.Keys("pipelines", "session", "steps"), Value: []any{map[string]any{"id": "override", "artifacts": map[string]any{"result": map[string]any{"extract": map[string]any{"module": "extract"}}}}}, Flag: "--steps"}} + replaced, err := Load(main, LoadOptions{Overrides: overrides}) + if err != nil { + t.Fatal(err) + } + if got := replaced.Config.Pipelines["session"].Steps; len(got) != 1 || got[0].ID != "override" { + t.Fatalf("override steps = %#v", got) + } +} + +func TestStepIdentityRejectsDuplicatePeersWithoutRegisteringOtherLists(t *testing.T) { + for _, tc := range []struct { + name, first, second string + }{ + {"duplicate steps", "steps: [{id: same, artifacts: {a: {extract: {module: extract}}}}]", "steps: [{id: same, artifacts: {b: {extract: {module: extract}}}}]"}, + {"unregistered validators", "input: {module: input, validators: [{module: one}]}", "input: {validators: [{module: two}]}"}, + } { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + for name, data := range map[string]string{ + "main.yml": "version: 5\nconfig:\n imports:\n - path: one.yml\n - path: two.yml\n", + "one.yml": "pipelines:\n session:\n " + tc.first + "\n", + "two.yml": "pipelines:\n session:\n " + tc.second + "\n", + } { + if err := os.WriteFile(filepath.Join(root, name), []byte(data), 0600); err != nil { + t.Fatal(err) + } + } + if _, err := Load(filepath.Join(root, "main.yml"), LoadOptions{}); err == nil { + t.Fatal("conflicting peer lists were accepted") + } + }) + } +} + +func TestStepRecordInheritanceReplacesMatchingRecord(t *testing.T) { + root := t.TempDir() + main := filepath.Join(root, "main.yml") + if err := os.WriteFile(main, []byte(`version: 5 +pipelines: + parent: + input: {module: input} + steps: + - id: first + artifacts: {old: {extract: {module: extract, retries: 4}}} + - id: second + artifacts: {second: {extract: {module: extract}}} + child: + extends: parent + steps: + - id: first + artifacts: {new: {extract: {module: extract}}} +`), 0600); err != nil { + t.Fatal(err) + } + loaded, err := Load(main, LoadOptions{}) + if err != nil { + t.Fatal(err) + } + steps := loaded.Config.Pipelines["child"].Steps + if len(steps) != 2 || steps[0].ID != "first" || steps[1].ID != "second" || len(steps[0].Artifacts) != 1 || steps[0].Artifacts["new"].Extract.Module != "extract" { + t.Fatalf("inherited steps = %#v", steps) + } + if _, err := loaded.Config.Resolve(ResolveInput{PipelineID: "child", Catalog: effectiveCatalog(t)}); err != nil { + t.Fatalf("resolve composed pipeline: %v", err) + } +} diff --git a/internal/core/config/document_normalizer.go b/internal/core/config/document_normalizer.go new file mode 100644 index 00000000..6102db76 --- /dev/null +++ b/internal/core/config/document_normalizer.go @@ -0,0 +1,16 @@ +package config + +import configkit "go.maximumdirect.net/configkit" + +// A step whose identity is normalized no longer corresponds to its original +// record. Its ancestor record covers its edits instead of invalid leaf records. +type documentNormalizer struct { + *configkit.Resolution + covered bool +} + +func (n documentNormalizer) RecordNormalization(path configkit.Path, reason string, dependencies ...configkit.Path) { + if !n.covered { + n.Resolution.RecordNormalization(path, reason, dependencies...) + } +} diff --git a/internal/core/config/document_paths.go b/internal/core/config/document_paths.go index 55d97fcf..d4d23b4b 100644 --- a/internal/core/config/document_paths.go +++ b/internal/core/config/document_paths.go @@ -8,6 +8,7 @@ import ( ) func normalizeDocumentPaths(document *Document, resolution *configkit.Resolution, state *documentLoadState) { + normalizer := documentNormalizer{Resolution: resolution} if document == nil { return } @@ -33,24 +34,29 @@ func normalizeDocumentPaths(document *Document, resolution *configkit.Resolution for _, id := range sortedPipelineDocumentKeys(document.Pipelines) { profile := document.Pipelines[id] path := configkit.Keys("pipelines").Key(id) - normalizeDocumentReferencePaths(profile.References, path.Key("references"), resolution, state) - normalizeDocumentBindingReferencePaths(&profile.Input, path.Key("input"), resolution, state) + normalizeDocumentReferencePaths(profile.References, path.Key("references"), normalizer, state) + normalizeDocumentBindingReferencePaths(&profile.Input, path.Key("input"), normalizer, state) if profile.Chunk != nil { - normalizeDocumentBindingReferencePaths(profile.Chunk, path.Key("chunk"), resolution, state) + normalizeDocumentBindingReferencePaths(profile.Chunk, path.Key("chunk"), normalizer, state) } if profile.Output != nil { - normalizeDocumentBindingReferencePaths(profile.Output, path.Key("output"), resolution, state) + normalizeDocumentBindingReferencePaths(profile.Output, path.Key("output"), normalizer, state) } for laneID, lane := range profile.Artifacts { - normalizeDocumentLaneReferencePaths(&lane, path.Key("artifacts").Key(laneID), resolution, state) + normalizeDocumentLaneReferencePaths(&lane, path.Key("artifacts").Key(laneID), normalizer, state) profile.Artifacts[laneID] = lane } for index := range profile.Steps { step := &profile.Steps[index] stepPath := path.Key("steps").Index(index) - normalizeDocumentReferencePaths(step.References, stepPath.Key("references"), resolution, state) + stepNormalizer := normalizer + if step.ID != strings.TrimSpace(step.ID) { + resolution.RecordNormalization(path.Key("steps"), "normalized step identity") + stepNormalizer.covered = true + } + normalizeDocumentReferencePaths(step.References, stepPath.Key("references"), stepNormalizer, state) for laneID, lane := range step.Artifacts { - normalizeDocumentLaneReferencePaths(&lane, stepPath.Key("artifacts").Key(laneID), resolution, state) + normalizeDocumentLaneReferencePaths(&lane, stepPath.Key("artifacts").Key(laneID), stepNormalizer, state) step.Artifacts[laneID] = lane } } @@ -75,13 +81,13 @@ func initializeDocumentMainDirectory(resolution *configkit.Resolution, state *do state.mainDirectory = cleanInvocationDirectory(".") } -func normalizeDocumentBindingReferencePaths(binding *BindingDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) { +func normalizeDocumentBindingReferencePaths(binding *BindingDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) { if binding != nil { normalizeDocumentReferencePaths(binding.References, path.Key("references"), resolution, state) } } -func normalizeDocumentLaneReferencePaths(lane *ArtifactLaneDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) { +func normalizeDocumentLaneReferencePaths(lane *ArtifactLaneDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) { if lane == nil { return } @@ -90,7 +96,7 @@ func normalizeDocumentLaneReferencePaths(lane *ArtifactLaneDocument, path config normalizeDocumentBindingReferencePaths(lane.Normalize, path.Key("normalize"), resolution, state) } -func normalizeDocumentReferencePaths(references map[string]ReferenceDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) { +func normalizeDocumentReferencePaths(references map[string]ReferenceDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) { for _, name := range sortedReferenceDocumentKeys(references) { source := references[name] sourcePath := path.Key(name).Key("path") @@ -102,7 +108,7 @@ func normalizeDocumentReferencePaths(references map[string]ReferenceDocument, pa } } if source.Path != nil && *source.Path != "" && !filepath.IsAbs(*source.Path) { - base, ok := documentPathBase(resolution, state, sourcePath) + base, ok := documentPathBase(resolution.Resolution, state, sourcePath) if !ok { resolution.Error(sourcePath, "relative path has no unambiguous source base") } else { diff --git a/internal/core/config/document_paths_test.go b/internal/core/config/document_paths_test.go index cd041791..a22e5ceb 100644 --- a/internal/core/config/document_paths_test.go +++ b/internal/core/config/document_paths_test.go @@ -77,6 +77,10 @@ func TestDiscoveredMainAndStepReferenceNormalization(t *testing.T) { artifacts: result: extract: {module: " extract "} + - id: second + artifacts: + next: + extract: {module: " extract "} `), 0600); err != nil { t.Fatal(err) } @@ -91,7 +95,7 @@ func TestDiscoveredMainAndStepReferenceNormalization(t *testing.T) { if step.ID != "first" || step.Artifacts["result"].Extract.Module != "extract" || step.References["source"].Path != filepath.Join(fragmentDir, "source.txt") { t.Fatalf("normalized step = %#v", step) } - history := loaded.result.Metadata().History(configkit.Keys("pipelines", "session", "steps").Index(0).Key("artifacts").Key("result").Key("extract").Key("module")) + history := loaded.result.Metadata().History(configkit.Keys("pipelines", "session", "steps").Index(1).Key("artifacts").Key("next").Key("extract").Key("module")) if len(history) < 2 || history[len(history)-1].Origin.Operation != configkit.ChangeNormalization { t.Fatalf("module leaf history = %#v", history) } diff --git a/internal/core/config/load.go b/internal/core/config/load.go index b3cc2aba..b45b4018 100644 --- a/internal/core/config/load.go +++ b/internal/core/config/load.go @@ -58,6 +58,13 @@ func documentDefinition(state *documentLoadState) configkit.Definition[Document] Check: validateDocumentVersion, }, Sensitive: sensitiveDocumentPath, + SequenceIdentities: []configkit.SequenceIdentity{ + configkit.SequenceIdentityMatching(configkit.SequenceSelector{ + configkit.SequenceLiteralKey("pipelines"), + configkit.SequenceAnyMappingKey(), + configkit.SequenceLiteralKey("steps"), + }, "id"), + }, } } diff --git a/internal/core/config/operational_document.go b/internal/core/config/operational_document.go index 8c74466c..3fec3ab9 100644 --- a/internal/core/config/operational_document.go +++ b/internal/core/config/operational_document.go @@ -21,7 +21,7 @@ func resolveDocumentOperations(document *Document, resolution *configkit.Resolut if document.Concurrency != nil { document.Concurrency.StageWorkers = normalizeDocumentMap(document.Concurrency.StageWorkers, configkit.Keys("concurrency", "stage_workers"), "stage worker identifier", - resolution, state) + documentNormalizer{Resolution: resolution}, state) } deriveDocumentExtractWorkers(document, resolution, state) return nil diff --git a/internal/core/config/pipeline_document.go b/internal/core/config/pipeline_document.go index ff1a4dae..81242886 100644 --- a/internal/core/config/pipeline_document.go +++ b/internal/core/config/pipeline_document.go @@ -64,13 +64,14 @@ func normalizeDocumentPipelines(document *Document, resolution *configkit.Resolu return } validateDocumentPipelinePresence(document, resolution) + normalizer := documentNormalizer{Resolution: resolution} pipelinesPath := configkit.Keys("pipelines") for _, id := range sortedPipelineDocumentKeys(document.Pipelines) { profile := document.Pipelines[id] - normalizePipelineDocument(&profile, pipelinesPath.Key(id), resolution, state) + normalizePipelineDocument(&profile, pipelinesPath.Key(id), normalizer, state) document.Pipelines[id] = profile } - document.Pipelines = normalizeDocumentMap(document.Pipelines, pipelinesPath, "pipeline identifier", resolution, state) + document.Pipelines = normalizeDocumentMap(document.Pipelines, pipelinesPath, "pipeline identifier", normalizer, state) } func validateDocumentPipelinePresence(document *Document, resolution *configkit.Resolution) { @@ -174,7 +175,7 @@ func validateDocumentPolicyPresence(policy *ValidationPolicyDocument, path confi rejectDocumentNull(resolution, path.Key("validator_failure"), policy.ValidatorFailure == nil) } -func normalizePipelineDocument(profile *PipelineDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) { +func normalizePipelineDocument(profile *PipelineDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) { normalizePipelineDocumentString(resolution, path.Key("llm_profile"), &profile.LLMProfile, "trimmed pipeline LLM profile") normalizeDocumentBinding(&profile.Input, path.Key("input"), resolution, state) if profile.Chunk != nil { @@ -193,18 +194,20 @@ func normalizePipelineDocument(profile *PipelineDocument, path configkit.Path, r for index := range profile.Steps { step := &profile.Steps[index] stepPath := path.Key("steps").Index(index) - normalizeDocumentValue(resolution, stepPath.Key("id"), &step.ID, strings.TrimSpace(step.ID), "trimmed step identifier") - step.References = normalizeDocumentReferences(step.References, stepPath.Key("references"), resolution, state) + stepNormalizer := resolution + stepNormalizer.covered = step.ID != strings.TrimSpace(step.ID) + normalizeDocumentValue(stepNormalizer, stepPath.Key("id"), &step.ID, strings.TrimSpace(step.ID), "trimmed step identifier") + step.References = normalizeDocumentReferences(step.References, stepPath.Key("references"), stepNormalizer, state) for _, laneID := range sortedLaneDocumentKeys(step.Artifacts) { lane := step.Artifacts[laneID] - normalizeArtifactLaneDocument(&lane, stepPath.Key("artifacts").Key(laneID), resolution, state) + normalizeArtifactLaneDocument(&lane, stepPath.Key("artifacts").Key(laneID), stepNormalizer, state) step.Artifacts[laneID] = lane } - step.Artifacts = normalizeDocumentMap(step.Artifacts, stepPath.Key("artifacts"), "artifact lane identifier", resolution, state) + step.Artifacts = normalizeDocumentMap(step.Artifacts, stepPath.Key("artifacts"), "artifact lane identifier", stepNormalizer, state) } } -func normalizeArtifactLaneDocument(lane *ArtifactLaneDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) { +func normalizeArtifactLaneDocument(lane *ArtifactLaneDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) { normalizeDocumentBinding(&lane.Extract, path.Key("extract"), resolution, state) if lane.Merge != nil { normalizeDocumentBinding(lane.Merge, path.Key("merge"), resolution, state) @@ -217,7 +220,7 @@ func normalizeArtifactLaneDocument(lane *ArtifactLaneDocument, path configkit.Pa } } -func normalizeDocumentBinding(binding *BindingDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) { +func normalizeDocumentBinding(binding *BindingDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) { normalizeDocumentValue(resolution, path.Key("module"), &binding.Module, strings.TrimSpace(binding.Module), "trimmed module identifier") normalizePipelineDocumentString(resolution, path.Key("llm_profile"), &binding.LLMProfile, "trimmed binding LLM profile") binding.References = normalizeDocumentReferences(binding.References, path.Key("references"), resolution, state) @@ -226,12 +229,12 @@ func normalizeDocumentBinding(binding *BindingDocument, path configkit.Path, res } } -func normalizeValidatorDocument(validator *ValidatorDocument, path configkit.Path, resolution *configkit.Resolution) { +func normalizeValidatorDocument(validator *ValidatorDocument, path configkit.Path, resolution documentNormalizer) { normalizeDocumentValue(resolution, path.Key("module"), &validator.Module, strings.TrimSpace(validator.Module), "trimmed validator module identifier") normalizePipelineDocumentString(resolution, path.Key("llm_profile"), &validator.LLMProfile, "trimmed validator LLM profile") } -func normalizeDocumentReferences(references map[string]ReferenceDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) map[string]ReferenceDocument { +func normalizeDocumentReferences(references map[string]ReferenceDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) map[string]ReferenceDocument { for _, name := range sortedReferenceDocumentKeys(references) { source := references[name] sourcePath := path.Key(name) @@ -245,7 +248,7 @@ func normalizeDocumentReferences(references map[string]ReferenceDocument, path c return normalizeDocumentMap(references, path, "reference name", resolution, state) } -func normalizeDocumentValue[T comparable](resolution *configkit.Resolution, path configkit.Path, target *T, value T, reason string) { +func normalizeDocumentValue[T comparable](resolution documentNormalizer, path configkit.Path, target *T, value T, reason string) { if target == nil || !resolution.IsExplicit(path) { return } @@ -255,7 +258,7 @@ func normalizeDocumentValue[T comparable](resolution *configkit.Resolution, path } } -func normalizePipelineDocumentString(resolution *configkit.Resolution, path configkit.Path, target **string, reason string) { +func normalizePipelineDocumentString(resolution documentNormalizer, path configkit.Path, target **string, reason string) { if target == nil || *target == nil || !resolution.IsExplicit(path) { return } @@ -266,7 +269,7 @@ func normalizePipelineDocumentString(resolution *configkit.Resolution, path conf } } -func normalizeDocumentMap[T any](values map[string]T, path configkit.Path, kind string, resolution *configkit.Resolution, state *documentLoadState) map[string]T { +func normalizeDocumentMap[T any](values map[string]T, path configkit.Path, kind string, resolution documentNormalizer, state *documentLoadState) map[string]T { if len(values) == 0 { return values }