Compose pipeline steps using native identities

This commit is contained in:
2026-11-17 20:48:05 +00:00
parent 2d1fa57d9b
commit 61ceab786c
8 changed files with 249 additions and 27 deletions

View File

@@ -228,6 +228,12 @@ Prompt size: one resolution/provenance boundary.
## Stage 4 — Register Steps And Exercise Native Collection Semantics
**Complete.** Registered step IDs and verified real-file peer composition,
upsert, replacement, append, clearing validation, inherited records, CLI list
assignment, source-relative references, duplicate peers, and unregistered lists.
Focused config and framework resolution tests pass. Identity-changing
normalization records the affected list, while unchanged identities retain leaf history.
Goal: implement the accepted step identity and composition policy.
Inspect documentDefinition, pipeline step validation/conversion, ConfigKit's

View File

@@ -0,0 +1,180 @@
package config
import (
"os"
"path/filepath"
"testing"
configkit "go.maximumdirect.net/configkit"
)
func TestStepCompositionAcrossSourcesAndOverlays(t *testing.T) {
root := t.TempDir()
write := func(name, data string) {
t.Helper()
path := filepath.Join(root, name)
if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, []byte(data), 0600); err != nil {
t.Fatal(err)
}
}
write("notarius.yml", `version: 5
config:
imports:
- path: one/pipeline.yml
required: true
- path: two/pipeline.yml
required: true
`)
write("one/pipeline.yml", `pipelines:
session:
input: {module: input}
steps:
- id: first
references: {source: {path: ./first.txt}}
artifacts: {first: {extract: {module: extract, retries: 4}}}
`)
write("two/pipeline.yml", `pipelines:
session:
steps:
- id: second
references: {source: {path: ./second.txt}}
artifacts: {second: {extract: {module: extract}}}
`)
write("overlays/upsert.yml", `pipelines:
session:
steps:
- id: first
artifacts: {replacement: {extract: {module: extract}}}
`)
write("overlays/replace.yml", `config:
sequences:
- destination: [pipelines, session, steps]
mode: replace
pipelines:
session:
steps:
- id: third
artifacts: {third: {extract: {module: extract}}}
`)
write("overlays/append.yml", `config:
sequences:
- destination: [pipelines, session, steps]
mode: append
pipelines:
session:
steps:
- id: third
artifacts: {third: {extract: {module: extract}}}
`)
write("overlays/clear.yml", `config:
sequences:
- destination: [pipelines, session, steps]
mode: replace
pipelines:
session: {steps: []}
`)
main := filepath.Join(root, "notarius.yml")
loaded, err := Load(main, LoadOptions{})
if err != nil {
t.Fatal(err)
}
steps := loaded.Config.Pipelines["session"].Steps
if len(steps) != 2 || steps[0].ID != "first" || steps[1].ID != "second" || steps[0].References["source"].Path != filepath.Join(root, "one", "first.txt") || steps[1].References["source"].Path != filepath.Join(root, "two", "second.txt") {
t.Fatalf("composed steps = %#v", steps)
}
for _, tc := range []struct {
overlay string
ids []string
}{{"upsert", []string{"first", "second"}}, {"replace", []string{"third"}}, {"append", []string{"first", "second", "third"}}} {
t.Run(tc.overlay, func(t *testing.T) {
got, err := Load(main, LoadOptions{Overlay: tc.overlay})
if err != nil {
t.Fatal(err)
}
steps := got.Config.Pipelines["session"].Steps
if len(steps) != len(tc.ids) {
t.Fatalf("steps = %#v", steps)
}
for i, id := range tc.ids {
if steps[i].ID != id {
t.Fatalf("steps = %#v", steps)
}
}
if tc.overlay == "upsert" && (len(steps[0].References) != 0 || steps[0].Artifacts["replacement"].Extract.Module != "extract") {
t.Fatalf("upsert did not replace record: %#v", steps[0])
}
})
}
if _, err := Load(main, LoadOptions{Overlay: "clear"}); err == nil {
t.Fatal("domain validation accepted empty steps")
}
overrides := []configkit.Override{{Path: configkit.Keys("pipelines", "session", "steps"), Value: []any{map[string]any{"id": "override", "artifacts": map[string]any{"result": map[string]any{"extract": map[string]any{"module": "extract"}}}}}, Flag: "--steps"}}
replaced, err := Load(main, LoadOptions{Overrides: overrides})
if err != nil {
t.Fatal(err)
}
if got := replaced.Config.Pipelines["session"].Steps; len(got) != 1 || got[0].ID != "override" {
t.Fatalf("override steps = %#v", got)
}
}
func TestStepIdentityRejectsDuplicatePeersWithoutRegisteringOtherLists(t *testing.T) {
for _, tc := range []struct {
name, first, second string
}{
{"duplicate steps", "steps: [{id: same, artifacts: {a: {extract: {module: extract}}}}]", "steps: [{id: same, artifacts: {b: {extract: {module: extract}}}}]"},
{"unregistered validators", "input: {module: input, validators: [{module: one}]}", "input: {validators: [{module: two}]}"},
} {
t.Run(tc.name, func(t *testing.T) {
root := t.TempDir()
for name, data := range map[string]string{
"main.yml": "version: 5\nconfig:\n imports:\n - path: one.yml\n - path: two.yml\n",
"one.yml": "pipelines:\n session:\n " + tc.first + "\n",
"two.yml": "pipelines:\n session:\n " + tc.second + "\n",
} {
if err := os.WriteFile(filepath.Join(root, name), []byte(data), 0600); err != nil {
t.Fatal(err)
}
}
if _, err := Load(filepath.Join(root, "main.yml"), LoadOptions{}); err == nil {
t.Fatal("conflicting peer lists were accepted")
}
})
}
}
func TestStepRecordInheritanceReplacesMatchingRecord(t *testing.T) {
root := t.TempDir()
main := filepath.Join(root, "main.yml")
if err := os.WriteFile(main, []byte(`version: 5
pipelines:
parent:
input: {module: input}
steps:
- id: first
artifacts: {old: {extract: {module: extract, retries: 4}}}
- id: second
artifacts: {second: {extract: {module: extract}}}
child:
extends: parent
steps:
- id: first
artifacts: {new: {extract: {module: extract}}}
`), 0600); err != nil {
t.Fatal(err)
}
loaded, err := Load(main, LoadOptions{})
if err != nil {
t.Fatal(err)
}
steps := loaded.Config.Pipelines["child"].Steps
if len(steps) != 2 || steps[0].ID != "first" || steps[1].ID != "second" || len(steps[0].Artifacts) != 1 || steps[0].Artifacts["new"].Extract.Module != "extract" {
t.Fatalf("inherited steps = %#v", steps)
}
if _, err := loaded.Config.Resolve(ResolveInput{PipelineID: "child", Catalog: effectiveCatalog(t)}); err != nil {
t.Fatalf("resolve composed pipeline: %v", err)
}
}

View File

@@ -0,0 +1,16 @@
package config
import configkit "go.maximumdirect.net/configkit"
// A step whose identity is normalized no longer corresponds to its original
// record. Its ancestor record covers its edits instead of invalid leaf records.
type documentNormalizer struct {
*configkit.Resolution
covered bool
}
func (n documentNormalizer) RecordNormalization(path configkit.Path, reason string, dependencies ...configkit.Path) {
if !n.covered {
n.Resolution.RecordNormalization(path, reason, dependencies...)
}
}

View File

@@ -8,6 +8,7 @@ import (
)
func normalizeDocumentPaths(document *Document, resolution *configkit.Resolution, state *documentLoadState) {
normalizer := documentNormalizer{Resolution: resolution}
if document == nil {
return
}
@@ -33,24 +34,29 @@ func normalizeDocumentPaths(document *Document, resolution *configkit.Resolution
for _, id := range sortedPipelineDocumentKeys(document.Pipelines) {
profile := document.Pipelines[id]
path := configkit.Keys("pipelines").Key(id)
normalizeDocumentReferencePaths(profile.References, path.Key("references"), resolution, state)
normalizeDocumentBindingReferencePaths(&profile.Input, path.Key("input"), resolution, state)
normalizeDocumentReferencePaths(profile.References, path.Key("references"), normalizer, state)
normalizeDocumentBindingReferencePaths(&profile.Input, path.Key("input"), normalizer, state)
if profile.Chunk != nil {
normalizeDocumentBindingReferencePaths(profile.Chunk, path.Key("chunk"), resolution, state)
normalizeDocumentBindingReferencePaths(profile.Chunk, path.Key("chunk"), normalizer, state)
}
if profile.Output != nil {
normalizeDocumentBindingReferencePaths(profile.Output, path.Key("output"), resolution, state)
normalizeDocumentBindingReferencePaths(profile.Output, path.Key("output"), normalizer, state)
}
for laneID, lane := range profile.Artifacts {
normalizeDocumentLaneReferencePaths(&lane, path.Key("artifacts").Key(laneID), resolution, state)
normalizeDocumentLaneReferencePaths(&lane, path.Key("artifacts").Key(laneID), normalizer, state)
profile.Artifacts[laneID] = lane
}
for index := range profile.Steps {
step := &profile.Steps[index]
stepPath := path.Key("steps").Index(index)
normalizeDocumentReferencePaths(step.References, stepPath.Key("references"), resolution, state)
stepNormalizer := normalizer
if step.ID != strings.TrimSpace(step.ID) {
resolution.RecordNormalization(path.Key("steps"), "normalized step identity")
stepNormalizer.covered = true
}
normalizeDocumentReferencePaths(step.References, stepPath.Key("references"), stepNormalizer, state)
for laneID, lane := range step.Artifacts {
normalizeDocumentLaneReferencePaths(&lane, stepPath.Key("artifacts").Key(laneID), resolution, state)
normalizeDocumentLaneReferencePaths(&lane, stepPath.Key("artifacts").Key(laneID), stepNormalizer, state)
step.Artifacts[laneID] = lane
}
}
@@ -75,13 +81,13 @@ func initializeDocumentMainDirectory(resolution *configkit.Resolution, state *do
state.mainDirectory = cleanInvocationDirectory(".")
}
func normalizeDocumentBindingReferencePaths(binding *BindingDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) {
func normalizeDocumentBindingReferencePaths(binding *BindingDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) {
if binding != nil {
normalizeDocumentReferencePaths(binding.References, path.Key("references"), resolution, state)
}
}
func normalizeDocumentLaneReferencePaths(lane *ArtifactLaneDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) {
func normalizeDocumentLaneReferencePaths(lane *ArtifactLaneDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) {
if lane == nil {
return
}
@@ -90,7 +96,7 @@ func normalizeDocumentLaneReferencePaths(lane *ArtifactLaneDocument, path config
normalizeDocumentBindingReferencePaths(lane.Normalize, path.Key("normalize"), resolution, state)
}
func normalizeDocumentReferencePaths(references map[string]ReferenceDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) {
func normalizeDocumentReferencePaths(references map[string]ReferenceDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) {
for _, name := range sortedReferenceDocumentKeys(references) {
source := references[name]
sourcePath := path.Key(name).Key("path")
@@ -102,7 +108,7 @@ func normalizeDocumentReferencePaths(references map[string]ReferenceDocument, pa
}
}
if source.Path != nil && *source.Path != "" && !filepath.IsAbs(*source.Path) {
base, ok := documentPathBase(resolution, state, sourcePath)
base, ok := documentPathBase(resolution.Resolution, state, sourcePath)
if !ok {
resolution.Error(sourcePath, "relative path has no unambiguous source base")
} else {

View File

@@ -77,6 +77,10 @@ func TestDiscoveredMainAndStepReferenceNormalization(t *testing.T) {
artifacts:
result:
extract: {module: " extract "}
- id: second
artifacts:
next:
extract: {module: " extract "}
`), 0600); err != nil {
t.Fatal(err)
}
@@ -91,7 +95,7 @@ func TestDiscoveredMainAndStepReferenceNormalization(t *testing.T) {
if step.ID != "first" || step.Artifacts["result"].Extract.Module != "extract" || step.References["source"].Path != filepath.Join(fragmentDir, "source.txt") {
t.Fatalf("normalized step = %#v", step)
}
history := loaded.result.Metadata().History(configkit.Keys("pipelines", "session", "steps").Index(0).Key("artifacts").Key("result").Key("extract").Key("module"))
history := loaded.result.Metadata().History(configkit.Keys("pipelines", "session", "steps").Index(1).Key("artifacts").Key("next").Key("extract").Key("module"))
if len(history) < 2 || history[len(history)-1].Origin.Operation != configkit.ChangeNormalization {
t.Fatalf("module leaf history = %#v", history)
}

View File

@@ -58,6 +58,13 @@ func documentDefinition(state *documentLoadState) configkit.Definition[Document]
Check: validateDocumentVersion,
},
Sensitive: sensitiveDocumentPath,
SequenceIdentities: []configkit.SequenceIdentity{
configkit.SequenceIdentityMatching(configkit.SequenceSelector{
configkit.SequenceLiteralKey("pipelines"),
configkit.SequenceAnyMappingKey(),
configkit.SequenceLiteralKey("steps"),
}, "id"),
},
}
}

View File

@@ -21,7 +21,7 @@ func resolveDocumentOperations(document *Document, resolution *configkit.Resolut
if document.Concurrency != nil {
document.Concurrency.StageWorkers = normalizeDocumentMap(document.Concurrency.StageWorkers,
configkit.Keys("concurrency", "stage_workers"), "stage worker identifier",
resolution, state)
documentNormalizer{Resolution: resolution}, state)
}
deriveDocumentExtractWorkers(document, resolution, state)
return nil

View File

@@ -64,13 +64,14 @@ func normalizeDocumentPipelines(document *Document, resolution *configkit.Resolu
return
}
validateDocumentPipelinePresence(document, resolution)
normalizer := documentNormalizer{Resolution: resolution}
pipelinesPath := configkit.Keys("pipelines")
for _, id := range sortedPipelineDocumentKeys(document.Pipelines) {
profile := document.Pipelines[id]
normalizePipelineDocument(&profile, pipelinesPath.Key(id), resolution, state)
normalizePipelineDocument(&profile, pipelinesPath.Key(id), normalizer, state)
document.Pipelines[id] = profile
}
document.Pipelines = normalizeDocumentMap(document.Pipelines, pipelinesPath, "pipeline identifier", resolution, state)
document.Pipelines = normalizeDocumentMap(document.Pipelines, pipelinesPath, "pipeline identifier", normalizer, state)
}
func validateDocumentPipelinePresence(document *Document, resolution *configkit.Resolution) {
@@ -174,7 +175,7 @@ func validateDocumentPolicyPresence(policy *ValidationPolicyDocument, path confi
rejectDocumentNull(resolution, path.Key("validator_failure"), policy.ValidatorFailure == nil)
}
func normalizePipelineDocument(profile *PipelineDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) {
func normalizePipelineDocument(profile *PipelineDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) {
normalizePipelineDocumentString(resolution, path.Key("llm_profile"), &profile.LLMProfile, "trimmed pipeline LLM profile")
normalizeDocumentBinding(&profile.Input, path.Key("input"), resolution, state)
if profile.Chunk != nil {
@@ -193,18 +194,20 @@ func normalizePipelineDocument(profile *PipelineDocument, path configkit.Path, r
for index := range profile.Steps {
step := &profile.Steps[index]
stepPath := path.Key("steps").Index(index)
normalizeDocumentValue(resolution, stepPath.Key("id"), &step.ID, strings.TrimSpace(step.ID), "trimmed step identifier")
step.References = normalizeDocumentReferences(step.References, stepPath.Key("references"), resolution, state)
stepNormalizer := resolution
stepNormalizer.covered = step.ID != strings.TrimSpace(step.ID)
normalizeDocumentValue(stepNormalizer, stepPath.Key("id"), &step.ID, strings.TrimSpace(step.ID), "trimmed step identifier")
step.References = normalizeDocumentReferences(step.References, stepPath.Key("references"), stepNormalizer, state)
for _, laneID := range sortedLaneDocumentKeys(step.Artifacts) {
lane := step.Artifacts[laneID]
normalizeArtifactLaneDocument(&lane, stepPath.Key("artifacts").Key(laneID), resolution, state)
normalizeArtifactLaneDocument(&lane, stepPath.Key("artifacts").Key(laneID), stepNormalizer, state)
step.Artifacts[laneID] = lane
}
step.Artifacts = normalizeDocumentMap(step.Artifacts, stepPath.Key("artifacts"), "artifact lane identifier", resolution, state)
step.Artifacts = normalizeDocumentMap(step.Artifacts, stepPath.Key("artifacts"), "artifact lane identifier", stepNormalizer, state)
}
}
func normalizeArtifactLaneDocument(lane *ArtifactLaneDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) {
func normalizeArtifactLaneDocument(lane *ArtifactLaneDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) {
normalizeDocumentBinding(&lane.Extract, path.Key("extract"), resolution, state)
if lane.Merge != nil {
normalizeDocumentBinding(lane.Merge, path.Key("merge"), resolution, state)
@@ -217,7 +220,7 @@ func normalizeArtifactLaneDocument(lane *ArtifactLaneDocument, path configkit.Pa
}
}
func normalizeDocumentBinding(binding *BindingDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) {
func normalizeDocumentBinding(binding *BindingDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) {
normalizeDocumentValue(resolution, path.Key("module"), &binding.Module, strings.TrimSpace(binding.Module), "trimmed module identifier")
normalizePipelineDocumentString(resolution, path.Key("llm_profile"), &binding.LLMProfile, "trimmed binding LLM profile")
binding.References = normalizeDocumentReferences(binding.References, path.Key("references"), resolution, state)
@@ -226,12 +229,12 @@ func normalizeDocumentBinding(binding *BindingDocument, path configkit.Path, res
}
}
func normalizeValidatorDocument(validator *ValidatorDocument, path configkit.Path, resolution *configkit.Resolution) {
func normalizeValidatorDocument(validator *ValidatorDocument, path configkit.Path, resolution documentNormalizer) {
normalizeDocumentValue(resolution, path.Key("module"), &validator.Module, strings.TrimSpace(validator.Module), "trimmed validator module identifier")
normalizePipelineDocumentString(resolution, path.Key("llm_profile"), &validator.LLMProfile, "trimmed validator LLM profile")
}
func normalizeDocumentReferences(references map[string]ReferenceDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) map[string]ReferenceDocument {
func normalizeDocumentReferences(references map[string]ReferenceDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) map[string]ReferenceDocument {
for _, name := range sortedReferenceDocumentKeys(references) {
source := references[name]
sourcePath := path.Key(name)
@@ -245,7 +248,7 @@ func normalizeDocumentReferences(references map[string]ReferenceDocument, path c
return normalizeDocumentMap(references, path, "reference name", resolution, state)
}
func normalizeDocumentValue[T comparable](resolution *configkit.Resolution, path configkit.Path, target *T, value T, reason string) {
func normalizeDocumentValue[T comparable](resolution documentNormalizer, path configkit.Path, target *T, value T, reason string) {
if target == nil || !resolution.IsExplicit(path) {
return
}
@@ -255,7 +258,7 @@ func normalizeDocumentValue[T comparable](resolution *configkit.Resolution, path
}
}
func normalizePipelineDocumentString(resolution *configkit.Resolution, path configkit.Path, target **string, reason string) {
func normalizePipelineDocumentString(resolution documentNormalizer, path configkit.Path, target **string, reason string) {
if target == nil || *target == nil || !resolution.IsExplicit(path) {
return
}
@@ -266,7 +269,7 @@ func normalizePipelineDocumentString(resolution *configkit.Resolution, path conf
}
}
func normalizeDocumentMap[T any](values map[string]T, path configkit.Path, kind string, resolution *configkit.Resolution, state *documentLoadState) map[string]T {
func normalizeDocumentMap[T any](values map[string]T, path configkit.Path, kind string, resolution documentNormalizer, state *documentLoadState) map[string]T {
if len(values) == 0 {
return values
}