From 2d1fa57d9b4a90499558ad9ae0a61f6df0e051f2 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Tue, 17 Nov 2026 20:44:07 +0000 Subject: [PATCH] Preserve source paths during field normalization --- docs/roadmap/implementation.md | 5 +++ internal/core/config/document_normalizer.go | 26 ------------ internal/core/config/document_paths.go | 32 +++++++++------ internal/core/config/document_paths_test.go | 43 +++++++++++++++++++- internal/core/config/inheritance_test.go | 4 +- internal/core/config/operational_document.go | 4 +- internal/core/config/pipeline_document.go | 25 +++++------- 7 files changed, 80 insertions(+), 59 deletions(-) delete mode 100644 internal/core/config/document_normalizer.go diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 3691e640..0ac0a5ae 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -191,6 +191,11 @@ Prompt size: API/handoff migration; no discovery implementation or broad fixture ## Stage 3 — Simplify Recorded Normalization And Preserve Source Bases +**Complete.** Leaf edits are recorded directly and reference paths are resolved +before key normalization moves their source coordinates. Focused path, +inheritance, discovered-main, and leaf-history tests pass. The full config +suite's remaining override-order expectation is owned by fixture migration. + Goal: use v1.1 field-level auditing instead of old blanket sequence coverage. Inspect document_normalizer.go, document_paths.go, pipeline_document.go, diff --git a/internal/core/config/document_normalizer.go b/internal/core/config/document_normalizer.go deleted file mode 100644 index aa8bd4fe..00000000 --- a/internal/core/config/document_normalizer.go +++ /dev/null @@ -1,26 +0,0 @@ -package config - -import configkit "go.maximumdirect.net/configkit" - -// documentNormalizer records leaf provenance and, while visiting a sequence, -// covers its atomic mutation boundary. ConfigKit audits sequences as whole -// values, so a record for only a descendant is insufficient. The outermost -// sequence covers nested sequences as well; unchanged visits record nothing. -type documentNormalizer struct { - *configkit.Resolution - sequence *configkit.Path -} - -func (n documentNormalizer) inSequence(path configkit.Path) documentNormalizer { - if n.sequence == nil { - n.sequence = &path - } - return n -} - -func (n documentNormalizer) RecordNormalization(path configkit.Path, reason string, dependencies ...configkit.Path) { - n.Resolution.RecordNormalization(path, reason, dependencies...) - if n.sequence != nil { - n.Resolution.RecordNormalization(*n.sequence, reason, append([]configkit.Path{path}, dependencies...)...) - } -} diff --git a/internal/core/config/document_paths.go b/internal/core/config/document_paths.go index 63b095b6..55d97fcf 100644 --- a/internal/core/config/document_paths.go +++ b/internal/core/config/document_paths.go @@ -2,12 +2,12 @@ package config import ( "path/filepath" + "strings" configkit "go.maximumdirect.net/configkit" ) -func normalizeDocumentPaths(document *Document, context *configkit.Resolution, state *documentLoadState) { - resolution := documentNormalizer{Resolution: context} +func normalizeDocumentPaths(document *Document, resolution *configkit.Resolution, state *documentLoadState) { if document == nil { return } @@ -45,13 +45,12 @@ func normalizeDocumentPaths(document *Document, context *configkit.Resolution, s normalizeDocumentLaneReferencePaths(&lane, path.Key("artifacts").Key(laneID), resolution, state) profile.Artifacts[laneID] = lane } - stepResolution := resolution.inSequence(path.Key("steps")) for index := range profile.Steps { step := &profile.Steps[index] stepPath := path.Key("steps").Index(index) - normalizeDocumentReferencePaths(step.References, stepPath.Key("references"), stepResolution, state) + normalizeDocumentReferencePaths(step.References, stepPath.Key("references"), resolution, state) for laneID, lane := range step.Artifacts { - normalizeDocumentLaneReferencePaths(&lane, stepPath.Key("artifacts").Key(laneID), stepResolution, state) + normalizeDocumentLaneReferencePaths(&lane, stepPath.Key("artifacts").Key(laneID), resolution, state) step.Artifacts[laneID] = lane } } @@ -59,7 +58,7 @@ func normalizeDocumentPaths(document *Document, context *configkit.Resolution, s } } -func initializeDocumentMainDirectory(resolution documentNormalizer, state *documentLoadState) { +func initializeDocumentMainDirectory(resolution *configkit.Resolution, state *documentLoadState) { if state == nil || state.mainDirectory != "" { return } @@ -76,13 +75,13 @@ func initializeDocumentMainDirectory(resolution documentNormalizer, state *docum state.mainDirectory = cleanInvocationDirectory(".") } -func normalizeDocumentBindingReferencePaths(binding *BindingDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) { +func normalizeDocumentBindingReferencePaths(binding *BindingDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) { if binding != nil { normalizeDocumentReferencePaths(binding.References, path.Key("references"), resolution, state) } } -func normalizeDocumentLaneReferencePaths(lane *ArtifactLaneDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) { +func normalizeDocumentLaneReferencePaths(lane *ArtifactLaneDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) { if lane == nil { return } @@ -91,10 +90,17 @@ func normalizeDocumentLaneReferencePaths(lane *ArtifactLaneDocument, path config normalizeDocumentBindingReferencePaths(lane.Normalize, path.Key("normalize"), resolution, state) } -func normalizeDocumentReferencePaths(references map[string]ReferenceDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) { +func normalizeDocumentReferencePaths(references map[string]ReferenceDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) { for _, name := range sortedReferenceDocumentKeys(references) { source := references[name] sourcePath := path.Key(name).Key("path") + if source.Path != nil && resolution.IsExplicit(sourcePath) { + value := strings.TrimSpace(*source.Path) + if value != *source.Path { + resolution.RecordNormalization(sourcePath, "trimmed reference path") + source.Path = documentValue(value) + } + } if source.Path != nil && *source.Path != "" && !filepath.IsAbs(*source.Path) { base, ok := documentPathBase(resolution, state, sourcePath) if !ok { @@ -106,12 +112,12 @@ func normalizeDocumentReferencePaths(references map[string]ReferenceDocument, pa source.Path = documentValue(value) } } - references[name] = source } + references[name] = source } } -func normalizeDocumentPath(resolution documentNormalizer, state *documentLoadState, path configkit.Path, target **string, optional bool, reason string) { +func normalizeDocumentPath(resolution *configkit.Resolution, state *documentLoadState, path configkit.Path, target **string, optional bool, reason string) { if target == nil || *target == nil || **target == "" || filepath.IsAbs(**target) { return } @@ -125,11 +131,11 @@ func normalizeDocumentPath(resolution documentNormalizer, state *documentLoadSta return } if *target != nil && **target != value { - configkit.Normalize(resolution.Resolution, path, target, documentValue(value), reason) + configkit.Normalize(resolution, path, target, documentValue(value), reason) } } -func documentPathBase(resolution documentNormalizer, state *documentLoadState, path configkit.Path) (string, bool) { +func documentPathBase(resolution *configkit.Resolution, state *documentLoadState, path configkit.Path) (string, bool) { original := path if state != nil { original = state.originalPath(path) diff --git a/internal/core/config/document_paths_test.go b/internal/core/config/document_paths_test.go index af4d1ad6..cd041791 100644 --- a/internal/core/config/document_paths_test.go +++ b/internal/core/config/document_paths_test.go @@ -27,7 +27,7 @@ func TestDocumentPathsUseDeclaringFilesAcrossImports(t *testing.T) { t.Fatal(err) } if err := os.WriteFile(main, []byte(`version: 5 -configkit: +config: imports: - path: fragments/pipelines.yml output: {directory: ./output} @@ -56,6 +56,47 @@ pipelines: } } +func TestDiscoveredMainAndStepReferenceNormalization(t *testing.T) { + root := t.TempDir() + fragmentDir := filepath.Join(root, "fragments") + if err := os.Mkdir(fragmentDir, 0700); err != nil { + t.Fatal(err) + } + main := filepath.Join(root, "notarius.yml") + fragment := filepath.Join(fragmentDir, "pipeline.yml") + if err := os.WriteFile(main, []byte("version: 5\nconfig:\n imports:\n - path: fragments/pipeline.yml\n required: true\n"), 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(fragment, []byte(`pipelines: + session: + input: {module: input} + steps: + - id: " first " + references: + " source ": {path: " ./source.txt "} + artifacts: + result: + extract: {module: " extract "} +`), 0600); err != nil { + t.Fatal(err) + } + loaded, err := LoadSource(configkit.SearchConfig("", configkit.SearchPaths(main), configkit.SearchOptions{Required: true}), LoadOptions{}) + if err != nil { + t.Fatal(err) + } + if loaded.MainSourcePath() != main || loaded.Config.Output.Directory != filepath.Join(root, "notarius-output") { + t.Fatalf("discovered main or default directory: %q, %q", loaded.MainSourcePath(), loaded.Config.Output.Directory) + } + step := loaded.Config.Pipelines["session"].Steps[0] + if step.ID != "first" || step.Artifacts["result"].Extract.Module != "extract" || step.References["source"].Path != filepath.Join(fragmentDir, "source.txt") { + t.Fatalf("normalized step = %#v", step) + } + history := loaded.result.Metadata().History(configkit.Keys("pipelines", "session", "steps").Index(0).Key("artifacts").Key("result").Key("extract").Key("module")) + if len(history) < 2 || history[len(history)-1].Origin.Operation != configkit.ChangeNormalization { + t.Fatalf("module leaf history = %#v", history) + } +} + func TestDocumentPathsUseInvocationDirectoryForOverrides(t *testing.T) { t.Parallel() diff --git a/internal/core/config/inheritance_test.go b/internal/core/config/inheritance_test.go index 78e5509c..d7a758a7 100644 --- a/internal/core/config/inheritance_test.go +++ b/internal/core/config/inheritance_test.go @@ -391,7 +391,7 @@ func TestInheritedReferencePathsKeepDeclaringSource(t *testing.T) { } main := filepath.Join(root, "notarius.yml") if err := os.WriteFile(main, []byte(`version: 5 -configkit: +config: imports: - path: parents/pipelines.yml - path: children/pipelines.yml @@ -488,7 +488,7 @@ func TestInheritedInspectionRetainsOriginsRedactionAndOwnership(t *testing.T) { } main := filepath.Join(root, "notarius.yml") if err := os.WriteFile(main, []byte(`version: 5 -configkit: +config: imports: - path: parents/pipelines.yml - path: children/pipelines.yml diff --git a/internal/core/config/operational_document.go b/internal/core/config/operational_document.go index 592734c9..8c74466c 100644 --- a/internal/core/config/operational_document.go +++ b/internal/core/config/operational_document.go @@ -16,12 +16,12 @@ func resolveDocumentOperations(document *Document, resolution *configkit.Resolut } normalizeDocumentPromptKit(document.PromptKit, resolution) normalizeDocumentState(document, resolution) - normalizeDocumentPipelines(document, resolution, state) normalizeDocumentPaths(document, resolution, state) + normalizeDocumentPipelines(document, resolution, state) if document.Concurrency != nil { document.Concurrency.StageWorkers = normalizeDocumentMap(document.Concurrency.StageWorkers, configkit.Keys("concurrency", "stage_workers"), "stage worker identifier", - documentNormalizer{Resolution: resolution}, state) + resolution, state) } deriveDocumentExtractWorkers(document, resolution, state) return nil diff --git a/internal/core/config/pipeline_document.go b/internal/core/config/pipeline_document.go index ce3783cd..ff1a4dae 100644 --- a/internal/core/config/pipeline_document.go +++ b/internal/core/config/pipeline_document.go @@ -64,15 +64,13 @@ func normalizeDocumentPipelines(document *Document, resolution *configkit.Resolu return } validateDocumentPipelinePresence(document, resolution) - normalizer := documentNormalizer{Resolution: resolution} - pipelinesPath := configkit.Keys("pipelines") for _, id := range sortedPipelineDocumentKeys(document.Pipelines) { profile := document.Pipelines[id] - normalizePipelineDocument(&profile, pipelinesPath.Key(id), normalizer, state) + normalizePipelineDocument(&profile, pipelinesPath.Key(id), resolution, state) document.Pipelines[id] = profile } - document.Pipelines = normalizeDocumentMap(document.Pipelines, pipelinesPath, "pipeline identifier", normalizer, state) + document.Pipelines = normalizeDocumentMap(document.Pipelines, pipelinesPath, "pipeline identifier", resolution, state) } func validateDocumentPipelinePresence(document *Document, resolution *configkit.Resolution) { @@ -176,7 +174,7 @@ func validateDocumentPolicyPresence(policy *ValidationPolicyDocument, path confi rejectDocumentNull(resolution, path.Key("validator_failure"), policy.ValidatorFailure == nil) } -func normalizePipelineDocument(profile *PipelineDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) { +func normalizePipelineDocument(profile *PipelineDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) { normalizePipelineDocumentString(resolution, path.Key("llm_profile"), &profile.LLMProfile, "trimmed pipeline LLM profile") normalizeDocumentBinding(&profile.Input, path.Key("input"), resolution, state) if profile.Chunk != nil { @@ -192,7 +190,6 @@ func normalizePipelineDocument(profile *PipelineDocument, path configkit.Path, r profile.Artifacts[laneID] = lane } profile.Artifacts = normalizeDocumentMap(profile.Artifacts, path.Key("artifacts"), "artifact lane identifier", resolution, state) - resolution = resolution.inSequence(path.Key("steps")) for index := range profile.Steps { step := &profile.Steps[index] stepPath := path.Key("steps").Index(index) @@ -207,7 +204,7 @@ func normalizePipelineDocument(profile *PipelineDocument, path configkit.Path, r } } -func normalizeArtifactLaneDocument(lane *ArtifactLaneDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) { +func normalizeArtifactLaneDocument(lane *ArtifactLaneDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) { normalizeDocumentBinding(&lane.Extract, path.Key("extract"), resolution, state) if lane.Merge != nil { normalizeDocumentBinding(lane.Merge, path.Key("merge"), resolution, state) @@ -215,28 +212,26 @@ func normalizeArtifactLaneDocument(lane *ArtifactLaneDocument, path configkit.Pa if lane.Normalize != nil { normalizeDocumentBinding(lane.Normalize, path.Key("normalize"), resolution, state) } - resolution = resolution.inSequence(path.Key("validators")) for index := range lane.Validators { normalizeValidatorDocument(&lane.Validators[index], path.Key("validators").Index(index), resolution) } } -func normalizeDocumentBinding(binding *BindingDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) { +func normalizeDocumentBinding(binding *BindingDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) { normalizeDocumentValue(resolution, path.Key("module"), &binding.Module, strings.TrimSpace(binding.Module), "trimmed module identifier") normalizePipelineDocumentString(resolution, path.Key("llm_profile"), &binding.LLMProfile, "trimmed binding LLM profile") binding.References = normalizeDocumentReferences(binding.References, path.Key("references"), resolution, state) - resolution = resolution.inSequence(path.Key("validators")) for index := range binding.Validators { normalizeValidatorDocument(&binding.Validators[index], path.Key("validators").Index(index), resolution) } } -func normalizeValidatorDocument(validator *ValidatorDocument, path configkit.Path, resolution documentNormalizer) { +func normalizeValidatorDocument(validator *ValidatorDocument, path configkit.Path, resolution *configkit.Resolution) { normalizeDocumentValue(resolution, path.Key("module"), &validator.Module, strings.TrimSpace(validator.Module), "trimmed validator module identifier") normalizePipelineDocumentString(resolution, path.Key("llm_profile"), &validator.LLMProfile, "trimmed validator LLM profile") } -func normalizeDocumentReferences(references map[string]ReferenceDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) map[string]ReferenceDocument { +func normalizeDocumentReferences(references map[string]ReferenceDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) map[string]ReferenceDocument { for _, name := range sortedReferenceDocumentKeys(references) { source := references[name] sourcePath := path.Key(name) @@ -250,7 +245,7 @@ func normalizeDocumentReferences(references map[string]ReferenceDocument, path c return normalizeDocumentMap(references, path, "reference name", resolution, state) } -func normalizeDocumentValue[T comparable](resolution documentNormalizer, path configkit.Path, target *T, value T, reason string) { +func normalizeDocumentValue[T comparable](resolution *configkit.Resolution, path configkit.Path, target *T, value T, reason string) { if target == nil || !resolution.IsExplicit(path) { return } @@ -260,7 +255,7 @@ func normalizeDocumentValue[T comparable](resolution documentNormalizer, path co } } -func normalizePipelineDocumentString(resolution documentNormalizer, path configkit.Path, target **string, reason string) { +func normalizePipelineDocumentString(resolution *configkit.Resolution, path configkit.Path, target **string, reason string) { if target == nil || *target == nil || !resolution.IsExplicit(path) { return } @@ -271,7 +266,7 @@ func normalizePipelineDocumentString(resolution documentNormalizer, path configk } } -func normalizeDocumentMap[T any](values map[string]T, path configkit.Path, kind string, resolution documentNormalizer, state *documentLoadState) map[string]T { +func normalizeDocumentMap[T any](values map[string]T, path configkit.Path, kind string, resolution *configkit.Resolution, state *documentLoadState) map[string]T { if len(values) == 0 { return values }