From 428f4357343e0f8053fde9bf436c3fa768b3c896 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 4 Oct 2026 13:10:53 +0000 Subject: [PATCH] Accept composed ConfigKit artifact definitions --- docs/config.md | 7 +- docs/internal/configuration.md | 6 +- docs/roadmap/config.md | 18 +-- docs/roadmap/implementation.md | 2 +- internal/config/artifact_families.go | 3 - internal/config/artifact_families_test.go | 79 ++++++++++- internal/config/pipeline_configkit.go | 50 ------- internal/config/pipeline_configkit_test.go | 156 ++++++++++++++++++--- 8 files changed, 226 insertions(+), 95 deletions(-) diff --git a/docs/config.md b/docs/config.md index 8aafd6e..005fcb5 100644 --- a/docs/config.md +++ b/docs/config.md @@ -169,9 +169,10 @@ another `configkit` control section. Base merging is additive. A map may be extended by multiple files when every leaf is distinct, but a scalar, list, or map/list/scalar kind cannot be claimed more than once, even when the repeated values are identical. Conflict errors -name the full field path and every source that claimed it. Narratio additionally -rejects an artifact or artifact-family key declared by more than one base file, -even when those files contribute disjoint children. +name the full field path and every source that claimed it. These rules apply at +every map path, including artifact and artifact-family entries, so distinct +fields of one entry can compose across imports. Narratio validates the resulting +effective entry using its ordinary domain rules. Overlay names are case-sensitive and use ConfigKit's safe fixed-path naming rules. Only the selected overlay is opened; a malformed unselected overlay does diff --git a/docs/internal/configuration.md b/docs/internal/configuration.md index ba2f1b7..6ca73a2 100644 --- a/docs/internal/configuration.md +++ b/docs/internal/configuration.md @@ -20,9 +20,9 @@ pipeline-loading boundary implemented by `internal/config`. 4. The definition validator performs read-only checks that require only the pipeline file. Checks requiring canonical-party family expansion remain at the existing explicit pipeline/campaign resolution boundary. -5. Narratio inspects public ConfigKit metadata to reject artifact and family keys - declared by multiple base files, adapts source ownership records, attaches - runtime-only state, and computes the normalized effective digest. +5. Narratio retains public ConfigKit metadata for source ownership, attaches + runtime-only state, and computes the normalized effective digest. Artifact + and family validation applies to the composed effective entries. This ordering preserves monolithic configuration behavior. Moving a field to an imported fragment changes its source ownership, not its path base, default, diff --git a/docs/roadmap/config.md b/docs/roadmap/config.md index 2f90557..2185e1a 100644 --- a/docs/roadmap/config.md +++ b/docs/roadmap/config.md @@ -38,25 +38,25 @@ The [configuration reference](../config.md) owns Narratio's current public semantics; [configuration internals](../internal/configuration.md) describe loading and inspection. -- Pipeline loading uses ConfigKit with explicit additive imports and one +- Pipeline loading uses ConfigKit v0.5.0 with explicit additive imports and one selected overlay, typed defaults, recorded resolution, and domain validation. - Campaign, session, and canonical-party inputs use separate single-source loads. Secret-directory bootstrap remains an explicit invocation operation. - Fields with root-relative pipeline path semantics retain that base when their declarations move into imports. For example, enabled Notarius configuration and working-directory paths resolve from the root pipeline directory. -- Narratio adds a declaration-ownership check: each artifact or artifact-family - key may be declared by only one base file. Overlay modifications are allowed. +- Distinct fields of one artifact or artifact-family entry can compose across + base files under ConfigKit's structural rules. Narratio validates the resulting + effective entry, and the selected overlay may replace or remove those values. - Source inspection and artifact catalogs consume retained ConfigKit metadata. Effective show, diff, and digest projections describe the resolved domain configuration rather than source layout. -The review selected v0.5.0 through a temporary module file and ran Narratio's -existing test suite successfully. An additional probe exposed an integration -gap: a child artifact inheriting from a parent in another imported file is -rejected as declared by multiple base files. The same definitions in one file -load successfully. The check currently counts inherited history contributions -as if they were direct declarations. +The released v0.5.0 module is now selected directly, and its attribution points +to the reviewed commit. Cross-file artifact and family inheritance and disjoint +entry composition now pass through Narratio's effective configuration checks. +The remaining integration work verifies native provenance, explicit presence, +safe projections, and the assembled inspection workflow described below. ## Targeted End State diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 93194e7..644a573 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -100,7 +100,7 @@ Complete when the released dependency and notices agree and current focused tests pass. The cross-file inheritance defect is addressed in the next stage; this stage does not claim that new inheritance behavior is fully integrated. -## Stage 2: Remove source-authoring acceptance gates +## Stage 2: Remove source-authoring acceptance gates [x] Purpose: accept structurally composed configuration from ConfigKit and reject only invalid effective Narratio domain configuration. diff --git a/internal/config/artifact_families.go b/internal/config/artifact_families.go index de67ca0..4ea7f54 100644 --- a/internal/config/artifact_families.go +++ b/internal/config/artifact_families.go @@ -169,9 +169,6 @@ func expandFamilyPublishRules(cfg *PipelineConfig, catalog ArtifactFamilyCatalog if family.Publish == nil || !family.Publish.Enabled { continue } - if cfg.resolution == nil || !cfg.resolution.publishDeclared { - return fmt.Errorf("pipeline.scriptorium.artifact_families.%s.publish requires pipeline.publish", familyKey) - } pattern := strings.TrimSpace(family.Publish.DestPattern) if pattern != "" { if err := validateFamilyOutputPattern("pipeline.scriptorium.artifact_families."+familyKey+".publish.dest_pattern", pattern); err != nil { diff --git a/internal/config/artifact_families_test.go b/internal/config/artifact_families_test.go index 4910198..dee1ecf 100644 --- a/internal/config/artifact_families_test.go +++ b/internal/config/artifact_families_test.go @@ -312,8 +312,7 @@ func TestFamilyPublishRulesExpandToConcreteArtifacts(t *testing.T) { "character_meta_alpha": {OutputPath: "artifacts/characters/alpha/meta.md"}, "character_meta_zeta": {OutputPath: "artifacts/characters/zeta/meta.md"}, }}, - Publish: &PublishConfig{}, - resolution: &pipelineResolutionMetadata{publishDeclared: true}, + Publish: &PublishConfig{}, } catalog := ArtifactFamilyCatalog{Families: map[string]ArtifactFamilyOrigin{ "character_meta": {Members: []string{"character_meta_alpha", "character_meta_zeta"}, Publish: &ScriptoriumArtifactFamilyPublishConfig{Enabled: true, Required: required, DestPattern: "published/{character_id}.md"}}, @@ -328,12 +327,86 @@ func TestFamilyPublishRulesExpandToConcreteArtifacts(t *testing.T) { } cfg.Publish.Outputs = nil catalog.Families["character_meta"] = ArtifactFamilyOrigin{Members: []string{"character_meta_alpha"}, Publish: &ScriptoriumArtifactFamilyPublishConfig{Enabled: true}} - cfg.resolution.publishDeclared = false + cfg.Publish = nil if err := expandFamilyPublishRules(cfg, catalog); err == nil || !strings.Contains(err.Error(), "requires pipeline.publish") { t.Fatalf("missing publish error = %v", err) } } +func TestArtifactFamilyInheritanceAcrossImports(t *testing.T) { + dir := t.TempDir() + writePartyResolutionFile(t, filepath.Join(dir, "party.yml"), `schema_version: narratio.party.v1 +characters: + alice: + player: {name: Alice Player} + character: + name: Alice + classes: [{name: wizard}] +`) + pipeline := `configkit: + imports: + - path: families/summary.yml + - path: families/notes.yml +whisperx: + transcribe_url: https://transcription.example.test/transcribe +` + writePipelineFixture(t, dir, "families/summary.yml", `scriptorium: + artifact_families: + character_summary: + enabled: true + for_each: party.characters + prompt_id: dnd.character_summary + profile_id: production + output_path_pattern: artifacts/characters/{character_id}/summary.md + publish: + enabled: true + required: true + dest_pattern: published/characters/{character_id}/summary.md +`) + writePipelineFixture(t, dir, "families/notes.yml", `scriptorium: + artifact_families: + character_notes: + extends: character_summary + output_path_pattern: artifacts/characters/{character_id}/notes.md + publish: + dest_pattern: published/characters/{character_id}/notes.md +`) + pipelinePath, campaignPath, sessionPath := writeArtifactFamilyConfig(t, dir, pipeline) + cfg, err := LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, SessionLoadOptions{}) + if err != nil { + t.Fatal(err) + } + + for key, wantPath := range map[string]string{ + "character_summary_alice": "artifacts/characters/alice/summary.md", + "character_notes_alice": "artifacts/characters/alice/notes.md", + } { + artifact, ok := cfg.Pipeline.Scriptorium.Artifacts[key] + if !ok { + t.Fatalf("inherited family member %q is missing", key) + } + if !artifact.Enabled || artifact.PromptID != "dnd.character_summary" || artifact.ProfileID != "production" || artifact.OutputPath != wantPath { + t.Fatalf("expanded family member %q = %#v", key, artifact) + } + } + + wantPublish := map[string]string{ + "narratio.artifact.character_summary_alice": "published/characters/alice/summary.md", + "narratio.artifact.character_notes_alice": "published/characters/alice/notes.md", + } + for _, rule := range cfg.Pipeline.Publish.Outputs { + if want, ok := wantPublish[rule.Source]; ok { + if rule.Dest != want { + t.Errorf("family publish destination for %q = %q, want %q", rule.Source, rule.Dest, want) + } + delete(wantPublish, rule.Source) + } + } + if len(wantPublish) != 0 { + t.Fatalf("inherited family publish rules missing: %#v", wantPublish) + } +} + func TestExplicitFamilyMemberPublishDestinationResolution(t *testing.T) { dir := t.TempDir() writePartyResolutionFile(t, filepath.Join(dir, "party.yml"), `schema_version: narratio.party.v1 diff --git a/internal/config/pipeline_configkit.go b/internal/config/pipeline_configkit.go index b2832c4..02ec2ec 100644 --- a/internal/config/pipeline_configkit.go +++ b/internal/config/pipeline_configkit.go @@ -6,7 +6,6 @@ import ( "fmt" "path/filepath" "reflect" - "sort" "strings" configkit "gitea.maximumdirect.net/eric/ConfigKit" @@ -24,7 +23,6 @@ type pipelineResolutionMetadata struct { ownership []pipelineFieldOwnership artifactFamilies map[string]ScriptoriumArtifactFamilyConfig artifactFamiliesExpanded bool - publishDeclared bool configkitMetadata configkit.Metadata configkitSources []configkit.SourceInfo } @@ -262,12 +260,8 @@ func validatePipelineConfigKit(cfg *PipelineConfig, validation *configkit.Valida func pipelineMetadataFromConfigKit(cfg *PipelineConfig, result *configkit.Result[PipelineConfig], opts PipelineLoadOptions) (*pipelineResolutionMetadata, error) { sources := result.Sources() metadata := result.Metadata() - if err := rejectDuplicateBaseDefinitions(metadata, sources); err != nil { - return nil, err - } resolved := &pipelineResolutionMetadata{ configkitMetadata: metadata, configkitSources: sources, - publishDeclared: metadata.IsExplicit(configkit.Keys("publish")), } pathsByID := make(map[string]string, len(sources)) for index, source := range sources { @@ -341,50 +335,6 @@ func ownershipSources(origin configkit.Origin, pathsByID map[string]string) []st return sources } -func rejectDuplicateBaseDefinitions(metadata configkit.Metadata, sources []configkit.SourceInfo) error { - pathsByID := make(map[string]string, len(sources)) - for _, source := range sources { - pathsByID[source.ID] = source.Path - } - // Enumerate retained declarations, not just surviving effective keys. Loading - // the base separately would also resolve credentials removed by the overlay. - // ConfigKit returns canonical JSON Pointers in deterministic order. - for _, path := range metadata.HistoricalPaths() { - parts := strings.Split(path.String(), "/") - if len(parts) != 4 || parts[1] != "scriptorium" { - continue - } - var kind string - switch parts[2] { - case "artifacts": - kind = "artifact" - case "artifact_families": - kind = "artifact family" - default: - continue - } - seen := map[string]struct{}{} - for _, change := range metadata.History(path) { - if change.Origin.Layer != configkit.LayerBase { - continue - } - for _, location := range change.Origin.Locations { - seen[location.SourceID] = struct{}{} - } - } - if len(seen) > 1 { - claims := make([]string, 0, len(seen)) - for sourceID := range seen { - claims = append(claims, pathsByID[sourceID]) - } - sort.Strings(claims) - key := strings.NewReplacer("~1", "/", "~0", "~").Replace(parts[3]) - return fmt.Errorf("pipeline Scriptorium %s %q is declared by multiple base files: %s", kind, key, strings.Join(claims, ", ")) - } - } - return nil -} - func intPtr(value int) *int { return &value } func float64Ptr(value float64) *float64 { return &value } func boolPtr(value bool) *bool { return &value } diff --git a/internal/config/pipeline_configkit_test.go b/internal/config/pipeline_configkit_test.go index b25d7a5..f05d935 100644 --- a/internal/config/pipeline_configkit_test.go +++ b/internal/config/pipeline_configkit_test.go @@ -80,41 +80,151 @@ func TestPipelineOverlayDoesNotDiscoverLegacyDirectory(t *testing.T) { } } -func TestPipelineConfigKitRejectsDuplicateBaseDeclarations(t *testing.T) { +func TestPipelineConfigKitComposesDisjointDefinitionsAndHonorsOverlay(t *testing.T) { for _, test := range []struct { - name, first, second, clearedContainer string + name, first, second, replacement, removal string }{ - {name: "artifact", first: "scriptorium:\n artifacts:\n recap:\n prompt_id: recap\n", second: "scriptorium:\n artifacts:\n recap:\n output_path: artifacts/recap.md\n", clearedContainer: "scriptorium:\n artifacts: null\n"}, - {name: "artifact family", first: "scriptorium:\n artifact_families:\n recap:\n for_each: party.characters\n", second: "scriptorium:\n artifact_families:\n recap:\n output_path_pattern: artifacts/{character_id}.md\n", clearedContainer: "scriptorium:\n artifact_families: null\n"}, + { + name: "artifact", + first: "scriptorium:\n artifacts:\n recap:\n enabled: true\n prompt_id: dnd.recap\n", + second: "scriptorium:\n artifacts:\n recap:\n output_path: artifacts/recap.md\n", + replacement: "scriptorium:\n artifacts:\n recap:\n output_path: artifacts/replaced.md\n", + removal: "scriptorium:\n artifacts: null\n", + }, + { + name: "artifact family", + first: "scriptorium:\n artifact_families:\n recap:\n enabled: true\n for_each: party.characters\n prompt_id: dnd.recap\n profile_id: production\n", + second: "scriptorium:\n artifact_families:\n recap:\n output_path_pattern: artifacts/{character_id}.md\n", + replacement: "scriptorium:\n artifact_families:\n recap:\n output_path_pattern: artifacts/{character_id}_replaced.md\n", + removal: "scriptorium:\n artifact_families: null\n", + }, } { t.Run(test.name, func(t *testing.T) { dir := t.TempDir() - writePipelineFixture(t, dir, "pipeline.yml", `configkit: + writePartyResolutionFile(t, filepath.Join(dir, "party.yml"), `schema_version: narratio.party.v1 +characters: + alice: + player: {name: Alice Player} + character: + name: Alice + classes: [{name: wizard}] +`) + pipeline := `configkit: + imports: + - path: first.yml + - path: second.yml +whisperx: + transcribe_url: https://transcription.example.test/transcribe +` + pipelinePath, campaignPath, sessionPath := writeArtifactFamilyConfig(t, dir, pipeline) + writePipelineFixture(t, dir, "first.yml", test.first) + writePipelineFixture(t, dir, "second.yml", test.second) + writePipelineFixture(t, dir, "overlays/replacement.yml", test.replacement) + writePipelineFixture(t, dir, "overlays/removal.yml", test.removal) + + cfg, err := LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, SessionLoadOptions{}) + if err != nil { + t.Fatalf("load disjoint contributions: %v", err) + } + if test.name == "artifact" { + artifact := cfg.Pipeline.Scriptorium.Artifacts["recap"] + if !artifact.Enabled || artifact.PromptID != "dnd.recap" || artifact.OutputPath != "artifacts/recap.md" { + t.Fatalf("composed artifact = %#v", artifact) + } + } else { + artifact := cfg.Pipeline.Scriptorium.Artifacts["recap_alice"] + if !artifact.Enabled || artifact.PromptID != "dnd.recap" || artifact.OutputPath != "artifacts/alice.md" { + t.Fatalf("composed artifact family member = %#v", artifact) + } + } + + replacement := "replacement" + cfg, err = LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, SessionLoadOptions{Overlay: &replacement}) + if err != nil { + t.Fatalf("load replacement overlay: %v", err) + } + if test.name == "artifact" && cfg.Pipeline.Scriptorium.Artifacts["recap"].OutputPath != "artifacts/replaced.md" { + t.Fatalf("overlay artifact replacement = %#v", cfg.Pipeline.Scriptorium.Artifacts["recap"]) + } + if test.name == "artifact family" && cfg.Pipeline.Scriptorium.Artifacts["recap_alice"].OutputPath != "artifacts/alice_replaced.md" { + t.Fatalf("overlay family replacement = %#v", cfg.Pipeline.Scriptorium.Artifacts["recap_alice"]) + } + + removal := "removal" + cfg, err = LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, SessionLoadOptions{Overlay: &removal}) + if err != nil { + t.Fatalf("load removal overlay: %v", err) + } + if len(cfg.Pipeline.Scriptorium.Artifacts) != 0 { + t.Fatalf("overlay left artifacts after removal: %#v", cfg.Pipeline.Scriptorium.Artifacts) + } + }) + } + + t.Run("structural conflict remains a loader error", func(t *testing.T) { + dir := t.TempDir() + writePipelineFixture(t, dir, "pipeline.yml", `configkit: imports: - path: first.yml - path: second.yml whisperx: transcribe_url: https://transcription.example.test/transcribe `) - writePipelineFixture(t, dir, "first.yml", test.first) - writePipelineFixture(t, dir, "second.yml", test.second) - writePipelineFixture(t, dir, "overlays/clear-container.yml", test.clearedContainer) - writePipelineFixture(t, dir, "overlays/clear-scriptorium.yml", "scriptorium: null\n") + writePipelineFixture(t, dir, "first.yml", "workspace:\n root: /srv/first\n") + writePipelineFixture(t, dir, "second.yml", "workspace:\n root: /srv/second\n") + if _, err := LoadPipeline(filepath.Join(dir, "pipeline.yml")); err == nil || !strings.Contains(strings.ToLower(err.Error()), "conflict") { + t.Fatalf("structural conflict error = %v", err) + } + }) +} - for _, profile := range []string{"", "clear-container", "clear-scriptorium"} { - t.Run(profile, func(t *testing.T) { - options := PipelineLoadOptions{} - if profile != "" { - options.Overlay = &profile - } - _, err := LoadPipelineWithOptions(filepath.Join(dir, "pipeline.yml"), options) - if err == nil || !strings.Contains(err.Error(), "declared by multiple base files") || - !strings.Contains(err.Error(), "first.yml") || !strings.Contains(err.Error(), "second.yml") { - t.Fatalf("duplicate declaration error = %v", err) - } - }) - } - }) +func TestPipelineConfigKitInheritsArtifactAcrossImports(t *testing.T) { + dir := t.TempDir() + writePartyResolutionFile(t, filepath.Join(dir, "party.yml"), `schema_version: narratio.party.v1 +characters: + alice: + player: {name: Alice Player} + character: + name: Alice + classes: [{name: wizard}] +`) + pipeline := `configkit: + imports: + - path: artifacts/parent.yml + - path: artifacts/child.yml +whisperx: + transcribe_url: https://transcription.example.test/transcribe +` + pipelinePath, campaignPath, sessionPath := writeArtifactFamilyConfig(t, dir, pipeline) + writePipelineFixture(t, dir, "artifacts/parent.yml", `scriptorium: + artifacts: + session_recap: + enabled: true + prompt_id: dnd.session_recap + profile_id: production + output_path: artifacts/session_recap.md +`) + writePipelineFixture(t, dir, "artifacts/child.yml", `scriptorium: + artifacts: + session_notes: + extends: session_recap + output_path: artifacts/session_notes.md +`) + + cfg, err := LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, SessionLoadOptions{}) + if err != nil { + t.Fatal(err) + } + parent := cfg.Pipeline.Scriptorium.Artifacts["session_recap"] + child, ok := cfg.Pipeline.Scriptorium.Artifacts["session_notes"] + if !ok { + t.Fatal("inherited child artifact identity was not retained") + } + if parent.OutputPath != "artifacts/session_recap.md" || child.OutputPath != "artifacts/session_notes.md" { + t.Fatalf("artifact destinations = parent %q, child %q", parent.OutputPath, child.OutputPath) + } + if !child.Enabled || child.PromptID != "dnd.session_recap" || child.ProfileID != "production" { + t.Fatalf("inherited child artifact = %#v", child) } }