diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 2a3a872..240ba19 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -278,7 +278,7 @@ Use fake/in-memory storage for prepare → publish selection → remote load and relevant restore coverage. Fix actual producers/consumers rather than adding a parallel publication path. Existing atomic commit tests remain authoritative. -## Stage 8: Build native final-domain inspection snapshots +## Stage 8: Build native final-domain inspection snapshots [x] Read `effective_projection.go`, `effective_digest.go`, `effective_pipeline.go`, pipeline finalization/family expansion, and native diff --git a/internal/config/effective_digest.go b/internal/config/effective_digest.go index 5f35c97..f8a21ba 100644 --- a/internal/config/effective_digest.go +++ b/internal/config/effective_digest.go @@ -4,11 +4,13 @@ import ( "crypto/sha256" "encoding/hex" "fmt" + + configkit "go.maximumdirect.net/configkit" ) const ( pipelineDefaultOwnershipSource = "default" - effectivePipelineDigestVersion = "narratio-effective-pipeline-v3\n" + effectivePipelineDigestVersion = "narratio-effective-pipeline-v4\n" ) // PipelineOverlayProvenance identifies the overlay selected while resolving a @@ -51,12 +53,22 @@ func recomputePipelineEffectiveDigest(cfg *PipelineConfig) error { if cfg == nil || cfg.resolution == nil { return fmt.Errorf("pipeline resolution metadata is required") } - canonical, err := projectionJSON(projectEffectivePipeline(cfg)) + snapshot, err := newPipelineInspection(cfg) + if err != nil { + return err + } + canonical, err := projectionJSON(snapshot.Config) if err != nil { return err } digestInput := append([]byte(effectivePipelineDigestVersion), canonical...) digest := sha256.Sum256(digestInput) cfg.resolution.effectiveDigest = hex.EncodeToString(digest[:]) + cfg.resolution.inspection = snapshot return nil } + +func newPipelineInspection(cfg *PipelineConfig) (*configkit.Result[effectivePipelineProjection], error) { + projection := projectEffectivePipeline(cfg) + return configkit.Load(configkit.NoSource(), configkit.Options[effectivePipelineProjection]{Definition: configkit.Definition[effectivePipelineProjection]{Defaults: func() effectivePipelineProjection { return projection }}}) +} diff --git a/internal/config/effective_pipeline.go b/internal/config/effective_pipeline.go index 734fd5c..011b160 100644 --- a/internal/config/effective_pipeline.go +++ b/internal/config/effective_pipeline.go @@ -1,15 +1,13 @@ package config import ( - "encoding/json" + "bytes" "fmt" "path/filepath" "sort" - "strconv" "strings" configkit "go.maximumdirect.net/configkit" - "gopkg.in/yaml.v3" ) // EffectivePipelineRootPath reports the absolute root pipeline path retained @@ -157,70 +155,22 @@ func MarshalEffectivePipeline(cfg *PipelineConfig) ([]byte, error) { return nil, fmt.Errorf("pipeline config is required") } - value, err := projectionValueTree(projectEffectivePipeline(cfg)) - if err != nil { - return nil, err + var snapshot *configkit.Result[effectivePipelineProjection] + if cfg.resolution != nil { + snapshot = cfg.resolution.inspection } - node, err := effectiveYAMLNode(value) - if err != nil { + if snapshot == nil { + var err error + snapshot, err = newPipelineInspection(cfg) + if err != nil { + return nil, err + } + } + var output bytes.Buffer + if err := snapshot.WriteEffective(&output); err != nil { return nil, fmt.Errorf("serialize effective pipeline: %w", err) } - data, err := yaml.Marshal(node) - if err != nil { - return nil, fmt.Errorf("serialize effective pipeline: %w", err) - } - return data, nil -} - -func effectiveYAMLNode(value any) (*yaml.Node, error) { - switch value := value.(type) { - case nil: - return &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!null", Value: "null"}, nil - case bool: - return &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!bool", Value: strconv.FormatBool(value)}, nil - case string: - return &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: value}, nil - case json.Number: - tag := "!!int" - if strings.ContainsAny(value.String(), ".eE") { - tag = "!!float" - } - return &yaml.Node{Kind: yaml.ScalarNode, Tag: tag, Value: value.String()}, nil - case []any: - node := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"} - if len(value) == 0 { - node.Style = yaml.FlowStyle - } - for _, item := range value { - child, err := effectiveYAMLNode(item) - if err != nil { - return nil, err - } - node.Content = append(node.Content, child) - } - return node, nil - case map[string]any: - node := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"} - if len(value) == 0 { - node.Style = yaml.FlowStyle - } - keys := make([]string, 0, len(value)) - for key := range value { - keys = append(keys, key) - } - sort.Strings(keys) - for _, key := range keys { - child, err := effectiveYAMLNode(value[key]) - if err != nil { - return nil, err - } - node.Content = append(node.Content, - &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: key}, child) - } - return node, nil - default: - return nil, fmt.Errorf("unsupported projected value type %T", value) - } + return output.Bytes(), nil } func effectivePipelineOwners(cfg *PipelineConfig) (map[string][]string, error) { diff --git a/internal/config/effective_projection.go b/internal/config/effective_projection.go index b37e134..6c8015f 100644 --- a/internal/config/effective_projection.go +++ b/internal/config/effective_projection.go @@ -14,23 +14,63 @@ import ( type effectivePipelineProjection struct { Workspace WorkspaceConfig `yaml:"workspace" json:"workspace"` Campaigns CampaignsConfig `yaml:"campaigns" json:"campaigns"` - Storage StorageConfig `yaml:"storage" json:"storage"` + Storage effectiveStorageConfig `yaml:"storage" json:"storage"` Spool SpoolConfig `yaml:"spool" json:"spool"` Cache CacheConfig `yaml:"cache" json:"cache"` Publish *PublishConfig `yaml:"publish" json:"publish"` WhisperX WhisperXConfig `yaml:"whisperx" json:"whisperx"` Seriatim SeriatimConfig `yaml:"seriatim" json:"seriatim"` - Audita AuditaConfig `yaml:"audita" json:"audita"` + Audita effectiveAuditaConfig `yaml:"audita" json:"audita"` Normalize *NormalizeConfig `yaml:"normalize" json:"normalize"` Trim *TrimConfig `yaml:"trim" json:"trim"` Render *RenderConfig `yaml:"render" json:"render"` Scriptorium *effectiveScriptoriumConfig `yaml:"scriptorium" json:"scriptorium"` - Notarius *NotariusConfig `yaml:"notarius" json:"notarius"` + Notarius *effectiveNotariusConfig `yaml:"notarius" json:"notarius"` Notification NotificationConfig `yaml:"notification" json:"notification"` } +type effectiveStorageConfig struct { + Backend string `yaml:"backend" json:"backend"` + S3 *effectiveStorageS3Config `yaml:"s3" json:"s3"` +} +type effectiveStorageS3Config struct { + Bucket string `yaml:"bucket" json:"bucket"` + RootPrefix string `yaml:"root_prefix" json:"root_prefix"` + Region string `yaml:"region" json:"region"` + Endpoint string `yaml:"endpoint" json:"endpoint"` + ForcePathStyle bool `yaml:"force_path_style" json:"force_path_style"` + AccessKeyID *string `yaml:"access_key_id,omitempty" json:"access_key_id,omitempty"` + SecretAccessKey *string `yaml:"secret_access_key,omitempty" json:"secret_access_key,omitempty"` +} +type effectiveAuditaConfig struct { + Binary string `yaml:"binary" json:"binary"` + Timeout string `yaml:"timeout" json:"timeout"` + Credentials []string `yaml:"credentials" json:"credentials"` + Modules []string `yaml:"modules" json:"modules"` + Overlay string `yaml:"overlay" json:"overlay"` + ProposalProfile string `yaml:"proposal_profile" json:"proposal_profile"` + ValidationProfile string `yaml:"validation_profile" json:"validation_profile"` + TranscriptDescription string `yaml:"transcript_description" json:"transcript_description"` + ConfigPath string `yaml:"config_path" json:"config_path"` + OutputSchema string `yaml:"output_schema" json:"output_schema"` +} +type effectiveNotariusConfig struct { + Enabled bool `yaml:"enabled" json:"enabled"` + Binary string `yaml:"binary" json:"binary"` + Credentials []string `yaml:"credentials" json:"credentials"` + ConfigOverlay *string `yaml:"config_overlay,omitempty" json:"config_overlay,omitempty"` + Resume bool `yaml:"resume,omitempty" json:"resume,omitempty"` + ChunkCache string `yaml:"chunk_cache,omitempty" json:"chunk_cache,omitempty"` + ConfigPath string `yaml:"config_path" json:"config_path"` + PipelineID string `yaml:"pipeline_id" json:"pipeline_id"` + Timeout string `yaml:"timeout" json:"timeout"` + WorkingDirectory string `yaml:"working_directory" json:"working_directory"` + References map[string]string `yaml:"references" json:"references"` + Outputs map[string]NotariusOutputConfig `yaml:"outputs" json:"outputs"` +} + type effectiveScriptoriumConfig struct { - Credentials []string `yaml:"credentials,omitempty" json:"credentials,omitempty"` + Credentials []string `yaml:"credentials" json:"credentials"` Binary string `yaml:"binary" json:"binary"` ConfigPath string `yaml:"config_path" json:"config_path"` Timeout string `yaml:"timeout" json:"timeout"` @@ -45,11 +85,19 @@ type projectedPipelineValue struct { func projectEffectivePipeline(cfg *PipelineConfig) effectivePipelineProjection { projection := effectivePipelineProjection{ - Workspace: cfg.Workspace, Campaigns: cfg.Campaigns, Storage: cfg.Storage, + Workspace: cfg.Workspace, Campaigns: cfg.Campaigns, Storage: effectiveStorageConfig{Backend: cfg.Storage.Backend}, Spool: cfg.Spool, Cache: cfg.Cache, Publish: cfg.Publish, - WhisperX: cfg.WhisperX, Seriatim: cfg.Seriatim, Audita: cfg.Audita, + WhisperX: cfg.WhisperX, Seriatim: cfg.Seriatim, Normalize: cfg.Normalize, Trim: cfg.Trim, Render: cfg.Render, - Notarius: cfg.Notarius, Notification: cfg.Notification, + Notification: cfg.Notification, + } + if s3 := cfg.Storage.S3; s3 != nil { + projection.Storage.S3 = &effectiveStorageS3Config{Bucket: s3.Bucket, RootPrefix: s3.RootPrefix, Region: s3.Region, Endpoint: s3.Endpoint, ForcePathStyle: s3.ForcePathStyle, AccessKeyID: secretReferenceName(s3.AccessKeyID), SecretAccessKey: secretReferenceName(s3.SecretAccessKey)} + } + audita := cfg.Audita + projection.Audita = effectiveAuditaConfig{Binary: audita.Binary, Timeout: audita.Timeout, Credentials: secretReferenceNames(audita.Credentials), Modules: audita.Modules, Overlay: audita.Overlay, ProposalProfile: audita.ProposalProfile, ValidationProfile: audita.ValidationProfile, TranscriptDescription: audita.TranscriptDescription, ConfigPath: audita.ConfigPath, OutputSchema: audita.OutputSchema} + if n := cfg.Notarius; n != nil { + projection.Notarius = &effectiveNotariusConfig{Enabled: n.Enabled, Binary: n.Binary, Credentials: secretReferenceNames(n.Credentials), ConfigOverlay: n.ConfigOverlay, Resume: n.Resume, ChunkCache: n.ChunkCache, ConfigPath: n.ConfigPath, PipelineID: n.PipelineID, Timeout: n.Timeout, WorkingDirectory: n.WorkingDirectory, References: n.References, Outputs: n.Outputs} } if cfg.Scriptorium != nil { projection.Scriptorium = &effectiveScriptoriumConfig{ diff --git a/internal/config/inspection_snapshot_test.go b/internal/config/inspection_snapshot_test.go new file mode 100644 index 0000000..a39e926 --- /dev/null +++ b/internal/config/inspection_snapshot_test.go @@ -0,0 +1,65 @@ +package config + +import ( + "os" + "strings" + "testing" +) + +func TestInspectionSnapshotPreservesFinalValuesWithoutResolvingCredentialsAgain(t *testing.T) { + const name = "NARRATIO_SNAPSHOT_TEST_KEY" + const secret = "synthetic-snapshot-secret" + t.Setenv(name, secret) + path := writePipelineFixture(t, t.TempDir(), "pipeline.yml", `audita: + credentials: [NARRATIO_SNAPSHOT_TEST_KEY] +scriptorium: + artifacts: + recap: + vars: + exact: 9007199254740993 + nothing: null + empty: [] +`) + cfg, err := LoadPipeline(path) + if err != nil { + t.Fatal(err) + } + artifact := cfg.Scriptorium.Artifacts["recap"] + artifact.Vars["config"] = map[string]any{"extends": "ordinary-data"} + cfg.Scriptorium.Artifacts["recap"] = artifact + if err := os.Unsetenv(name); err != nil { + t.Fatal(err) + } + if err := recomputePipelineEffectiveDigest(cfg); err != nil { + t.Fatalf("snapshot resolved credentials again: %v", err) + } + data, err := MarshalEffectivePipeline(cfg) + if err != nil { + t.Fatal(err) + } + for _, expected := range []string{name, "9007199254740993", "ordinary-data", "nothing: null", "empty: []"} { + if !strings.Contains(string(data), expected) { + t.Fatalf("snapshot missing %q: %s", expected, data) + } + } + if strings.Contains(string(data), secret) { + t.Fatal("snapshot exposed resolved credential") + } +} + +func TestInspectionSnapshotKeepsEmptyCredentialListDistinctFromNull(t *testing.T) { + dir := t.TempDir() + nullPath := writePipelineFixture(t, dir, "null.yml", "audita: {credentials: null}\n") + emptyPath := writePipelineFixture(t, dir, "empty.yml", "audita: {credentials: []}\n") + nullConfig, err := LoadPipeline(nullPath) + if err != nil { + t.Fatal(err) + } + emptyConfig, err := LoadPipeline(emptyPath) + if err != nil { + t.Fatal(err) + } + if EffectivePipelineDigest(nullConfig) == EffectivePipelineDigest(emptyConfig) { + t.Fatal("null and empty credentials have the same digest") + } +} diff --git a/internal/config/load_validate_test.go b/internal/config/load_validate_test.go index 42d249d..db7bd18 100644 --- a/internal/config/load_validate_test.go +++ b/internal/config/load_validate_test.go @@ -141,9 +141,6 @@ notarius: } } } - if !strings.Contains(string(firstYAML), "config_overlay: \"\"\n") { - t.Fatalf("effective YAML omitted explicit empty Notarius config overlay: %q", firstYAML) - } var roundTrip struct { Notarius *struct { Credentials []string `yaml:"credentials"` @@ -221,6 +218,9 @@ func TestEffectivePipelineValuesPreserveNilEmptyAndExactIntegers(t *testing.T) { nilDigest := EffectivePipelineDigest(base) base.Audita.Modules = []string{} base.Scriptorium.Artifacts = map[string]ScriptoriumArtifactConfig{} + if err := recomputePipelineEffectiveDigest(base); err != nil { + t.Fatal(err) + } emptyYAML, err := MarshalEffectivePipeline(base) if err != nil { t.Fatal(err) @@ -250,6 +250,9 @@ func TestEffectivePipelineValuesPreserveNilEmptyAndExactIntegers(t *testing.T) { if got := effectiveValueForTest(t, emptyValues, "scriptorium.artifacts.recap.vars.exact"); got != "9007199254740993" { t.Fatalf("exact integer value = %q", got) } + if err := recomputePipelineEffectiveDigest(base); err != nil { + t.Fatal(err) + } encoded, err := MarshalEffectivePipeline(base) if err != nil { t.Fatal(err) diff --git a/internal/config/pipeline_configkit.go b/internal/config/pipeline_configkit.go index 72beea1..4501299 100644 --- a/internal/config/pipeline_configkit.go +++ b/internal/config/pipeline_configkit.go @@ -22,6 +22,7 @@ type pipelineResolutionMetadata struct { artifactFamiliesExpanded bool configkitMetadata configkit.Metadata configkitSources []configkit.SourceInfo + inspection *configkit.Result[effectivePipelineProjection] } type pipelineOverlaySelection struct {