Preserve derived publication and preparation evidence
This commit is contained in:
@@ -44,9 +44,9 @@ func materializeCanonicalPartyInputs(
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("materialize canonical party: %w", err)
|
||||
}
|
||||
playersBytes, err := resolved.Canonical.PlayersYAML()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("render canonical players projection: %w", err)
|
||||
playersBytes := resolved.Players
|
||||
if playersBytes == nil {
|
||||
return nil, fmt.Errorf("derived players projection is required")
|
||||
}
|
||||
playersPath := filepath.Join(inputsDir, players.Filename)
|
||||
playersChecksum, err := writeBytesIfChanged(store, playersPath, playersBytes)
|
||||
|
||||
@@ -148,9 +148,9 @@ func currentPreparePartySourceInputs(env *Env) ([]prepareExpectedInput, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
playersBytes, err := env.Config.Party.Canonical.PlayersYAML()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
playersBytes := env.Config.Party.Players
|
||||
if playersBytes == nil {
|
||||
return nil, fmt.Errorf("derived players projection is unavailable")
|
||||
}
|
||||
return []prepareExpectedInput{
|
||||
{kind: party.ManifestKind, checksum: checksumPrepareBytes(partyBytes)},
|
||||
|
||||
@@ -139,7 +139,7 @@ func setCanonicalPrepareParty(t *testing.T, env *Env, raw []byte) {
|
||||
t.Fatalf("ParseParty() error = %v", err)
|
||||
}
|
||||
env.Config.Party = config.ResolvedParty{
|
||||
Source: config.PartySource{Source: "campaign_config"}, Canonical: document,
|
||||
Source: config.PartySource{Source: "campaign_config"}, Canonical: document, Players: projectedTestPlayers(t, document),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -178,7 +178,7 @@ characters:
|
||||
t.Fatalf("ParseParty() error = %v", err)
|
||||
}
|
||||
env.Config.Party = config.ResolvedParty{
|
||||
Canonical: document,
|
||||
Canonical: document, Players: projectedTestPlayers(t, document),
|
||||
Source: config.PartySource{
|
||||
Path: filepath.Join(filepath.Dir(env.Config.CampaignPath), "party.yml"),
|
||||
ConfigPath: env.Config.CampaignPath,
|
||||
@@ -890,7 +890,7 @@ inputs:
|
||||
SessionPath: sessionPath,
|
||||
CampaignPath: campaignPath,
|
||||
PipelinePath: pipelinePath,
|
||||
Party: config.ResolvedParty{Canonical: party, Source: config.PartySource{
|
||||
Party: config.ResolvedParty{Canonical: party, Players: projectedTestPlayers(t, party), Source: config.PartySource{
|
||||
Path: filepath.Join(cfgDir, "party.yml"), ConfigPath: campaignPath, Source: "campaign_config",
|
||||
}},
|
||||
Session: &config.SessionConfig{
|
||||
@@ -980,3 +980,12 @@ func mustReadFileEquals(t *testing.T, path, want string) {
|
||||
t.Fatalf("%q = %q, want %q", path, string(data), want)
|
||||
}
|
||||
}
|
||||
|
||||
func projectedTestPlayers(t *testing.T, party *config.CanonicalParty) []byte {
|
||||
t.Helper()
|
||||
data, err := party.PlayersYAML()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
@@ -133,7 +133,7 @@ func (publishStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
||||
}
|
||||
publishRules, err := resolvePublishRules(
|
||||
runtimeCatalog,
|
||||
env.Config.Pipeline.Publish.Outputs,
|
||||
env.Config.PublicationOutputs(),
|
||||
env.SelectedArtifactKeys,
|
||||
)
|
||||
if err != nil {
|
||||
|
||||
@@ -443,7 +443,8 @@ func assertNoPublishUploads(t *testing.T, fake *storage.FakeBackend, forbidden s
|
||||
|
||||
func TestPublishUsesCustomOutputRules(t *testing.T) {
|
||||
env, m, _ := publishFixture(t)
|
||||
env.Config.Pipeline.Publish.Outputs = []config.PublishOutputRule{
|
||||
env.Config.Pipeline.Publish.Outputs = []config.PublishOutputRule{{Source: "invalid-authoring-source"}}
|
||||
env.Config.ArtifactCatalog.PublishOutputs = []config.PublishOutputRule{
|
||||
{Source: "narratio.transcript.final_trimmed", Dest: "published/trimmed.json", Required: boolPtr(true)},
|
||||
{Source: "narratio.artifact.session_recap", Dest: "published/recap.md", Required: boolPtr(true)},
|
||||
}
|
||||
@@ -466,7 +467,7 @@ func TestPublishUploadsExplicitExtractionAndPreservesManifestMetadata(t *testing
|
||||
env, m, _ := publishFixture(t)
|
||||
lanePath := configurePublishExtractionFixture(t, env, m)
|
||||
env.SelectedArtifactKeys = []string{"session_recap"}
|
||||
env.Config.Pipeline.Publish.Outputs = []config.PublishOutputRule{
|
||||
env.Config.ArtifactCatalog.PublishOutputs = []config.PublishOutputRule{
|
||||
{Source: artifacts.ExtractionArtifactSourceID("encounters"), Dest: "artifacts/encounters.json", Required: boolPtr(true)},
|
||||
}
|
||||
|
||||
@@ -519,7 +520,7 @@ func TestPublishUploadsExplicitExtractionAndPreservesManifestMetadata(t *testing
|
||||
func TestPublishRequiredInvalidExtractionFails(t *testing.T) {
|
||||
env, m, _ := publishFixture(t)
|
||||
lanePath := configurePublishExtractionFixture(t, env, m)
|
||||
env.Config.Pipeline.Publish.Outputs = []config.PublishOutputRule{
|
||||
env.Config.ArtifactCatalog.PublishOutputs = []config.PublishOutputRule{
|
||||
{Source: artifacts.ExtractionArtifactSourceID("encounters"), Dest: "artifacts/encounters.json", Required: boolPtr(true)},
|
||||
}
|
||||
writeStageTestFile(t, lanePath, `{"tampered":true}`)
|
||||
@@ -537,7 +538,7 @@ func TestPublishDisabledExtractionConfigurationIsUnavailable(t *testing.T) {
|
||||
env, m, _ := publishFixture(t)
|
||||
configurePublishExtractionFixture(t, env, m)
|
||||
env.Config.Pipeline.Notarius.Enabled = false
|
||||
env.Config.Pipeline.Publish.Outputs = []config.PublishOutputRule{
|
||||
env.Config.ArtifactCatalog.PublishOutputs = []config.PublishOutputRule{
|
||||
{Source: artifacts.ExtractionArtifactSourceID("encounters"), Dest: "artifacts/encounters.json", Required: boolPtr(true)},
|
||||
}
|
||||
|
||||
@@ -550,7 +551,7 @@ func TestPublishDisabledExtractionConfigurationIsUnavailable(t *testing.T) {
|
||||
func TestPublishOptionalMissingExtractionIsSkipped(t *testing.T) {
|
||||
env, m, _ := publishFixture(t)
|
||||
lanePath := configurePublishExtractionFixture(t, env, m)
|
||||
env.Config.Pipeline.Publish.Outputs = []config.PublishOutputRule{
|
||||
env.Config.ArtifactCatalog.PublishOutputs = []config.PublishOutputRule{
|
||||
{Source: artifacts.ExtractionArtifactSourceID("encounters"), Dest: "artifacts/encounters.json", Required: boolPtr(false)},
|
||||
}
|
||||
if err := os.Remove(lanePath); err != nil {
|
||||
@@ -570,7 +571,7 @@ func TestPublishArtifactSelectionDoesNotFilterExtractionOutputs(t *testing.T) {
|
||||
env, m, _ := publishFixture(t)
|
||||
configurePublishExtractionFixture(t, env, m)
|
||||
env.SelectedArtifactKeys = []string{"session_recap"}
|
||||
env.Config.Pipeline.Publish.Outputs = []config.PublishOutputRule{
|
||||
env.Config.ArtifactCatalog.PublishOutputs = []config.PublishOutputRule{
|
||||
{Source: artifacts.ExtractionArtifactSourceID("encounters"), Dest: "artifacts/encounters.json", Required: boolPtr(true)},
|
||||
}
|
||||
|
||||
@@ -584,7 +585,7 @@ func TestPublishArtifactSelectionDoesNotFilterExtractionOutputs(t *testing.T) {
|
||||
|
||||
func TestPublishSkipsOptionalMissingOutput(t *testing.T) {
|
||||
env, m, _ := publishFixture(t)
|
||||
env.Config.Pipeline.Publish.Outputs = []config.PublishOutputRule{
|
||||
env.Config.ArtifactCatalog.PublishOutputs = []config.PublishOutputRule{
|
||||
{Source: "narratio.transcript.final_trimmed", Dest: "transcripts/final.trimmed.json", Required: boolPtr(true)},
|
||||
{Source: "narratio.transcript.base", Dest: "transcripts/base.json", Required: boolPtr(false)},
|
||||
}
|
||||
@@ -673,7 +674,7 @@ func TestPublishOmitsStaleConfiguredArtifactAndPublishesUnrelatedCurrentArtifact
|
||||
recap.OutputSize = 0
|
||||
m.Stages["analyze"].AnalyzeArtifacts["session_recap"] = recap
|
||||
env.SelectedArtifactKeys = []string{"session_recap", "player_handout"}
|
||||
env.Config.Pipeline.Publish.Outputs = []config.PublishOutputRule{
|
||||
env.Config.ArtifactCatalog.PublishOutputs = []config.PublishOutputRule{
|
||||
{Source: "narratio.artifact.session_recap", Dest: "artifacts/session_recap.md", Required: boolPtr(false)},
|
||||
{Source: "narratio.artifact.player_handout", Dest: "artifacts/player_handout.md", Required: boolPtr(true)},
|
||||
}
|
||||
@@ -817,7 +818,7 @@ func TestPublishCarriesLockedRequiredOutputAndCommits(t *testing.T) {
|
||||
|
||||
func TestPublishLockedRequiredMissingOutputSucceeds(t *testing.T) {
|
||||
env, m, _ := publishFixture(t)
|
||||
env.Config.Pipeline.Publish.Outputs = []config.PublishOutputRule{
|
||||
env.Config.ArtifactCatalog.PublishOutputs = []config.PublishOutputRule{
|
||||
{Source: "narratio.transcript.base", Dest: "transcripts/base.json", Required: boolPtr(true)},
|
||||
}
|
||||
seedPublishRemoteLock(t, env.ObjectStore.(*storage.FakeBackend), m.S3SessionPrefix, "narratio.transcript.base", "manual merge is locked")
|
||||
@@ -929,7 +930,7 @@ func TestPublishRejectsChangedCarriedObjectMetadata(t *testing.T) {
|
||||
|
||||
func TestPublishFailsWhenRequiredOutputMissing(t *testing.T) {
|
||||
env, m, _ := publishFixture(t)
|
||||
env.Config.Pipeline.Publish.Outputs = []config.PublishOutputRule{
|
||||
env.Config.ArtifactCatalog.PublishOutputs = []config.PublishOutputRule{
|
||||
{Source: "narratio.transcript.base", Dest: "transcripts/base.json", Required: boolPtr(true)},
|
||||
}
|
||||
|
||||
@@ -1085,7 +1086,7 @@ func TestPublishRejectsAmbiguousOrCollidingOutputMappings(t *testing.T) {
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
env, m, _ := publishFixture(t)
|
||||
env.Config.Pipeline.Publish.Outputs = test.outputs
|
||||
env.Config.ArtifactCatalog.PublishOutputs = test.outputs
|
||||
_, err := (publishStage{}).Run(context.Background(), env, m)
|
||||
if err == nil || !strings.Contains(err.Error(), "resolve publish output rules") {
|
||||
t.Fatalf("Run() error = %v, want invalid mapping", err)
|
||||
@@ -1105,7 +1106,7 @@ func TestPublishKeepsSameBasenameSourcesDistinct(t *testing.T) {
|
||||
Enabled: true, PromptID: "dnd.other_recap", OutputPath: "reports/session_recap.md",
|
||||
}
|
||||
setCurrentAnalyzeEvidence(t, m, "other_recap", "reports/session_recap.md", otherPath)
|
||||
env.Config.Pipeline.Publish.Outputs = []config.PublishOutputRule{
|
||||
env.Config.ArtifactCatalog.PublishOutputs = []config.PublishOutputRule{
|
||||
{Source: "narratio.artifact.session_recap", Dest: "published/first/session_recap.md", Required: boolPtr(true)},
|
||||
{Source: "narratio.artifact.other_recap", Dest: "published/second/session_recap.md", Required: boolPtr(true)},
|
||||
}
|
||||
@@ -1233,7 +1234,7 @@ func publishFixture(t *testing.T) (*Env, *manifest.Manifest, string) {
|
||||
},
|
||||
ObjectStore: &storage.FakeBackend{},
|
||||
}
|
||||
env.Config.ArtifactCatalog = &config.ArtifactCatalog{Artifacts: env.Config.Pipeline.Scriptorium.Artifacts}
|
||||
env.Config.ArtifactCatalog = &config.ArtifactCatalog{Artifacts: env.Config.Pipeline.Scriptorium.Artifacts, PublishOutputs: env.Config.Pipeline.Publish.Outputs}
|
||||
return env, m, runRoot
|
||||
}
|
||||
|
||||
|
||||
@@ -143,8 +143,8 @@ func (publishStage) SemanticConfigFingerprint(env *Env) (manifest.SemanticConfig
|
||||
}
|
||||
payload.Enabled = cfg.Enabled == nil || *cfg.Enabled
|
||||
payload.UploadRun = cfg.UploadRun == nil || *cfg.UploadRun
|
||||
configured, extractions := publishSemanticSources(pipeline)
|
||||
for _, rule := range cfg.Outputs {
|
||||
configured, extractions := publishSemanticSources(env.Config)
|
||||
for _, rule := range env.Config.PublicationOutputs() {
|
||||
source, err := artifactpolicy.ValidatePublishSourceWithExtractions(rule.Source, configured, extractions)
|
||||
if err != nil {
|
||||
return manifest.SemanticConfigFingerprint{}, fmt.Errorf("publish semantic source %q: %w", rule.Source, err)
|
||||
@@ -186,16 +186,16 @@ func (s placeholderStage) SemanticConfigFingerprint(env *Env) (manifest.Semantic
|
||||
return FingerprintSemanticConfig(notifySemanticConfigVersion, notifySemanticConfig{Mode: mode})
|
||||
}
|
||||
|
||||
func publishSemanticSources(pipeline *config.PipelineConfig) (map[string]string, map[string]struct{}) {
|
||||
func publishSemanticSources(cfg *config.Config) (map[string]string, map[string]struct{}) {
|
||||
configured := map[string]string{}
|
||||
if pipeline.Scriptorium != nil {
|
||||
for key, artifact := range pipeline.Scriptorium.Artifacts {
|
||||
if cfg.ArtifactCatalog != nil {
|
||||
for key, artifact := range cfg.ConcreteArtifacts() {
|
||||
configured[key] = artifact.OutputPath
|
||||
}
|
||||
}
|
||||
extractions := map[string]struct{}{}
|
||||
if pipeline.Notarius != nil {
|
||||
for key := range pipeline.Notarius.Outputs {
|
||||
if cfg.Pipeline.Notarius != nil {
|
||||
for key := range cfg.Pipeline.Notarius.Outputs {
|
||||
extractions[key] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,10 +6,10 @@ import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
configkit "go.maximumdirect.net/configkit"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
configkit "go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
func testSecretReference(name string) *configkit.Secret {
|
||||
@@ -257,9 +257,11 @@ func TestPublishSemanticConfigSensitivityAndOperationalExclusions(t *testing.T)
|
||||
}{
|
||||
{name: "enablement", mutate: func(env *Env) { env.Config.Pipeline.Publish.Enabled = boolPointer(false) }},
|
||||
{name: "run upload", mutate: func(env *Env) { env.Config.Pipeline.Publish.UploadRun = boolPointer(false) }},
|
||||
{name: "source", mutate: func(env *Env) { env.Config.Pipeline.Publish.Outputs[0].Source = "narratio.transcript.final_markdown" }},
|
||||
{name: "destination", mutate: func(env *Env) { env.Config.Pipeline.Publish.Outputs[0].Dest = "published/other.json" }},
|
||||
{name: "required", mutate: func(env *Env) { env.Config.Pipeline.Publish.Outputs[0].Required = boolPointer(false) }},
|
||||
{name: "source", mutate: func(env *Env) {
|
||||
env.Config.ArtifactCatalog.PublishOutputs[0].Source = "narratio.transcript.final_markdown"
|
||||
}},
|
||||
{name: "destination", mutate: func(env *Env) { env.Config.ArtifactCatalog.PublishOutputs[0].Dest = "published/other.json" }},
|
||||
{name: "required", mutate: func(env *Env) { env.Config.ArtifactCatalog.PublishOutputs[0].Required = boolPointer(false) }},
|
||||
{name: "backend", mutate: func(env *Env) { env.Config.Pipeline.Storage.Backend = config.StorageBackendLocal }},
|
||||
{name: "bucket", mutate: func(env *Env) { env.Config.Pipeline.Storage.S3.Bucket = "other-bucket" }},
|
||||
{name: "region", mutate: func(env *Env) { env.Config.Pipeline.Storage.S3.Region = "eu-west-1" }},
|
||||
@@ -296,8 +298,8 @@ func TestPublishSemanticConfigSensitivityAndOperationalExclusions(t *testing.T)
|
||||
func TestDeliverySemanticConfigCanonicalOrderingAndNotifyMode(t *testing.T) {
|
||||
first := downstreamSemanticEnv()
|
||||
second := downstreamSemanticEnv()
|
||||
second.Config.Pipeline.Publish.Outputs[0], second.Config.Pipeline.Publish.Outputs[1] =
|
||||
second.Config.Pipeline.Publish.Outputs[1], second.Config.Pipeline.Publish.Outputs[0]
|
||||
second.Config.ArtifactCatalog.PublishOutputs[0], second.Config.ArtifactCatalog.PublishOutputs[1] =
|
||||
second.Config.ArtifactCatalog.PublishOutputs[1], second.Config.ArtifactCatalog.PublishOutputs[0]
|
||||
firstPublish, err := (publishStage{}).SemanticConfigFingerprint(first)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -353,7 +355,7 @@ func TestCanonicalStageSemanticReuseCoverage(t *testing.T) {
|
||||
}
|
||||
|
||||
func downstreamSemanticEnv() *Env {
|
||||
return &Env{Config: &config.Config{
|
||||
env := &Env{Config: &config.Config{
|
||||
Pipeline: &config.PipelineConfig{
|
||||
Workspace: config.WorkspaceConfig{Root: "/workspace"},
|
||||
Storage: config.StorageConfig{Backend: config.StorageBackendS3, S3: &config.StorageS3Config{
|
||||
@@ -382,4 +384,6 @@ func downstreamSemanticEnv() *Env {
|
||||
},
|
||||
Session: &config.SessionConfig{SessionID: "2026-08-30", Campaign: "campaign"},
|
||||
}}
|
||||
env.Config.ArtifactCatalog = &config.ArtifactCatalog{PublishOutputs: env.Config.Pipeline.Publish.Outputs}
|
||||
return env
|
||||
}
|
||||
|
||||
@@ -119,7 +119,7 @@ func buildPrepareSemanticConfig(env *Env) (prepareSemanticConfig, error) {
|
||||
for _, requirement := range requirements {
|
||||
entry := preparePreviousRequirement{Name: strings.TrimSpace(requirement.Name), Required: requirement.Required}
|
||||
if cfg.Pipeline.Scriptorium != nil {
|
||||
entry.OutputPath = strings.TrimSpace(cfg.Pipeline.Scriptorium.Artifacts[requirement.Name].OutputPath)
|
||||
entry.OutputPath = strings.TrimSpace(cfg.ConcreteArtifacts()[requirement.Name].OutputPath)
|
||||
}
|
||||
payload.PreviousArtifacts = append(payload.PreviousArtifacts, entry)
|
||||
}
|
||||
|
||||
@@ -441,7 +441,7 @@ func setupTranscribeEnv(t *testing.T, audioFiles []string) (*Env, *manifest.Mani
|
||||
CampaignPath: campaignPath,
|
||||
SessionPath: sessionPath,
|
||||
Campaign: &config.CampaignConfig{CampaignID: "sample-campaign"},
|
||||
Party: config.ResolvedParty{Canonical: party, Source: config.PartySource{
|
||||
Party: config.ResolvedParty{Canonical: party, Players: projectedTestPlayers(t, party), Source: config.PartySource{
|
||||
Path: filepath.Join(cfgDir, "party.yml"), ConfigPath: campaignPath, Source: "campaign_config",
|
||||
}},
|
||||
Pipeline: &config.PipelineConfig{
|
||||
|
||||
Reference in New Issue
Block a user