package cli import ( "context" "crypto/sha256" "encoding/hex" "errors" "flag" "fmt" "io" "os" "path" "path/filepath" "sort" "strings" "time" "gitea.maximumdirect.net/eric/notarius/internal/buildinfo" "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" "gitea.maximumdirect.net/eric/notarius/internal/core/config" "gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle" "gitea.maximumdirect.net/eric/notarius/internal/core/fileio" "gitea.maximumdirect.net/eric/notarius/internal/framework/checkpoint" "gitea.maximumdirect.net/eric/notarius/internal/framework/chunkplan" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" frameworkdebug "gitea.maximumdirect.net/eric/notarius/internal/framework/debug" frameworkllm "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "go.maximumdirect.net/commandkit" configkit "go.maximumdirect.net/configkit" "go.maximumdirect.net/promptkit" ) type Options struct { LLMClient promptkit.LLMClient LLMRuntime promptkit.RuntimeConfig Catalog pipeline.ModuleCatalog Registries pipeline.Registries LLMClientFactory LLMClientFactory RunIDGenerator RunIDGenerator LookupEnv func(string) (string, bool) Now func() time.Time UserCacheDir func() (string, error) ChunkPlanStoreFactory pipeline.ChunkPlanStoreFactory DebugRecorderFactory func(string) (pipeline.DebugRecorder, error) DebugTerminalFactory func(*debugbundle.SummaryWriter) DebugTerminalWriter promptKitAssets *frameworkllm.AssetRegistry } type LLMRuntimeOverrides struct { ReasoningEffort *string } type LLMClientFactory func(ctx context.Context, cfg config.Config, profileID string, overrides LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) // Run executes the command-line interface and returns a process exit code. func Run(args []string, stdout, stderr io.Writer) int { return RunWithOptions(args, stdout, stderr, Options{}) } func RunWithOptions(args []string, stdout, stderr io.Writer, opts Options) int { version, err := buildinfo.Version() if err != nil { fmt.Fprintf(stderr, "notarius: %v\n", err) return 1 } app, err := NewApplication(version, opts) if err != nil { fmt.Fprintf(stderr, "notarius: %v\n", err) return 1 } return app.Execute(context.Background(), args, commandkit.Streams{Out: stdout, Err: stderr}).Status } func normalizeOptions(opts Options) (Options, error) { if opts.LookupEnv == nil { opts.LookupEnv = os.LookupEnv } if opts.Now == nil { opts.Now = time.Now } if opts.RunIDGenerator == nil { opts.RunIDGenerator = defaultRunIDGenerator } if opts.UserCacheDir == nil { opts.UserCacheDir = os.UserCacheDir } if opts.ChunkPlanStoreFactory == nil { opts.ChunkPlanStoreFactory = chunkplan.NewFilesystemStore } if opts.DebugRecorderFactory == nil { opts.DebugRecorderFactory = frameworkdebug.NewFilesystemRecorder } if opts.DebugTerminalFactory == nil { opts.DebugTerminalFactory = func(writer *debugbundle.SummaryWriter) DebugTerminalWriter { return writer } } if isEmptyCatalog(opts.Catalog) && isEmptyRegistries(opts.Registries) { components, err := newProductionComponents() if err != nil { return Options{}, err } opts.Registries = components.registries opts.Catalog = catalogFromRegistries(components.registries) opts.promptKitAssets = components.assets } if opts.LLMClientFactory == nil { if opts.promptKitAssets == nil { assets, err := productionPromptAssets() if err != nil { return Options{}, err } opts.promptKitAssets = assets } opts.LLMClientFactory = productionLLMClientFactoryWithAssets(opts.promptKitAssets) } return opts, nil } func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) int { fs := flag.NewFlagSet("run", flag.ContinueOnError) fs.SetOutput(io.Discard) configPath := fs.String("config", "", "config file path") configOverlay := fs.String("config-overlay", "", "configuration overlay") inputPath := fs.String("input", "", "source input file path") onlyRaw := fs.String("only", "", "comma-separated artifact lanes") outputDir := fs.String("output-dir", "", "output directory") machineOutput := fs.Bool("json", false, "write the successful run result as JSON") debug := fs.Bool("debug", false, "write a debug bundle") debugDir := fs.String("debug-dir", "", "debug bundle directory") llmProfile := singleValueFlag{name: "--llm-profile"} reasoningEffort := singleValueFlag{name: "--reasoning-effort"} clearReasoningEffort := fs.Bool("clear-reasoning-effort", false, "clear the LLM profile reasoning effort") resume := fs.Bool("resume", false, "reuse compatible recorded checkpoints") recomputeStep := singleValueFlag{name: "--recompute-step"} chunkCache := chunkCacheFlag{} requestedSessionID := sessionIDFlag{} referenceFlags := stringListFlag{} withoutReferenceFlags := stringListFlag{} fs.Var(&requestedSessionID, "session-id", "prompt session identifier") fs.Var(&llmProfile, "llm-profile", "LLM profile override") fs.Var(&reasoningEffort, "reasoning-effort", "reasoning effort override") fs.Var(&chunkCache, "chunk_cache", "chunk plan cache mode: auto, bypass, or refresh") fs.Var(&referenceFlags, "reference", "reference binding, as slot=path, chunk.slot=path, lane.slot=path, lane.extract.slot=path, lane.merge.slot=path, or lane.normalize.slot=path") fs.Var(&withoutReferenceFlags, "without-reference", "unbind a reference, using the same selector forms as --reference") fs.Var(&recomputeStep, "recompute-step", "recompute one ordered pipeline step and dependent lanes") if err := validateRunFlagValues(args); err != nil { fmt.Fprintf(stderr, "notarius: %v\n", err) return 2 } if err := fs.Parse(reorderRunArgs(args)); err != nil { fmt.Fprintf(stderr, "notarius: %v\n", err) return 2 } if fs.NArg() == 0 { fmt.Fprintln(stderr, "notarius: run requires a pipeline ID") return 2 } if fs.NArg() > 1 { fmt.Fprintf(stderr, "notarius: unexpected argument %q\n", fs.Arg(1)) return 2 } pipelineID := strings.TrimSpace(fs.Arg(0)) if pipelineID == "" { fmt.Fprintln(stderr, "notarius: run requires a pipeline ID") return 2 } if strings.TrimSpace(*inputPath) == "" { fmt.Fprintln(stderr, "notarius: run requires --input") return 2 } if strings.TrimSpace(*debugDir) != "" && !*debug { fmt.Fprintln(stderr, "notarius: --debug-dir requires --debug") return 2 } if strings.TrimSpace(*outputDir) == "" && flagWasProvided(args, "--output-dir") { fmt.Fprintln(stderr, "notarius: --output-dir must not be empty") return 2 } if strings.TrimSpace(*debugDir) == "" && flagWasProvided(args, "--debug-dir") { fmt.Fprintln(stderr, "notarius: --debug-dir must not be empty") return 2 } if requestedSessionID.set && strings.TrimSpace(requestedSessionID.value) == "" { fmt.Fprintln(stderr, "notarius: --session-id must not be empty") return 2 } if llmProfile.set && strings.TrimSpace(llmProfile.value) == "" { fmt.Fprintln(stderr, "notarius: --llm-profile must not be empty") return 2 } if reasoningEffort.set && *clearReasoningEffort { fmt.Fprintln(stderr, "notarius: --reasoning-effort cannot be combined with --clear-reasoning-effort") return 2 } if reasoningEffort.set && strings.TrimSpace(reasoningEffort.value) == "" { fmt.Fprintln(stderr, "notarius: --reasoning-effort must not be empty") return 2 } runtimeOverrides := LLMRuntimeOverrides{} if reasoningEffort.set { value := strings.TrimSpace(reasoningEffort.value) runtimeOverrides.ReasoningEffort = &value } else if *clearReasoningEffort { value := "" runtimeOverrides.ReasoningEffort = &value } only, err := parseOnly(*onlyRaw) if err != nil { fmt.Fprintf(stderr, "notarius: %v\n", err) return 2 } if recomputeStep.set && strings.TrimSpace(recomputeStep.value) == "" { fmt.Fprintln(stderr, "notarius: --recompute-step must not be empty") return 2 } if recomputeStep.set && len(only) > 0 { fmt.Fprintln(stderr, "notarius: --recompute-step cannot be combined with --only") return 2 } referenceRequests, err := parseReferenceFlags(referenceFlags) if err != nil { fmt.Fprintf(stderr, "notarius: %v\n", err) return 2 } referenceUnbindRequests, err := parseReferenceUnbindFlags(withoutReferenceFlags) if err != nil { fmt.Fprintf(stderr, "notarius: %v\n", err) return 2 } loaded, loadedConfigPath, err := loadConfigKit(*configPath, *configOverlay, runConfigurationOverrides(chunkCache, *outputDir, *debugDir), opts) if err != nil { writeConfigLoadError(stderr, err) return 1 } cfg := loaded.RuntimeConfig() if err := cfg.Validate(); err != nil { fmt.Fprintf(stderr, "notarius: %v\n", err) return 1 } if *resume && !cfg.Cache.Checkpoints.Enabled { fmt.Fprintln(stderr, "notarius: --resume requires cache.checkpoints.enabled: true") return 1 } if recomputeStep.set && !*resume { fmt.Fprintln(stderr, "notarius: --recompute-step requires --resume") return 2 } if recomputeStep.set && !cfg.Cache.Checkpoints.Enabled { fmt.Fprintln(stderr, "notarius: --recompute-step requires cache.checkpoints.enabled: true") return 1 } startedAt := opts.Now().UTC() runID, err := opts.RunIDGenerator(startedAt) if err != nil { fmt.Fprintf(stderr, "notarius: generate run ID: %v\n", err) return 1 } if err := validateRunID(runID); err != nil { fmt.Fprintf(stderr, "notarius: invalid generated run ID: %v\n", err) return 1 } runOutputDir := filepath.Join(cfg.Output.Directory, runID) commandState := newPipelineCommandState(runID, pipelineID, runOutputDir) var summary *debugbundle.SummaryWriter var terminalWriter DebugTerminalWriter debugPath := "" debugRecorder := pipeline.NoopDebugRecorder() if *debug { bundle, err := debugbundle.Allocate(cfg.Debug.Directory, runID, startedAt) if err != nil { fmt.Fprintf(stderr, "notarius: %v\n", err) return 1 } debugPath, summary = bundle.Path(), bundle.Summary() commandState.setDebugPath(debugPath) terminalWriter = opts.DebugTerminalFactory(summary) if terminalWriter == nil { terminalWriter = summary } debugRecorder, err = opts.DebugRecorderFactory(bundle.TraceRoot()) if err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("create debug recorder: %w", err)) } debugRecorder = pipeline.SynchronizedDebugRecorder(debugRecorder) } invocation := debugbundle.Invocation{ Operation: "run", PipelineID: pipelineID, InputPath: strings.TrimSpace(*inputPath), ConfigPath: loadedConfigPath, ConfigSource: configSource(*configPath), ConfigOverlay: effectiveConfigOverlay(*configOverlay), OnlyLanes: append([]string(nil), only...), ChunkCacheOverride: chunkCache.explicitValue(), ReasoningEffortOverride: runtimeOverrides.ReasoningEffort, Resume: *resume, RecomputeStep: strings.TrimSpace(recomputeStep.value), RunID: runID, StartedAt: startedAt, } if err := writeSummary(summary, func() error { return summary.WriteInvocation(invocation) }); err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug invocation metadata: %w", err)) } catalog, err := effectiveCatalog(opts) if err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, err) } referenceOverrides, referenceUnbinds, err := resolveCLIReferenceRequests(cfg, pipelineID, only, catalog, referenceRequests, referenceUnbindRequests) if err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, err) } effective, err := cfg.Resolve(config.ResolveInput{ PipelineID: pipelineID, Only: only, Catalog: catalog, LLMProfileOverride: strings.TrimSpace(llmProfile.value), ReferenceOverrides: referenceOverrides, ReferenceUnbinds: referenceUnbinds, }) if err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, err) } profileIDs := effectiveLLMProfileIDs(effective.ResolvedPipeline) if err := validateExplicitPromptKitProfiles(context.Background(), effective.Config, profileIDs, opts.promptKitAssets); err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, err) } workingDir, err := os.Getwd() if err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("resolve working directory: %w", err)) } materialized, referenceDiagnostics, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{ ConfigPath: loadedConfigPath, WorkingDir: workingDir, }) if err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, err) } effective.ResolvedPipeline = materialized checkpointPolicy := pipeline.CheckpointExecutionPolicy{} if recomputeStep.set { checkpointPolicy, err = recomputePolicy(effective.ResolvedPipeline, recomputeStep.value) if err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, err) } } invocation.PipelineDigest = effective.ResolvedPipeline.Digest if err := writeSummary(summary, func() error { return summary.WriteInvocation(invocation) }); err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug invocation metadata: %w", err)) } if err := writeSummary(summary, func() error { return summary.WriteEffectiveConfig(loaded) }); err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug effective config: %w", err)) } if err := writeSummary(summary, func() error { return summary.WriteResolvedPipeline(effective) }); err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug resolved pipeline: %w", err)) } if err := writeSummary(summary, func() error { return summary.WriteResolvedReferences(pipeline.ReferenceProvenance(effective.ResolvedPipeline)) }); err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug resolved references: %w", err)) } registries, err := effectiveRegistries(opts) if err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, err) } ctx := context.Background() factoryProfileID := "" if len(profileIDs) == 1 { factoryProfileID = profileIDs[0] } llmClient, llmProfiles, err := opts.LLMClientFactory(ctx, effective.Config, factoryProfileID, runtimeOverrides) if err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("create LLM client for profile %q: %w", factoryProfileID, err)) } var llmFingerprints []checkpoint.Fingerprint if effective.Config.Cache.Checkpoints.Enabled { llmFingerprints, err = llmCheckpointFingerprints(llmClient) if err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("prepare LLM checkpoint identity: %w", err)) } } llmClient = pipeline.WithDebugLLMRecording(llmClient, debugRecorder) prepared, err := pipeline.Prepare(effective.ResolvedPipeline, registries, pipeline.ModuleDependencies{LLM: llmClient}) if err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("prepare pipeline %q: %w", pipelineID, err)) } rawInput, err := os.ReadFile(strings.TrimSpace(*inputPath)) if err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("read input %q: %w", strings.TrimSpace(*inputPath), err)) } effectiveSessionID, err := resolvePromptSessionID(requestedSessionID.value, effective.ResolvedPipeline.Input.Module, rawInput) if err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, err) } invocation.SessionID = effectiveSessionID if err := writeSummary(summary, func() error { return summary.WriteInvocation(invocation) }); err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug invocation metadata: %w", err)) } chunkPlans, err := chunkPlanStoreForRun(effective.Config.Cache.ChunkPlans, opts) if err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, err) } checkpointRecorder, checkpointLoader, err := checkpointHandlersForRun(effective.Config.Cache.Checkpoints, opts, effective.ResolvedPipeline, prepared.CheckpointFingerprints(), llmFingerprints, rawInput, only, llmProfiles, strings.TrimSpace(llmProfile.value), effectiveSessionID, runtimeOverrides, *resume) if err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, err) } output, err := pipeline.New().Run(ctx, pipeline.RunInput{ Prepared: prepared, Path: strings.TrimSpace(*inputPath), RawInput: rawInput, SessionID: effectiveSessionID, RunID: runID, StartedAt: startedAt, LLMProfiles: llmProfiles, Metadata: runMetadata(effective.Config.Output.Directory, debugPath), Diagnostics: referenceDiagnostics, ChunkCacheMode: effective.Config.Cache.ChunkPlans.Mode, ChunkPlans: chunkPlans, Checkpoints: checkpointRecorder, Checkpoint: checkpointLoader, CheckpointPolicy: checkpointPolicy, Debug: debugRecorder, ExtractWorkers: cfg.Concurrency.StageWorkers["extract"], }) diagnosticProjection, diagnosticErr := contracts.ProjectDiagnosticCollection(output.Diagnostics) if diagnosticErr == nil { commandState.observeOutput(output, diagnosticProjection) } if err != nil { primaryErr := fmt.Errorf("run pipeline %q: %w", pipelineID, err) if output.Manifest.PipelineID != "" { if summaryErr := writePartialSummary(summary, output); summaryErr != nil { return failPipelineCommand(stderr, commandState, terminalWriter, primaryErr, fmt.Errorf("write debug summary: %w", summaryErr)) } } if diagnosticErr != nil { return failPipelineCommand(stderr, commandState, terminalWriter, primaryErr, fmt.Errorf("summarize run diagnostics: %w", diagnosticErr)) } return failPipelineCommand(stderr, commandState, terminalWriter, primaryErr) } if diagnosticErr != nil { return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("summarize run diagnostics: %w", diagnosticErr)) } if err := writePartialSummary(summary, output); err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug summary: %w", err)) } var encodedResult []byte if *machineOutput { result, err := newRunResultWithDiagnostics(effective.ResolvedPipeline, output, runOutputDir, debugPath, diagnosticProjection) if err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, err) } encodedResult, err = encodeRunResult(result) if err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, err) } } if err := writeOutputFiles(runOutputDir, output.OutputFiles); err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, err) } if primaryErr, persistenceErr := commandState.terminalize(terminalWriter, nil); primaryErr != nil { return writePipelineCommandFailure(stderr, commandState, primaryErr, persistenceErr) } if *machineOutput { if err := writeRunResult(stdout, encodedResult); err != nil { return writePipelineCommandFailure(stderr, commandState, errors.New("write run result"), nil) } } else { fmt.Fprintf(stdout, "pipeline %q complete: outputs=%d rejected=%d output=%s\n", effective.PipelineID, len(output.NormalizeOutputs), len(output.Rejected), runOutputDir) if debugPath != "" { fmt.Fprintf(stdout, "debug=%s\n", debugPath) } } if warningGroups := len(diagnosticProjection.Warnings); warningGroups > 0 { fmt.Fprintf(stderr, "notarius: run completed with %d warning group(s), %d occurrence(s)", warningGroups, diagnosticProjection.WarningOccurrenceCount) if warningFile, ok := logicalOutputFile(output.OutputFiles, "warnings.json"); ok { fmt.Fprintf(stderr, "; details=%s", filepath.Join(runOutputDir, warningFile)) } fmt.Fprintln(stderr) } return 0 } func logicalOutputFile(files []contracts.OutputFile, name string) (string, bool) { for _, file := range files { if file.Name == name { return file.Name, true } } return "", false } func writeSummary(summary *debugbundle.SummaryWriter, write func() error) error { if summary == nil { return nil } return write() } func writePartialSummary(summary *debugbundle.SummaryWriter, output pipeline.RunOutput) error { if summary == nil { return nil } if err := summary.WriteRunManifest(output.Manifest); err != nil { return err } if output.ChunkPlan != nil { if err := summary.WriteChunkPlan(*output.ChunkPlan); err != nil { return err } } if err := summary.WriteDiagnostics(output.Diagnostics); err != nil { return err } return summary.WriteCheckpointEvents(output.CheckpointEvents) } func checkpointHandlersForRun( settings config.CheckpointCacheConfig, opts Options, resolved pipeline.ResolvedPipeline, componentFingerprints []pipeline.CheckpointFingerprint, llmFingerprints []checkpoint.Fingerprint, rawInput []byte, only []string, llmProfiles []artifacts.LLMProfileManifest, llmProfileOverride string, sessionID string, runtimeOverrides LLMRuntimeOverrides, resume bool, ) (pipeline.CheckpointRecorder, pipeline.CheckpointLoader, error) { if !settings.Enabled { if resume { return nil, nil, fmt.Errorf("--resume requires cache.checkpoints.enabled: true") } return pipeline.NoopCheckpointRecorder(), pipeline.NoopCheckpointLoader(), nil } identity, err := checkpoint.NewIdentity(checkpoint.IdentityInput{ Pipeline: resolved, InputKey: resolved.Input.Module, RawInputDigest: rawInputDigest(rawInput), SelectedLanes: only, RuntimeOverrides: runtimeOverrideFingerprints(llmProfileOverride, sessionID, runtimeOverrides), References: pipeline.ReferenceProvenance(resolved), ProvenanceFingerprints: combineCheckpointFingerprints( llmProfileFingerprints(llmProfiles), llmFingerprints, checkpointIdentityFingerprints(componentFingerprints), ), }) if err != nil { return nil, nil, fmt.Errorf("create checkpoint identity: %w", err) } checkpointRoot := strings.TrimSpace(settings.Directory) if checkpointRoot == "" { checkpointRoot, err = config.DefaultCheckpointRoot(opts.UserCacheDir) if err != nil { return nil, nil, fmt.Errorf("resolve checkpoint root: %w", err) } } recorder, err := checkpoint.NewFilesystemRecorder(checkpointRoot, identity) if err != nil { return nil, nil, fmt.Errorf("create checkpoint recorder: %w", err) } loader := pipeline.NoopCheckpointLoader() if resume { loader, err = checkpoint.NewFilesystemLoader(checkpointRoot, identity) if err != nil { return nil, nil, fmt.Errorf("create checkpoint loader: %w", err) } } return recorder, loader, nil } func llmCheckpointFingerprints(client contracts.StructuredLLMClient) ([]checkpoint.Fingerprint, error) { provider, ok := client.(frameworkllm.CheckpointFingerprintProvider) if !ok { return nil, nil } values, err := provider.LLMCheckpointFingerprints() if err != nil { return nil, err } out := make([]checkpoint.Fingerprint, 0, len(values)) for _, value := range values { out = append(out, checkpoint.Fingerprint{Name: value.Name, Value: value.Value}) } return out, nil } func combineCheckpointFingerprints(sources ...[]checkpoint.Fingerprint) []checkpoint.Fingerprint { var out []checkpoint.Fingerprint for _, source := range sources { out = append(out, source...) } return out } func recomputePolicy(resolved pipeline.ResolvedPipeline, requestedStep string) (pipeline.CheckpointExecutionPolicy, error) { requestedStep = strings.TrimSpace(requestedStep) if requestedStep == "" { return pipeline.CheckpointExecutionPolicy{}, fmt.Errorf("--recompute-step must not be empty") } var selected *pipeline.ResolvedPipelineStep for index := range resolved.Steps { if strings.TrimSpace(resolved.Steps[index].ID) == requestedStep { selected = &resolved.Steps[index] break } } if selected == nil { return pipeline.CheckpointExecutionPolicy{}, fmt.Errorf("unknown pipeline step %q", requestedStep) } policy := pipeline.CheckpointExecutionPolicy{ForcedLanes: make(map[string]struct{}), RequireReusableLanes: make(map[string]struct{})} var forced []pipeline.ResolvedArtifactLane for _, lane := range selected.ArtifactLanes { lane.StepID = selected.ID policy.ForcedLanes[pipeline.CheckpointLaneKey(selected.ID, lane.ID)] = struct{}{} forced = append(forced, lane) } lanes := make(map[string]pipeline.ResolvedArtifactLane) dependents := make(map[string][]pipeline.ResolvedArtifactLane) for _, step := range resolved.Steps { for _, lane := range step.ArtifactLanes { lane.StepID = step.ID lanes[pipeline.CheckpointLaneKey(step.ID, lane.ID)] = lane for _, target := range []pipeline.ResolvedReferenceTarget{lane.ExtractReferences, lane.MergeReferences, lane.NormalizeReferences} { for _, binding := range target.Bindings { if binding.Artifact != nil { producerKey := pipeline.CheckpointLaneKey(binding.Artifact.Step, binding.Artifact.Lane) dependents[producerKey] = append(dependents[producerKey], lane) } } } } } processed := make(map[string]struct{}) queue := append([]pipeline.ResolvedArtifactLane(nil), forced...) for len(queue) > 0 { lane := queue[0] queue = queue[1:] key := pipeline.CheckpointLaneKey(lane.StepID, lane.ID) if _, ok := processed[key]; ok { continue } processed[key] = struct{}{} for _, dependent := range dependents[key] { dependentKey := pipeline.CheckpointLaneKey(dependent.StepID, dependent.ID) if _, alreadyForced := policy.ForcedLanes[dependentKey]; alreadyForced { continue } policy.ForcedLanes[dependentKey] = struct{}{} forced = append(forced, dependent) queue = append(queue, dependent) } } processed = make(map[string]struct{}) queue = append([]pipeline.ResolvedArtifactLane(nil), forced...) for len(queue) > 0 { lane := queue[0] queue = queue[1:] key := pipeline.CheckpointLaneKey(lane.StepID, lane.ID) if _, ok := processed[key]; ok { continue } processed[key] = struct{}{} for _, target := range []pipeline.ResolvedReferenceTarget{lane.ExtractReferences, lane.MergeReferences, lane.NormalizeReferences} { for _, binding := range target.Bindings { if binding.Artifact == nil { continue } producerKey := pipeline.CheckpointLaneKey(binding.Artifact.Step, binding.Artifact.Lane) if _, forcedAlready := policy.ForcedLanes[producerKey]; !forcedAlready { policy.RequireReusableLanes[producerKey] = struct{}{} } if producer, ok := lanes[producerKey]; ok { queue = append(queue, producer) } } } } return policy, nil } func checkpointIdentityFingerprints(values []pipeline.CheckpointFingerprint) []checkpoint.Fingerprint { if len(values) == 0 { return nil } out := make([]checkpoint.Fingerprint, len(values)) for index, value := range values { out[index] = checkpoint.Fingerprint{Name: "component:" + value.Name, Value: value.Value} } return out } func rawInputDigest(data []byte) string { sum := sha256.Sum256(data) return "sha256:" + hex.EncodeToString(sum[:]) } func runtimeOverrideFingerprints(llmProfileOverride string, sessionID string, runtimeOverrides LLMRuntimeOverrides) []checkpoint.Fingerprint { var values []checkpoint.Fingerprint if strings.TrimSpace(llmProfileOverride) != "" { values = append(values, checkpoint.Fingerprint{Name: "llm_profile_override", Value: strings.TrimSpace(llmProfileOverride)}) } if strings.TrimSpace(sessionID) != "" { values = append(values, checkpoint.Fingerprint{Name: "session_id", Value: strings.TrimSpace(sessionID)}) } if runtimeOverrides.ReasoningEffort != nil { value := strings.TrimSpace(*runtimeOverrides.ReasoningEffort) if value == "" { value = "" } values = append(values, checkpoint.Fingerprint{Name: "reasoning_effort_override", Value: value}) } return values } func llmProfileFingerprints(profiles []artifacts.LLMProfileManifest) []checkpoint.Fingerprint { if len(profiles) == 0 { return nil } values := make([]checkpoint.Fingerprint, 0, len(profiles)) for _, profile := range profiles { id := strings.TrimSpace(profile.ID) if id == "" { continue } values = append(values, checkpoint.Fingerprint{ Name: "llm_profile:" + id, Value: strings.TrimSpace(profile.Provider) + ":" + strings.TrimSpace(profile.Model), }) } return values } func configSource(configPath string) string { if strings.TrimSpace(configPath) != "" { return "flag" } return "discovered" } func writeOutputFiles(runOutputDir string, files []contracts.OutputFile) error { for _, file := range files { if _, err := outputFilePath(runOutputDir, file.Name); err != nil { return err } } outputParent := filepath.Dir(runOutputDir) if err := os.MkdirAll(outputParent, 0o755); err != nil { return fmt.Errorf("create output parent %q: %w", outputParent, err) } if err := os.Mkdir(runOutputDir, 0o755); err != nil { if os.IsExist(err) { return fmt.Errorf("output run directory %q already exists", runOutputDir) } return fmt.Errorf("create output run directory %q: %w", runOutputDir, err) } for _, file := range files { if err := fileio.WriteBytes(runOutputDir, file.Name, file.Bytes, 0o755, 0o644); err != nil { return fmt.Errorf("write output file %q: %w", file.Name, err) } } return nil } func outputFilePath(runOutputDir, logicalName string) (string, error) { name := strings.TrimSpace(logicalName) if name == "" { return "", fmt.Errorf("output file name must not be empty") } if strings.Contains(name, `\`) { return "", fmt.Errorf("output file name %q must use slash-separated relative paths", name) } if path.IsAbs(name) || filepath.IsAbs(name) { return "", fmt.Errorf("output file name %q must be relative", name) } if strings.Contains(name, "..") { return "", fmt.Errorf("output file name %q must not contain ..", name) } cleaned := path.Clean(name) if cleaned == "." || cleaned != name { return "", fmt.Errorf("output file name %q must be clean", name) } root, err := filepath.Abs(runOutputDir) if err != nil { return "", fmt.Errorf("resolve output directory %q: %w", runOutputDir, err) } target, err := filepath.Abs(filepath.Join(root, filepath.FromSlash(cleaned))) if err != nil { return "", fmt.Errorf("resolve output file %q: %w", name, err) } rel, err := filepath.Rel(root, target) if err != nil { return "", fmt.Errorf("resolve output file %q: %w", name, err) } if rel == "." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) || rel == ".." { return "", fmt.Errorf("output file name %q resolves outside output directory", name) } return target, nil } func reorderRunArgs(args []string) []string { var flags []string var positionals []string for i := 0; i < len(args); i++ { arg := args[i] if arg == "--" { positionals = append(positionals, args[i+1:]...) break } if strings.HasPrefix(arg, "-") { flags = append(flags, arg) if runFlagTakesValue(arg) && !strings.Contains(arg, "=") && i+1 < len(args) { i++ flags = append(flags, args[i]) } continue } positionals = append(positionals, arg) } return append(flags, positionals...) } func runFlagTakesValue(arg string) bool { switch arg { case "--config", "--config-overlay", "--input", "--only", "--output-dir", "--debug-dir", "--llm-profile", "--session-id", "--reasoning-effort", "--chunk_cache", "--reference", "--without-reference", "--recompute-step": return true default: return false } } func runConfigurationOverrides(chunkCache chunkCacheFlag, outputDir, debugDir string) []configkit.Override { overrides := make([]configkit.Override, 0, 3) if chunkCache.set { overrides = append(overrides, configkit.Override{ Path: configkit.Keys("cache", "chunk_plans", "mode"), Value: string(chunkCache.value), Flag: "--chunk_cache", }) } if outputDir = strings.TrimSpace(outputDir); outputDir != "" { overrides = append(overrides, configkit.Override{ Path: configkit.Keys("output", "directory"), Value: outputDir, Flag: "--output-dir", }) } if debugDir = strings.TrimSpace(debugDir); debugDir != "" { overrides = append(overrides, configkit.Override{ Path: configkit.Keys("debug", "directory"), Value: debugDir, Flag: "--debug-dir", }) } return overrides } type chunkCacheFlag struct { value pipeline.ChunkCacheMode set bool } func (f *chunkCacheFlag) String() string { if f == nil { return "" } return string(f.value) } func (f *chunkCacheFlag) Set(raw string) error { mode, err := pipeline.ParseChunkCacheMode(raw) if err != nil { return err } f.value = mode f.set = true return nil } func (f chunkCacheFlag) explicitValue() string { if !f.set { return "" } return string(f.value) } func chunkPlanStoreForRun(cfg config.ChunkPlanCacheConfig, opts Options) (pipeline.ChunkPlanStore, error) { if cfg.Mode == pipeline.ChunkCacheBypass { return nil, nil } root := strings.TrimSpace(cfg.Directory) if root == "" { var err error root, err = config.DefaultChunkPlanRoot(opts.UserCacheDir) if err != nil { return nil, fmt.Errorf("resolve chunk plan root: %w", err) } } store, err := opts.ChunkPlanStoreFactory(root) if err != nil { return nil, fmt.Errorf("create chunk plan store at %q: %w", root, err) } if store == nil { return nil, fmt.Errorf("create chunk plan store at %q: factory returned nil", root) } return store, nil } func validateRunFlagValues(args []string) error { for i, arg := range args { if arg != "--config-overlay" && arg != "--session-id" && arg != "--reasoning-effort" { continue } if i+1 >= len(args) || strings.HasPrefix(args[i+1], "-") { return fmt.Errorf("flag needs an argument: %s", arg) } } return nil } func effectiveConfigOverlay(overlay string) string { if overlay == "" || overlay == "default" { return "" } return overlay } func flagWasProvided(args []string, name string) bool { for _, arg := range args { if arg == name || strings.HasPrefix(arg, name+"=") { return true } } return false } func effectiveLLMProfileIDs(resolved pipeline.ResolvedPipeline) []string { seen := make(map[string]struct{}) add := func(binding pipeline.ModuleBinding) { id := strings.TrimSpace(binding.LLMProfile) if id != "" { seen[id] = struct{}{} } } if resolved.InputExecutionClass == contracts.ExecutionClassLLMBacked { add(resolved.Input) } if resolved.ChunkExecutionClass == contracts.ExecutionClassLLMBacked { add(resolved.Chunk) } for _, lane := range resolved.AllArtifactLanes() { if lane.ExtractExecutionClass == contracts.ExecutionClassLLMBacked { add(lane.Extract) } if lane.MergeExecutionClass == contracts.ExecutionClassLLMBacked { add(lane.Merge) } if lane.NormalizeExecutionClass == contracts.ExecutionClassLLMBacked { add(lane.Normalize) } } for _, chain := range resolved.ValidatorChains { for _, validator := range chain.Validators { if validator.ExecutionClass == contracts.ExecutionClassLLMBacked { add(validator.Binding) } } } if resolved.OutputExecutionClass == contracts.ExecutionClassLLMBacked { add(resolved.Output) } ids := make([]string, 0, len(seen)) for id := range seen { ids = append(ids, id) } sort.Strings(ids) return ids } func runMetadata(outputDir, debugDir string) map[string]any { metadata := make(map[string]any) if dir := strings.TrimSpace(outputDir); dir != "" { metadata["output_dir"] = dir } if dir := strings.TrimSpace(debugDir); dir != "" { metadata["debug_dir"] = dir } if len(metadata) == 0 { return nil } return metadata } func parseOnly(raw string) ([]string, error) { if strings.TrimSpace(raw) == "" { return nil, nil } parts := strings.Split(raw, ",") result := make([]string, 0, len(parts)) for _, part := range parts { trimmed := strings.TrimSpace(part) if trimmed == "" { return nil, fmt.Errorf("--only must contain comma-separated non-empty artifact lane IDs") } result = append(result, trimmed) } return result, nil } type stringListFlag []string func (flag *stringListFlag) String() string { if flag == nil { return "" } return strings.Join(*flag, ",") } func (flag *stringListFlag) Set(value string) error { *flag = append(*flag, value) return nil } type sessionIDFlag struct { value string set bool } type singleValueFlag struct { name string value string set bool } func (flag *singleValueFlag) String() string { if flag == nil { return "" } return flag.value } func (flag *singleValueFlag) Set(value string) error { if flag.set { return fmt.Errorf("%s may be specified only once", flag.name) } flag.value = value flag.set = true return nil } func (flag *sessionIDFlag) String() string { if flag == nil { return "" } return flag.value } func (flag *sessionIDFlag) Set(value string) error { flag.value = value flag.set = true return nil } type cliReferenceRequest struct { Selector cliReferenceSelector Source string } type cliReferenceUnbindRequest struct { Selector cliReferenceSelector } type cliReferenceSelector struct { Scope cliReferenceSelectorScope LaneID string Stage pipeline.ModuleStage SlotName string } type cliReferenceSelectorScope uint8 const ( cliReferenceScopePipeline cliReferenceSelectorScope = iota cliReferenceScopeLane cliReferenceScopeChunk cliReferenceScopeBinding ) func parseReferenceFlags(values []string) ([]cliReferenceRequest, error) { if len(values) == 0 { return nil, nil } requests := make([]cliReferenceRequest, 0, len(values)) for _, raw := range values { name, source, ok := strings.Cut(raw, "=") if !ok { return nil, fmt.Errorf("--reference must use slot=path, lane.slot=path, or lane.stage.slot=path") } if strings.TrimSpace(source) == "" { return nil, fmt.Errorf("--reference path must not be empty; use --without-reference to unbind") } selector, err := parseReferenceSelector(name, "--reference") if err != nil { return nil, err } requests = append(requests, cliReferenceRequest{ Selector: selector, Source: strings.TrimSpace(source), }) } return requests, nil } func parseReferenceUnbindFlags(values []string) ([]cliReferenceUnbindRequest, error) { if len(values) == 0 { return nil, nil } requests := make([]cliReferenceUnbindRequest, 0, len(values)) for _, raw := range values { if strings.Contains(raw, "=") { return nil, fmt.Errorf("--without-reference must use a reference selector without =path") } selector, err := parseReferenceSelector(raw, "--without-reference") if err != nil { return nil, err } requests = append(requests, cliReferenceUnbindRequest{ Selector: selector, }) } return requests, nil } func parseReferenceSelector(raw string, flagName string) (cliReferenceSelector, error) { selector := strings.TrimSpace(raw) if selector == "" { return cliReferenceSelector{}, fmt.Errorf("%s reference slot must not be empty", flagName) } parts := strings.Split(selector, ".") for _, part := range parts { if strings.TrimSpace(part) == "" { return cliReferenceSelector{}, fmt.Errorf("%s must use non-empty reference selector values", flagName) } } switch len(parts) { case 1: return cliReferenceSelector{Scope: cliReferenceScopePipeline, SlotName: strings.TrimSpace(parts[0])}, nil case 2: first := strings.TrimSpace(parts[0]) slotName := strings.TrimSpace(parts[1]) if first == string(pipeline.StageChunk) { return cliReferenceSelector{Scope: cliReferenceScopeChunk, Stage: pipeline.StageChunk, SlotName: slotName}, nil } return cliReferenceSelector{Scope: cliReferenceScopeLane, LaneID: first, SlotName: slotName}, nil case 3: laneID := strings.TrimSpace(parts[0]) stage := pipeline.ModuleStage(strings.TrimSpace(parts[1])) slotName := strings.TrimSpace(parts[2]) if stage != pipeline.StageExtract && stage != pipeline.StageMerge && stage != pipeline.StageNormalize { return cliReferenceSelector{}, fmt.Errorf("%s lane-qualified selector must use lane.extract.slot, lane.merge.slot, or lane.normalize.slot", flagName) } return cliReferenceSelector{Scope: cliReferenceScopeBinding, LaneID: laneID, Stage: stage, SlotName: slotName}, nil default: return cliReferenceSelector{}, fmt.Errorf("%s must use slot, chunk.slot, lane.slot, lane.extract.slot, lane.merge.slot, or lane.normalize.slot", flagName) } } func resolveCLIReferenceRequests( cfg config.Config, pipelineID string, only []string, catalog pipeline.ModuleCatalog, referenceRequests []cliReferenceRequest, unbindRequests []cliReferenceUnbindRequest, ) ([]pipeline.ReferenceBinding, []pipeline.ReferenceUnbind, error) { if len(referenceRequests) == 0 && len(unbindRequests) == 0 { return nil, nil, nil } targets, err := selectedReferenceTargets(cfg, pipelineID, only, catalog) if err != nil { return nil, nil, err } // Broad CLI selectors are only presentation syntax. Collapse them into one // highest-specificity action per concrete framework target before pipeline // resolution so the generic reference contract stays stage-and-lane exact. actions := make(map[cliReferenceTargetKey]resolvedCLIReferenceAction) for _, request := range referenceRequests { matches, err := resolveCLIReferenceTargets(targets, request.Selector) if err != nil { return nil, nil, err } for _, target := range matches { candidate := resolvedCLIReferenceAction{ kind: cliReferenceActionBind, selector: request.Selector, target: target, slotName: request.Selector.SlotName, source: request.Source, specificity: request.Selector.specificity(), } if err := mergeCLIReferenceAction(actions, candidate); err != nil { return nil, nil, err } } } for _, request := range unbindRequests { matches, err := resolveCLIReferenceTargets(targets, request.Selector) if err != nil { return nil, nil, err } for _, target := range matches { candidate := resolvedCLIReferenceAction{ kind: cliReferenceActionUnbind, selector: request.Selector, target: target, slotName: request.Selector.SlotName, specificity: request.Selector.specificity(), } if err := mergeCLIReferenceAction(actions, candidate); err != nil { return nil, nil, err } } } resolved := make([]resolvedCLIReferenceAction, 0, len(actions)) for _, action := range actions { resolved = append(resolved, action) } sort.Slice(resolved, func(i, j int) bool { left, right := resolved[i], resolved[j] if left.target.laneID != right.target.laneID { return left.target.laneID < right.target.laneID } if left.target.stage != right.target.stage { return referenceStageOrder(left.target.stage) < referenceStageOrder(right.target.stage) } return left.slotName < right.slotName }) overrides := make([]pipeline.ReferenceBinding, 0, len(resolved)) unbinds := make([]pipeline.ReferenceUnbind, 0, len(resolved)) for _, action := range resolved { switch action.kind { case cliReferenceActionBind: overrides = append(overrides, pipeline.ReferenceBinding{ Stage: action.target.stage, LaneID: action.target.laneID, SlotName: action.slotName, Source: action.source, BindingSource: contracts.ReferenceBindingSourceCLI, }) case cliReferenceActionUnbind: unbinds = append(unbinds, pipeline.ReferenceUnbind{ Stage: action.target.stage, LaneID: action.target.laneID, SlotName: action.slotName, }) } } return overrides, unbinds, nil } type cliReferenceActionKind uint8 const ( cliReferenceActionBind cliReferenceActionKind = iota cliReferenceActionUnbind ) type cliReferenceTargetKey struct { stage pipeline.ModuleStage laneID string slotName string } type resolvedCLIReferenceAction struct { kind cliReferenceActionKind selector cliReferenceSelector target selectedReferenceTarget slotName string source string specificity int } func mergeCLIReferenceAction(actions map[cliReferenceTargetKey]resolvedCLIReferenceAction, candidate resolvedCLIReferenceAction) error { key := cliReferenceTargetKey{stage: candidate.target.stage, laneID: candidate.target.laneID, slotName: candidate.slotName} current, ok := actions[key] if !ok || candidate.specificity > current.specificity { actions[key] = candidate return nil } if candidate.specificity < current.specificity { return nil } if candidate.kind != current.kind { return fmt.Errorf("reference target %q slot %q is both bound by %q and unbound by %q at the same specificity", targetLabel(candidate.target), candidate.slotName, current.selector.String(), candidate.selector.String()) } actions[key] = candidate return nil } func (selector cliReferenceSelector) specificity() int { switch selector.Scope { case cliReferenceScopePipeline: return 0 case cliReferenceScopeLane: return 1 case cliReferenceScopeChunk, cliReferenceScopeBinding: return 2 default: return -1 } } func (selector cliReferenceSelector) String() string { switch selector.Scope { case cliReferenceScopePipeline: return selector.SlotName case cliReferenceScopeLane: return selector.LaneID + "." + selector.SlotName case cliReferenceScopeChunk: return "chunk." + selector.SlotName case cliReferenceScopeBinding: return selector.LaneID + "." + string(selector.Stage) + "." + selector.SlotName default: return selector.SlotName } } type selectedReferenceTarget struct { laneID string stage pipeline.ModuleStage module string slots map[string]struct{} } func selectedReferenceTargets(cfg config.Config, pipelineID string, only []string, catalog pipeline.ModuleCatalog) ([]selectedReferenceTarget, error) { profile, ok := lookupCLIReferencePipeline(cfg.Pipelines, pipelineID) if !ok { return nil, fmt.Errorf("pipeline %q is not configured", strings.TrimSpace(pipelineID)) } if profile.Steps != nil && len(only) > 0 { return nil, fmt.Errorf("pipeline %q --only is not supported for explicit ordered steps", strings.TrimSpace(pipelineID)) } lanesByID := make(map[string]pipeline.ArtifactLaneProfile, len(profile.Artifacts)) addLanes := func(values map[string]pipeline.ArtifactLaneProfile) error { for rawLaneID, lane := range values { laneID := strings.TrimSpace(rawLaneID) if laneID == "" { return fmt.Errorf("pipeline %q artifact lane id must not be empty", strings.TrimSpace(pipelineID)) } if _, ok := lanesByID[laneID]; ok { return fmt.Errorf("pipeline %q artifact lane %q is duplicated after trimming", strings.TrimSpace(pipelineID), laneID) } lanesByID[laneID] = lane } return nil } if profile.Steps == nil { if err := addLanes(profile.Artifacts); err != nil { return nil, err } } else { for _, step := range profile.Steps { if err := addLanes(step.Artifacts); err != nil { return nil, err } } } selectedIDs := make([]string, 0, len(lanesByID)) if len(only) == 0 { for laneID := range lanesByID { selectedIDs = append(selectedIDs, laneID) } } else { seen := make(map[string]struct{}, len(only)) for _, rawLaneID := range only { laneID := strings.TrimSpace(rawLaneID) if laneID == "" { return nil, fmt.Errorf("pipeline %q selected artifact lane id must not be empty", strings.TrimSpace(pipelineID)) } if _, ok := lanesByID[laneID]; !ok { return nil, fmt.Errorf("pipeline %q selected artifact lane %q is not declared", strings.TrimSpace(pipelineID), laneID) } if _, ok := seen[laneID]; !ok { selectedIDs = append(selectedIDs, laneID) seen[laneID] = struct{}{} } } } sort.Strings(selectedIDs) targets := make([]selectedReferenceTarget, 0, 1+len(selectedIDs)*3) chunk := pipeline.Binding(profile.Chunk.Module) chunk.Module = strings.TrimSpace(profile.Chunk.Module) if chunk.Module == "" { chunk.Module = pipeline.DefaultChunkModule } chunkSpec, err := cliReferenceChunkerSpec(catalog, chunk.Module) if err != nil { return nil, fmt.Errorf("pipeline %q chunk module %q: %w", strings.TrimSpace(pipelineID), chunk.Module, err) } targets = append(targets, selectedReferenceTarget{ stage: pipeline.StageChunk, module: chunk.Module, slots: referenceSlotSet(chunkSpec.ReferenceSlots), }) for _, laneID := range selectedIDs { lane := lanesByID[laneID] extractModule := strings.TrimSpace(lane.Extract.Module) if extractModule == "" { return nil, fmt.Errorf("pipeline %q lane %q extract module must not be empty", strings.TrimSpace(pipelineID), laneID) } extractSpec, err := cliReferenceExtractorSpec(catalog, extractModule) if err != nil { return nil, fmt.Errorf("pipeline %q lane %q extract module %q: %w", strings.TrimSpace(pipelineID), laneID, extractModule, err) } artifactKind := extractSpec.ArtifactKind if artifactKind == "" { return nil, fmt.Errorf("pipeline %q lane %q extract module %q does not declare an artifact kind", strings.TrimSpace(pipelineID), laneID, extractModule) } targets = append(targets, selectedReferenceTarget{ laneID: laneID, stage: pipeline.StageExtract, module: extractModule, slots: referenceSlotSet(extractSpec.ReferenceSlots), }) mergeModule := strings.TrimSpace(lane.Merge.Module) if mergeModule == "" { mergeModule = pipeline.DefaultMergeModule } mergeSpec, err := cliReferenceMergerSpec(catalog, mergeModule, artifactKind) if err != nil { return nil, fmt.Errorf("pipeline %q lane %q merge module %q: %w", strings.TrimSpace(pipelineID), laneID, mergeModule, err) } targets = append(targets, selectedReferenceTarget{ laneID: laneID, stage: pipeline.StageMerge, module: mergeModule, slots: referenceSlotSet(mergeSpec.ReferenceSlots), }) normalizeModule := strings.TrimSpace(lane.Normalize.Module) if normalizeModule == "" { normalizeModule = pipeline.DefaultNormalizeModule } normalizeSpec, err := cliReferenceNormalizerSpec(catalog, normalizeModule, artifactKind) if err != nil { return nil, fmt.Errorf("pipeline %q lane %q normalize module %q: %w", strings.TrimSpace(pipelineID), laneID, normalizeModule, err) } targets = append(targets, selectedReferenceTarget{ laneID: laneID, stage: pipeline.StageNormalize, module: normalizeModule, slots: referenceSlotSet(normalizeSpec.ReferenceSlots), }) } return targets, nil } func lookupCLIReferencePipeline(profiles map[string]pipeline.PipelineProfile, pipelineID string) (pipeline.PipelineProfile, bool) { pipelineID = strings.TrimSpace(pipelineID) for rawID, profile := range profiles { if strings.TrimSpace(rawID) == pipelineID { return profile, true } } return pipeline.PipelineProfile{}, false } func cliReferenceChunkerSpec(catalog pipeline.ModuleCatalog, module string) (pipeline.ModuleSpec, error) { if catalog.Chunkers == nil { return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module) } spec, ok := catalog.Chunkers.Spec(module) if !ok { return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module) } return spec, nil } func cliReferenceExtractorSpec(catalog pipeline.ModuleCatalog, module string) (pipeline.ModuleSpec, error) { if catalog.Extractors == nil { return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module) } spec, ok := catalog.Extractors.Spec(module) if !ok { return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module) } return spec, nil } func cliReferenceMergerSpec(catalog pipeline.ModuleCatalog, module string, kind contracts.ArtifactKind) (pipeline.ModuleSpec, error) { if catalog.Mergers == nil { return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module) } registered := catalog.Mergers.RegisteredArtifactKinds(module) if len(registered) == 0 { return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module) } spec, ok := catalog.Mergers.SpecForArtifact(module, kind) if !ok { return pipeline.ModuleSpec{}, cliReferenceArtifactVariantError("merger", module, kind, registered) } return spec, nil } func cliReferenceNormalizerSpec(catalog pipeline.ModuleCatalog, module string, kind contracts.ArtifactKind) (pipeline.ModuleSpec, error) { if catalog.Normalizers == nil { return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module) } registered := catalog.Normalizers.RegisteredArtifactKinds(module) if len(registered) == 0 { return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module) } spec, ok := catalog.Normalizers.SpecForArtifact(module, kind) if !ok { return pipeline.ModuleSpec{}, cliReferenceArtifactVariantError("normalizer", module, kind, registered) } return spec, nil } func cliReferenceArtifactVariantError(moduleType string, module string, kind contracts.ArtifactKind, registered []contracts.ArtifactKind) error { values := make([]string, len(registered)) for i, value := range registered { values[i] = string(value) } if len(values) == 0 { return fmt.Errorf("%s %q has no typed variant for artifact kind %q", moduleType, module, kind) } return fmt.Errorf("%s %q has no typed variant for artifact kind %q; registered kinds: %s", moduleType, module, kind, strings.Join(values, ", ")) } func referenceSlotSet(slots []contracts.ReferenceSlot) map[string]struct{} { slotSet := make(map[string]struct{}, len(slots)) for _, slot := range slots { slotSet[slot.Name] = struct{}{} } return slotSet } func resolveCLIReferenceTargets(targets []selectedReferenceTarget, selector cliReferenceSelector) ([]selectedReferenceTarget, error) { slotName := strings.TrimSpace(selector.SlotName) if slotName == "" { return nil, fmt.Errorf("reference slot must not be empty") } switch selector.Scope { case cliReferenceScopePipeline: matches := make([]selectedReferenceTarget, 0, len(targets)) for _, target := range targets { if _, ok := target.slots[slotName]; ok { matches = append(matches, target) } } if len(matches) == 0 { return nil, fmt.Errorf("reference slot %q is not declared by any selected target", slotName) } return matches, nil case cliReferenceScopeChunk: for _, target := range targets { if target.stage != pipeline.StageChunk { continue } if _, ok := target.slots[slotName]; !ok { return nil, fmt.Errorf("reference slot %q is not declared by chunk module %q", slotName, target.module) } return []selectedReferenceTarget{target}, nil } return nil, fmt.Errorf("reference chunk target is not selected") case cliReferenceScopeLane: laneSelected := false matches := make([]selectedReferenceTarget, 0, 3) for _, target := range targets { if target.laneID != selector.LaneID { continue } laneSelected = true if _, ok := target.slots[slotName]; ok { matches = append(matches, target) } } if !laneSelected { return nil, fmt.Errorf("reference lane %q is not selected", selector.LaneID) } if len(matches) == 0 { return nil, fmt.Errorf("reference slot %q is not declared by selected lane %q", slotName, selector.LaneID) } return matches, nil case cliReferenceScopeBinding: laneSelected := false for _, target := range targets { if target.laneID != selector.LaneID { continue } laneSelected = true if target.stage != selector.Stage { continue } if _, ok := target.slots[slotName]; !ok { return nil, fmt.Errorf("reference slot %q is not declared by selected %s target %q", slotName, selector.Stage, targetLabel(target)) } return []selectedReferenceTarget{target}, nil } if !laneSelected { return nil, fmt.Errorf("reference lane %q is not selected", selector.LaneID) } return nil, fmt.Errorf("reference %s target is not selected for lane %q", selector.Stage, selector.LaneID) default: return nil, fmt.Errorf("reference selector has unknown scope") } } func targetLabel(target selectedReferenceTarget) string { if target.stage == pipeline.StageChunk { return "chunk" } return target.laneID + "." + string(target.stage) } func referenceStageOrder(stage pipeline.ModuleStage) int { switch stage { case pipeline.StageChunk: return 0 case pipeline.StageExtract: return 1 case pipeline.StageMerge: return 2 case pipeline.StageNormalize: return 3 default: return 4 } } func sortedPipelineIDs(cfg config.Config) []string { ids := make([]string, 0, len(cfg.Pipelines)) for id := range cfg.Pipelines { ids = append(ids, strings.TrimSpace(id)) } sort.Strings(ids) return ids }