Add offline pipeline inspection and shared native resource preflight

This commit is contained in:
2026-11-22 03:32:31 +00:00
parent 9f66421f4f
commit 2a9c5a6075
9 changed files with 806 additions and 10 deletions

View File

@@ -1503,6 +1503,27 @@ operations. Check meaningful evidence, not the complete native JSON shape.
## Stage 32: Add Offline Preflight And Pipeline Inspection Commands
**Status:** Complete. Added native pipeline list/show/validate declarations using
the shared captured application load. List enumerates sorted configuration IDs
without constructing a catalog or engine. Show uses ordinary domain resolution
and reports steps, lanes, modules, controls, validator chains, prompt declarations,
and reference bindings without expanding profiles or serializing runtime options.
Validation resolves the same domain contracts and supplies the captured engine
only for selected model work. The new preflight package inspects every known
initial/correction/validator/input/output prompt and explicit/applicable default
profile, preserves native causes, and aggregates pipeline/step/lane/validator
context. Partial profiles pass; undeclared dynamic selections remain runtime
checks. No request preparation, transcript/reference reads, credential lookup,
provider call, module construction, or run-storage creation is involved.
Focused real-kit preflight and CLI command tests, vet, repeated race checks,
focused pipeline tests, production preflight/pipeline/config builds, and diff
checks pass. Native configuration validation retains no pipeline-specific mode;
added assembled help/config-command regressions. The full CLI suite, including
those assembled regressions, remains blocked at setup by removed reporting-package
imports and old ConfigKit's YAML plugin/limit dependency; it is not claimed passing.
Scheduled cleanup and final verification must execute it. Native run-handler
composition will reuse this preflight boundary in Stage 33.
**Goal:** execution and inspection share domain resolution and known dependencies.
**Inspect/change:** new `internal/preflight`, CLI pipeline declarations, catalog,

View File

@@ -2,10 +2,9 @@ package cli
import (
"context"
"fmt"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"go.maximumdirect.net/ckconfig"
"go.maximumdirect.net/ckprompt"
"go.maximumdirect.net/commandkit"
@@ -42,6 +41,7 @@ func NewApplication(version string, options Options) (*commandkit.App, error) {
definitions = ckprompt.WithNamespaces(definitions, promptkit.Namespaces{})
declarations := []commandkit.Declaration{
commandkit.Completion("notarius"),
pipelineCommands(newState, func() (pipeline.ModuleCatalog, error) { return effectiveCatalog(options) }),
ckconfig.Command(ckconfig.Suite[struct{}, *nativeLoaders]{
NewState: newState,
CheckInput: func(_ string, selection ckconfig.Selection, _ struct{}) error {
@@ -53,10 +53,3 @@ func NewApplication(version string, options Options) (*commandkit.App, error) {
declarations = append(declarations, ckprompt.Declarations(definitions)...)
return commandkit.New(commandkit.Identity{Name: "notarius", Version: version}, declarations...)
}
func checkConfigSelection(selection ckconfig.Selection) error {
if selection.Config.Set && strings.TrimSpace(selection.Config.Value) == "" {
return fmt.Errorf("--config must not be empty")
}
return nil
}

View File

@@ -17,7 +17,7 @@ func TestNativeApplicationBuiltinsNeedNoConfiguration(t *testing.T) {
if err != nil {
t.Fatal(err)
}
for _, args := range [][]string{{"--help"}, {"help"}, {"version"}, {"completion", "bash"}, {"completion", "zsh"}, {"completion", "fish"}, {"config", "--help"}} {
for _, args := range [][]string{{"--help"}, {"help"}, {"version"}, {"completion", "bash"}, {"completion", "zsh"}, {"completion", "fish"}, {"config", "--help"}, {"pipeline", "--help"}, {"pipeline", "validate", "--help"}} {
var out, diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &out, Err: &diagnostics})
if outcome.Status != 0 || out.Len() == 0 {
@@ -52,6 +52,7 @@ func TestNativeConfigurationScopesAndDiffUseSelectedLoadEvidence(t *testing.T) {
{"differences", []string{"config", "diff", "--config", left, "--against-config", right, "--check"}, 3, "workers"},
{"identical", []string{"config", "diff", "--config", left, "--against-config", left, "--check"}, 0, ""},
{"empty", []string{"config", "show", "--config", ""}, 2, ""},
{"configuration has no pipeline mode", []string{"config", "validate", "--config", left, "--pipeline", "demo"}, 2, ""},
} {
t.Run(test.name, func(t *testing.T) {
var out, diagnostics bytes.Buffer

View File

@@ -31,6 +31,13 @@ type loadedSelection struct {
engine *promptkit.Engine
}
func checkConfigSelection(selection ckconfig.Selection) error {
if selection.Config.Set && strings.TrimSpace(selection.Config.Value) == "" {
return fmt.Errorf("--config must not be empty")
}
return nil
}
func (loaders *nativeLoaders) application(ctx context.Context, selection ckconfig.Selection) (*configkit.Result[config.Document], error) {
if err := ctx.Err(); err != nil {
return nil, err

View File

@@ -0,0 +1,143 @@
package cli
import (
"context"
"errors"
"fmt"
"sort"
"strings"
"text/template"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/preflight"
"go.maximumdirect.net/ckconfig"
"go.maximumdirect.net/commandkit"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
type pipelineInspectionInput struct {
Selection ckconfig.Selection
PipelineID string
}
func configurationSelectionFlags[I any](destination func(*I) *ckconfig.Selection) []commandkit.Flag[I] {
return []commandkit.Flag[I]{
commandkit.ValueFlag(commandkit.FlagMeta{Name: "config", Usage: "Select the application configuration.", Completion: commandkit.CompleteFiles()}, (*string)(nil), func(value string) (string, error) {
if strings.TrimSpace(value) == "" {
return "", fmt.Errorf("--config must not be empty")
}
return value, nil
}, func(input *I, value commandkit.Supplied[string]) { destination(input).Config = value }),
commandkit.ValueFlag(commandkit.FlagMeta{Name: "overlay", Usage: "Select a configuration overlay.", Completion: commandkit.CompleteFiles()}, (*string)(nil), func(value string) (string, error) { return value, nil }, func(input *I, value commandkit.Supplied[string]) { destination(input).Overlay = value }),
commandkit.RepeatFlag(commandkit.FlagMeta{Name: "set", Usage: "Override POINTER=JSON."}, []configkit.Override(nil), parseConfigurationOverride, func(input *I, value commandkit.Supplied[[]configkit.Override]) {
destination(input).Overrides = value.Value
}),
}
}
func parseConfigurationOverride(raw string) (configkit.Override, error) {
pointer, encoded, ok := strings.Cut(raw, "=")
if !ok {
return configkit.Override{}, fmt.Errorf("override requires POINTER=JSON")
}
path, err := configkit.ParsePath(pointer)
if err != nil {
return configkit.Override{}, err
}
value, err := commandkit.ParseCLIJSON(encoded)
if err != nil {
return configkit.Override{}, err
}
return configkit.Override{Path: path, Value: value, Flag: "--set"}, nil
}
func pipelineCommands(newState func() *nativeLoaders, catalog func() (pipeline.ModuleCatalog, error)) commandkit.Command {
command := func(operation, summary string) commandkit.Command {
var arguments []commandkit.Argument[pipelineInspectionInput]
if operation != "list" {
arguments = []commandkit.Argument[pipelineInspectionInput]{commandkit.ValueArgument(commandkit.ArgumentMeta{Name: "pipeline-id", Usage: "Configured pipeline identifier."}, true, func(value string) (string, error) {
value = strings.TrimSpace(value)
if value == "" {
return "", fmt.Errorf("pipeline id must not be empty")
}
return value, nil
}, func(input *pipelineInspectionInput, value commandkit.Supplied[string]) {
input.PipelineID = value.Value
})}
}
return commandkit.NewCommand(commandkit.CommandSpec[pipelineInspectionInput]{Name: operation, Summary: summary,
Flags: configurationSelectionFlags(func(input *pipelineInspectionInput) *ckconfig.Selection { return &input.Selection }), Arguments: arguments,
CheckInput: func(input pipelineInspectionInput) error { return checkConfigSelection(input.Selection) },
Run: func(invocation commandkit.Invocation, input pipelineInspectionInput) commandkit.Outcome {
state := newState()
result, err := state.application(invocation.Context, input.Selection)
if err != nil {
return pipelineConfigurationFailure(err)
}
if operation == "list" {
ids := make([]string, 0, len(result.Config.Pipelines))
for id := range result.Config.Pipelines {
ids = append(ids, id)
}
sort.Strings(ids)
return reportPipelineInspection(invocation, commandkit.Report{Value: ids})
}
loaded, err := config.FromResult(result)
if err != nil {
return pipelineInvalidConfiguration(err)
}
modules, err := catalog()
if err != nil {
return commandkit.Failure(err.Error()).WithCause(err)
}
effective, err := loaded.RuntimeConfig().Resolve(config.ResolveInput{PipelineID: input.PipelineID, Catalog: modules})
if err != nil {
return pipelineInvalidConfiguration(err)
}
if operation == "show" {
return reportPipelineInspection(invocation, commandkit.Report{Value: describePipeline(effective.ResolvedPipeline, modules)})
}
var engine *promptkit.Engine
if preflight.RequiresEngine(effective.ResolvedPipeline) {
engine, err = state.engine(invocation.Context, input.Selection)
if err != nil {
return pipelineConfigurationFailure(fmt.Errorf("pipeline %q native resources: %w", input.PipelineID, err))
}
}
if err := preflight.Validate(invocation.Context, effective.ResolvedPipeline, modules, engine); err != nil {
return pipelineInvalidConfiguration(err)
}
return reportPipelineInspection(invocation, commandkit.Report{Value: struct {
Pipeline string `json:"pipeline"`
Valid bool `json:"valid"`
}{input.PipelineID, true}, TextTemplate: pipelineValidationTemplate})
},
})
}
return commandkit.Group("pipeline", "Inspect, validate, or execute configured pipelines.", command("list", "List configured pipeline identifiers."), command("show", "Show a resolved pipeline composition."), command("validate", "Validate known pipeline dependencies offline."))
}
var pipelineValidationTemplate = template.Must(commandkit.ParseTextTemplate("pipeline-validation", "Pipeline {{printf \"%q\" .Pipeline}} is valid.\n", nil))
func reportPipelineInspection(invocation commandkit.Invocation, report commandkit.Report) commandkit.Outcome {
if err := invocation.Reporter().Report(report); err != nil {
return commandkit.Failure("Could not report pipeline inspection.").WithCause(err)
}
return commandkit.Success()
}
func pipelineConfigurationFailure(err error) commandkit.Outcome {
if ckconfig.ClassifyConfigurationError(err) == commandkit.OutcomeUsage {
return commandkit.Usage(err.Error()).WithCause(err)
}
return commandkit.Failure(err.Error()).WithCause(err)
}
func pipelineInvalidConfiguration(err error) commandkit.Outcome {
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return commandkit.Failure(err.Error()).WithCause(err)
}
return commandkit.Usage(err.Error()).WithCause(err)
}

View File

@@ -0,0 +1,276 @@
package cli
import (
"bytes"
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"go.maximumdirect.net/commandkit"
"go.maximumdirect.net/promptkit"
)
const inspectionConfiguration = `version: 6
output: {directory: output-must-not-exist}
cache_directory: cache-must-not-exist
debug: {directory: debug-must-not-exist}
pipelines:
demo:
input: {module: fixture/input}
chunk: {module: fixture/chunk}
output: {module: fixture/output}
steps:
- id: build
artifacts:
notes:
extract:
module: fixture/extract
references: {context: {path: reference-must-not-exist.txt}}
validators: [{module: fixture/validator}]
merge: {module: fixture/merge}
normalize: {module: fixture/normalize}
promptkit:
profiles:
- id: partial
model: fixture-model
api_key_env: NOTARIUS_INSPECTION_NO_CREDENTIAL_REQUIRED
api_key_required: true
prompts:
- id: inspect-initial
default_profile: partial
inputs: [{name: source, required: true}]
messages: [{role: user, content: '{{input "source"}}'}]
output: {format: text, validation_mode: none}
- id: inspect-correction
default_profile: partial
messages: [{role: user, content: Correct.}]
output: {format: text, validation_mode: none}
- id: inspect-validator
default_profile: partial
messages: [{role: user, content: Assess.}]
output: {format: text, validation_mode: none}
`
type inspectionArtifact struct {
Value string `json:"value"`
}
type inspectionCodec struct{}
func (inspectionCodec) Kind() contracts.ArtifactKind { return "test/inspection" }
func (inspectionCodec) MediaType() string { return "application/json" }
func (inspectionCodec) Schema() contracts.ArtifactSchema {
return contracts.ArtifactSchema{ID: "test.inspection", Name: "inspection", Version: "v1", JSONSchema: []byte(`{"type":"object","properties":{"value":{"type":"string"}},"required":["value"]}`)}
}
func (inspectionCodec) Encode(value inspectionArtifact) ([]byte, error) { return json.Marshal(value) }
func (codec inspectionCodec) EncodeCandidate(value inspectionArtifact) ([]byte, error) {
return codec.Encode(value)
}
func (codec inspectionCodec) DecodeCandidate(content []byte) (inspectionArtifact, error) {
return codec.Decode(content)
}
func (inspectionCodec) Decode(content []byte) (inspectionArtifact, error) {
var value inspectionArtifact
err := json.Unmarshal(content, &value)
return value, err
}
func inspectionCatalog(t *testing.T, class contracts.ExecutionClass) pipeline.ModuleCatalog {
t.Helper()
catalog := pipeline.ModuleCatalog{Inputs: pipeline.NewInputAdapterRegistry(), Chunkers: pipeline.NewChunkerRegistry(), ArtifactCodecs: pipeline.NewArtifactCodecRegistry(), Extractors: pipeline.NewExtractorRegistry(), Mergers: pipeline.NewMergerRegistry(), Normalizers: pipeline.NewNormalizerRegistry(), Validators: pipeline.NewValidatorRegistry(), ValidatorChains: pipeline.NewValidatorChainRegistry(), Outputs: pipeline.NewOutputEncoderRegistry()}
check := func(err error) {
t.Helper()
if err != nil {
t.Fatal(err)
}
}
spec := func(stage pipeline.ModuleStage, key string) pipeline.ModuleSpec {
return pipeline.ModuleSpec{Key: key, Stage: stage, ExecutionClass: contracts.ExecutionClassDeterministic}
}
check(catalog.Inputs.RegisterWithSpec(spec(pipeline.StageInput, "fixture/input"), func() (contracts.InputAdapter, error) { t.Fatal("inspection constructed input"); return nil, nil }))
check(catalog.Chunkers.RegisterWithSpec(spec(pipeline.StageChunk, "fixture/chunk"), func() (contracts.Chunker, error) { t.Fatal("inspection constructed chunker"); return nil, nil }))
check(catalog.Outputs.RegisterWithSpec(spec(pipeline.StageOutput, "fixture/output"), func() (contracts.OutputEncoder, error) { t.Fatal("inspection constructed output"); return nil, nil }))
check(pipeline.RegisterArtifactCodec(catalog.ArtifactCodecs, inspectionCodec{}))
extract := spec(pipeline.StageExtract, "fixture/extract")
extract.ArtifactKind, extract.ExecutionClass, extract.Prompts = "test/inspection", class, []string{"inspect-initial", "inspect-correction"}
extract.ReferenceSlots = []contracts.ReferenceSlot{{Name: "context", Required: true, AcceptedMediaTypes: []string{"text/plain"}}}
check(pipeline.RegisterExtractor(catalog.Extractors, extract, func() (contracts.Extractor[inspectionArtifact], error) {
t.Fatal("inspection constructed extractor")
return nil, nil
}))
merge := spec(pipeline.StageMerge, "fixture/merge")
merge.ArtifactKind = "test/inspection"
check(pipeline.RegisterMerger(catalog.Mergers, merge, func() (contracts.Merger[inspectionArtifact], error) {
t.Fatal("inspection constructed merger")
return nil, nil
}))
normalize := spec(pipeline.StageNormalize, "fixture/normalize")
normalize.ArtifactKind = "test/inspection"
check(pipeline.RegisterNormalizer(catalog.Normalizers, normalize, func() (contracts.Normalizer[inspectionArtifact], error) {
t.Fatal("inspection constructed normalizer")
return nil, nil
}))
check(pipeline.RegisterTypedValidator(catalog.Validators, "test/inspection", pipeline.ValidatorSpec{Key: "fixture/validator", ExecutionClass: class, Prompts: []string{"inspect-validator"}}, func() (contracts.TypedValidator[inspectionArtifact], error) {
t.Fatal("inspection constructed validator")
return nil, nil
}))
return catalog
}
type inspectionProvider struct{ t *testing.T }
func (provider inspectionProvider) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
provider.t.Fatal("inspection called provider")
return nil, nil
}
func writeInspectionConfiguration(t *testing.T, content string) string {
t.Helper()
file := filepath.Join(t.TempDir(), "config.yml")
if err := os.WriteFile(file, []byte(content), 0600); err != nil {
t.Fatal(err)
}
return file
}
func newInspectionApplication(t *testing.T, catalog func() (pipeline.ModuleCatalog, error)) (*commandkit.App, **nativeLoaders) {
t.Helper()
var last *nativeLoaders
app, err := commandkit.New(commandkit.Identity{Name: "notarius"}, pipelineCommands(func() *nativeLoaders {
last = &nativeLoaders{engineOptions: []promptkit.Option{promptkit.WithLLMClient(inspectionProvider{t})}}
return last
}, catalog))
if err != nil {
t.Fatal(err)
}
return app, &last
}
func TestPipelineListOnlyEnumeratesCapturedConfiguration(t *testing.T) {
content := strings.Replace(inspectionConfiguration, " demo:", " zeta:", 1)
content = strings.Replace(content, "promptkit:", " alpha:\n input: {module: unavailable}\n artifacts:\n value:\n extract: {module: unavailable}\npromptkit:", 1)
file := writeInspectionConfiguration(t, content)
app, state := newInspectionApplication(t, func() (pipeline.ModuleCatalog, error) {
t.Fatal("list constructed a catalog")
return pipeline.ModuleCatalog{}, nil
})
var out, diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), []string{"pipeline", "list", "--config", file, "--format", "json"}, commandkit.Streams{Out: &out, Err: &diagnostics})
var ids []string
if outcome.Status != 0 || json.Unmarshal(out.Bytes(), &ids) != nil || !reflect.DeepEqual(ids, []string{"alpha", "zeta"}) {
t.Fatalf("list=%#v %s %s", outcome, out.String(), diagnostics.String())
}
if len((*state).selections) != 1 || (*state).selections[0].engine != nil {
t.Fatal("list expanded native resources")
}
}
func TestPipelineShowResolvesCompositionWithoutEngineOrExecutionInputs(t *testing.T) {
file := writeInspectionConfiguration(t, inspectionConfiguration)
catalog := inspectionCatalog(t, contracts.ExecutionClassLLMBacked)
app, state := newInspectionApplication(t, func() (pipeline.ModuleCatalog, error) { return catalog, nil })
var out, diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), []string{"pipeline", "show", "demo", "--config", file, "--format", "json"}, commandkit.Streams{Out: &out, Err: &diagnostics})
var report pipelineDescription
if err := json.Unmarshal(out.Bytes(), &report); err != nil || outcome.Status != 0 {
t.Fatalf("show=%#v %s %s %v", outcome, out.String(), diagnostics.String(), err)
}
if report.Pipeline != "demo" || len(report.Steps) != 1 || report.Steps[0].ID != "build" || len(report.Steps[0].Lanes) != 1 || report.Steps[0].Lanes[0].Extract.Module != "fixture/extract" || len(report.Steps[0].Lanes[0].Extract.References) != 1 || len(report.Validators) == 0 {
t.Fatalf("missing resolved composition: %#v", report)
}
if (*state).selections[0].engine != nil {
t.Fatal("show expanded profiles by constructing engine")
}
assertInspectionNoRunStorage(t, file)
}
func TestPipelineValidateChecksNativeDeclarationsOfflineAndUsesOneLoad(t *testing.T) {
for _, class := range []contracts.ExecutionClass{contracts.ExecutionClassDeterministic, contracts.ExecutionClassLLMBacked} {
file := writeInspectionConfiguration(t, inspectionConfiguration)
catalog := inspectionCatalog(t, class)
app, state := newInspectionApplication(t, func() (pipeline.ModuleCatalog, error) { return catalog, nil })
var out, diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), []string{"pipeline", "validate", "demo", "--config", file}, commandkit.Streams{Out: &out, Err: &diagnostics})
if outcome.Status != 0 || !strings.Contains(out.String(), "demo") {
t.Fatalf("validate=%#v %s %s", outcome, out.String(), diagnostics.String())
}
if len((*state).selections) != 1 || ((*state).selections[0].engine != nil) != (class == contracts.ExecutionClassLLMBacked) {
t.Fatal("validation loaded twice or built an unnecessary engine")
}
assertInspectionNoRunStorage(t, file)
}
}
func TestPipelineValidateRejectsKnownMissingResourcesWithScope(t *testing.T) {
for _, id := range []string{"inspect-initial", "inspect-correction", "inspect-validator"} {
file := writeInspectionConfiguration(t, strings.Replace(inspectionConfiguration, "id: "+id, "id: unused-"+id, 1))
catalog := inspectionCatalog(t, contracts.ExecutionClassLLMBacked)
app, _ := newInspectionApplication(t, func() (pipeline.ModuleCatalog, error) { return catalog, nil })
var out, diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), []string{"pipeline", "validate", "demo", "--config", file}, commandkit.Streams{Out: &out, Err: &diagnostics})
if outcome.Status != 2 || !errors.Is(outcome.Cause, promptkit.ErrPromptNotFound) || !strings.Contains(diagnostics.String(), id) || !strings.Contains(diagnostics.String(), "build") || !strings.Contains(diagnostics.String(), "notes") {
t.Fatalf("known dependency=%#v %s", outcome, diagnostics.String())
}
}
}
func TestPipelineInspectionUsesNativeArgumentsSelectionsAndDelivery(t *testing.T) {
file := writeInspectionConfiguration(t, inspectionConfiguration)
catalog := inspectionCatalog(t, contracts.ExecutionClassLLMBacked)
app, _ := newInspectionApplication(t, func() (pipeline.ModuleCatalog, error) { return catalog, nil })
for _, test := range []struct {
args []string
status int
}{
{[]string{"pipeline", "show"}, 2},
{[]string{"pipeline", "show", "demo", "--config", "", "--config", file}, 2},
{[]string{"pipeline", "show", "unknown", "--config", file}, 2},
{[]string{"pipeline", "show", "demo", "--config", file, "--set", `/pipelines/demo/steps/0/artifacts/notes/extract/options={"unsupported":true}`}, 2},
{[]string{"pipeline", "validate", "demo", "--config", file, "--set", `/pipelines/demo/llm_profile="missing-profile"`}, 2},
{[]string{"pipeline", "show", "demo", "--config", file, "--set", `/pipelines/demo/steps/0/artifacts/notes/extract/references=null`}, 2},
{[]string{"pipeline", "list", "--config", file, "--set", `/pipelines/demo/input/module="another-input"`}, 0},
{[]string{"pipeline", "validate", "demo", "--config", file, "--quiet"}, 0},
{[]string{"pipeline", "--help"}, 0},
} {
var out, diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), test.args, commandkit.Streams{Out: &out, Err: &diagnostics})
if outcome.Status != test.status {
t.Fatalf("%v: %#v %s", test.args, outcome, diagnostics.String())
}
if test.args[len(test.args)-1] == "--quiet" && out.Len() != 0 {
t.Fatal("quiet pipeline inspection wrote stdout")
}
}
}
type inspectionFailWriter struct{ err error }
func (writer inspectionFailWriter) Write([]byte) (int, error) { return 0, writer.err }
func TestPipelineInspectionReportFailureRemainsOperational(t *testing.T) {
file := writeInspectionConfiguration(t, inspectionConfiguration)
app, _ := newInspectionApplication(t, func() (pipeline.ModuleCatalog, error) {
t.Fatal("list loaded catalog")
return pipeline.ModuleCatalog{}, nil
})
primary := errors.New("inspection stdout delivery failed")
var diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), []string{"pipeline", "list", "--config", file}, commandkit.Streams{Out: inspectionFailWriter{primary}, Err: &diagnostics})
if outcome.Status != 1 || !errors.Is(outcome.Cause, primary) {
t.Fatalf("delivery failure classified as invalid configuration: %#v %s", outcome, diagnostics.String())
}
}
func assertInspectionNoRunStorage(t *testing.T, file string) {
t.Helper()
entries, err := os.ReadDir(filepath.Dir(file))
if err != nil || len(entries) != 1 || entries[0].Name() != "config.yml" {
t.Fatalf("inspection read/materialized inputs or created storage: %v %v", entries, err)
}
}

View File

@@ -0,0 +1,72 @@
package cli
import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
// Inspection reports identities and resolved composition, not another effective
// configuration or profile expansion. ConfigKit owns secret-aware settings views.
type pipelineModuleDescription struct {
Module string `json:"module"`
ExecutionClass contracts.ExecutionClass `json:"execution_class"`
LLMProfile string `json:"llm_profile,omitempty"`
Controls *contracts.ProducerControls `json:"controls,omitempty"`
Prompts []string `json:"prompts,omitempty"`
References []pipeline.ReferenceBinding `json:"references,omitempty"`
}
type pipelineLaneDescription struct {
ID string `json:"id"`
ArtifactKind contracts.ArtifactKind `json:"artifact_kind"`
Extract pipelineModuleDescription `json:"extract"`
Merge pipelineModuleDescription `json:"merge"`
Normalize pipelineModuleDescription `json:"normalize"`
}
type pipelineStepDescription struct {
ID string `json:"id"`
Lanes []pipelineLaneDescription `json:"lanes"`
}
type pipelineValidatorDescription struct {
Stage pipeline.ModuleStage `json:"stage"`
Lane string `json:"lane,omitempty"`
Module string `json:"module"`
Validators []pipelineModuleDescription `json:"validators"`
}
type pipelineDescription struct {
Pipeline string `json:"pipeline"`
Input pipelineModuleDescription `json:"input"`
Chunk pipelineModuleDescription `json:"chunk"`
Steps []pipelineStepDescription `json:"steps"`
Validators []pipelineValidatorDescription `json:"validators,omitempty"`
Output pipelineModuleDescription `json:"output"`
}
func describePipeline(resolved pipeline.ResolvedPipeline, catalog pipeline.ModuleCatalog) pipelineDescription {
module := func(binding pipeline.ModuleBinding, class contracts.ExecutionClass, prompts []string, controls *contracts.ProducerControls, references []pipeline.ReferenceBinding) pipelineModuleDescription {
return pipelineModuleDescription{Module: binding.Module, ExecutionClass: class, LLMProfile: binding.LLMProfile, Prompts: prompts, Controls: controls, References: references}
}
input, _ := catalog.Inputs.Spec(resolved.Input.Module)
output, _ := catalog.Outputs.Spec(resolved.Output.Module)
report := pipelineDescription{Pipeline: resolved.ID, Input: module(resolved.Input, resolved.InputExecutionClass, input.Prompts, nil, nil),
Chunk: module(resolved.Chunk, resolved.ChunkExecutionClass, resolved.ChunkPrompts, &resolved.ChunkControls, resolved.ChunkReferences.Bindings), Output: module(resolved.Output, resolved.OutputExecutionClass, output.Prompts, nil, nil)}
for _, step := range resolved.Steps {
description := pipelineStepDescription{ID: step.ID, Lanes: make([]pipelineLaneDescription, 0, len(step.ArtifactLanes))}
for _, lane := range step.ArtifactLanes {
description.Lanes = append(description.Lanes, pipelineLaneDescription{ID: lane.ID, ArtifactKind: lane.ArtifactKind,
Extract: module(lane.Extract, lane.ExtractExecutionClass, lane.ExtractPrompts, &lane.ExtractControls, lane.ExtractReferences.Bindings),
Merge: module(lane.Merge, lane.MergeExecutionClass, lane.MergePrompts, &lane.MergeControls, lane.MergeReferences.Bindings),
Normalize: module(lane.Normalize, lane.NormalizeExecutionClass, lane.NormalizePrompts, &lane.NormalizeControls, lane.NormalizeReferences.Bindings)})
}
report.Steps = append(report.Steps, description)
}
for _, chain := range resolved.ValidatorChains {
description := pipelineValidatorDescription{Stage: chain.Stage, Lane: chain.LaneID, Module: chain.ModuleKey, Validators: make([]pipelineModuleDescription, 0, len(chain.Validators))}
for _, validator := range chain.Validators {
description.Validators = append(description.Validators, module(validator.Binding, validator.ExecutionClass, validator.Prompts, nil, nil))
}
report.Validators = append(report.Validators, description)
}
return report
}

View File

@@ -0,0 +1,120 @@
// Package preflight checks statically known native resources for an already
// resolved pipeline. Acquisition, construction, and presentation belong to callers.
package preflight
import (
"context"
"errors"
"fmt"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"go.maximumdirect.net/promptkit"
)
// RequiresEngine reports selected model work, including nested validators.
// Deterministic pipelines need no engine merely to validate their composition.
func RequiresEngine(resolved pipeline.ResolvedPipeline) bool {
if resolved.InputExecutionClass == contracts.ExecutionClassLLMBacked || resolved.ChunkExecutionClass == contracts.ExecutionClassLLMBacked || resolved.OutputExecutionClass == contracts.ExecutionClassLLMBacked {
return true
}
for _, step := range resolved.Steps {
for _, lane := range step.ArtifactLanes {
if lane.ExtractExecutionClass == contracts.ExecutionClassLLMBacked || lane.MergeExecutionClass == contracts.ExecutionClassLLMBacked || lane.NormalizeExecutionClass == contracts.ExecutionClassLLMBacked {
return true
}
}
}
for _, chain := range resolved.ValidatorChains {
for _, validator := range chain.Validators {
if validator.ExecutionClass == contracts.ExecutionClassLLMBacked {
return true
}
}
}
return false
}
// Validate checks the same declarations and resource selections used by native
// request builders. It does not render, prepare, read input/reference files,
// resolve credential values, or call providers. Dynamic selections remain runtime
// checks. Callers first use their ordinary application resolver for module,
// options, type, validator-chain, and ordered-reference compatibility.
func Validate(ctx context.Context, resolved pipeline.ResolvedPipeline, catalog pipeline.ModuleCatalog, engine *promptkit.Engine) error {
if ctx == nil {
return errors.New("preflight context must not be nil")
}
if err := ctx.Err(); err != nil {
return err
}
var findings []error
check := func(stage pipeline.ModuleStage, step, lane, module, validator string, class contracts.ExecutionClass, profile string, prompts []string) {
if class != contracts.ExecutionClassLLMBacked || ctx.Err() != nil {
return
}
scope := fmt.Sprintf("pipeline %q", resolved.ID)
if step != "" {
scope += fmt.Sprintf(" step %q", step)
}
if lane != "" {
scope += fmt.Sprintf(" lane %q", lane)
}
scope += fmt.Sprintf(" %s module %q", stage, module)
if validator != "" {
scope += fmt.Sprintf(" validator %q", validator)
}
if engine == nil {
findings = append(findings, fmt.Errorf("%s: %w: native engine is required", scope, promptkit.ErrInvalidConfig))
return
}
if profile != "" {
if _, err := engine.InspectProfile(ctx, profile); err != nil {
findings = append(findings, fmt.Errorf("%s profile %q: %w", scope, profile, err))
}
}
for _, id := range prompts {
if ctx.Err() != nil {
return
}
prompt, err := engine.InspectPrompt(ctx, id)
if err != nil {
findings = append(findings, fmt.Errorf("%s prompt %q: %w", scope, id, err))
continue
}
if profile != "" {
continue
}
if prompt.DefaultProfileID == nil || *prompt.DefaultProfileID == "" {
// A module may select a profile dynamically when constructing its
// actual request. No declared selection is not a missing resource.
continue
}
if _, err := engine.InspectProfile(ctx, *prompt.DefaultProfileID); err != nil {
findings = append(findings, fmt.Errorf("%s prompt %q default profile %q: %w", scope, id, *prompt.DefaultProfileID, err))
}
}
}
input, _ := catalog.Inputs.Spec(resolved.Input.Module)
check(pipeline.StageInput, "", "", resolved.Input.Module, "", resolved.InputExecutionClass, resolved.Input.LLMProfile, input.Prompts)
check(pipeline.StageChunk, "", "", resolved.Chunk.Module, "", resolved.ChunkExecutionClass, resolved.Chunk.LLMProfile, resolved.ChunkPrompts)
steps := make(map[string]string)
for _, step := range resolved.Steps {
for _, lane := range step.ArtifactLanes {
steps[lane.ID] = step.ID
check(pipeline.StageExtract, step.ID, lane.ID, lane.Extract.Module, "", lane.ExtractExecutionClass, lane.Extract.LLMProfile, lane.ExtractPrompts)
check(pipeline.StageMerge, step.ID, lane.ID, lane.Merge.Module, "", lane.MergeExecutionClass, lane.Merge.LLMProfile, lane.MergePrompts)
check(pipeline.StageNormalize, step.ID, lane.ID, lane.Normalize.Module, "", lane.NormalizeExecutionClass, lane.Normalize.LLMProfile, lane.NormalizePrompts)
}
}
for _, chain := range resolved.ValidatorChains {
for _, validator := range chain.Validators {
check(chain.Stage, steps[chain.LaneID], chain.LaneID, chain.ModuleKey, validator.Binding.Module, validator.ExecutionClass, validator.Binding.LLMProfile, validator.Prompts)
}
}
output, _ := catalog.Outputs.Spec(resolved.Output.Module)
check(pipeline.StageOutput, "", "", resolved.Output.Module, "", resolved.OutputExecutionClass, resolved.Output.LLMProfile, output.Prompts)
if err := ctx.Err(); err != nil {
findings = append(findings, err)
}
return errors.Join(findings...)
}

View File

@@ -0,0 +1,163 @@
package preflight
import (
"context"
"errors"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
type noGeneration struct{ t *testing.T }
func (client noGeneration) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
client.t.Fatal("offline preflight invoked a provider")
return nil, nil
}
func inspectionEngine(t *testing.T) *promptkit.Engine {
t.Helper()
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{promptkit.SettingsInput(configkit.Bytes("offline resources", []byte(`profiles:
- id: partial
model: fixture
api_key_env: NOTARIUS_PREFLIGHT_MUST_NOT_REQUIRE_THIS_CREDENTIAL
api_key_required: true
prompts:
- id: initial
default_profile: partial
inputs: [{name: source, required: true}]
messages: [{role: user, content: '{{input "source"}}'}]
output: {format: text, validation_mode: none}
- id: correction
default_profile: partial
messages: [{role: user, content: 'Correct the previous response.'}]
output: {format: text, validation_mode: none}
- id: validator
default_profile: partial
messages: [{role: user, content: 'Assess.'}]
output: {format: text, validation_mode: none}
- id: unknown-default
default_profile: missing-profile
messages: [{role: user, content: 'Select a profile.'}]
output: {format: text, validation_mode: none}
- id: no-default
messages: [{role: user, content: 'Select a profile.'}]
output: {format: text, validation_mode: none}
`)))}}}}}, promptkit.WithBundledOpenRouter(false), promptkit.WithLLMClient(noGeneration{t}))
if err != nil {
t.Fatal(err)
}
return engine
}
func resolvedFixture() pipeline.ResolvedPipeline {
return pipeline.ResolvedPipeline{ID: "campaign", Steps: []pipeline.ResolvedPipelineStep{{ID: "identify", ArtifactLanes: []pipeline.ResolvedArtifactLane{{ID: "characters", Extract: pipeline.Binding("character-producer"), ExtractExecutionClass: contracts.ExecutionClassLLMBacked, ExtractPrompts: []string{"initial", "correction"}}}}},
ValidatorChains: []pipeline.ResolvedValidatorChain{{Stage: pipeline.StageExtract, LaneID: "characters", ModuleKey: "character-producer", Validators: []pipeline.ResolvedValidator{{Binding: pipeline.Binding("domain-validator"), ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{"validator"}}}}},
}
}
func TestOfflinePreflightAcceptsPartialDefaultsWithoutExecutionInputs(t *testing.T) {
engine := inspectionEngine(t)
resolved := resolvedFixture()
resolved.Steps[0].ArtifactLanes[0].ExtractReferences.Bindings = []pipeline.ReferenceBinding{{Source: "/not-present/reference.json"}}
if !RequiresEngine(resolved) {
t.Fatal("selected LLM roles were missed")
}
if err := Validate(context.Background(), resolved, pipeline.ModuleCatalog{}, engine); err != nil {
t.Fatalf("partial profile or absent execution inputs rejected: %v", err)
}
}
func TestOfflinePreflightReportsEveryKnownResourceWithScopeAndCause(t *testing.T) {
resolved := resolvedFixture()
lane := &resolved.Steps[0].ArtifactLanes[0]
lane.Extract.LLMProfile = "missing-explicit"
lane.ExtractPrompts = []string{"missing-initial", "missing-correction"}
resolved.ValidatorChains[0].Validators[0].Prompts = []string{"missing-validator"}
err := Validate(context.Background(), resolved, pipeline.ModuleCatalog{}, inspectionEngine(t))
if !errors.Is(err, promptkit.ErrPromptNotFound) || !errors.Is(err, promptkit.ErrProfileNotFound) {
t.Fatalf("native causes lost: %v", err)
}
for _, value := range []string{"campaign", "identify", "characters", "extract", "character-producer", "domain-validator", "missing-explicit", "missing-initial", "missing-correction", "missing-validator"} {
if !strings.Contains(err.Error(), value) {
t.Fatalf("missing preflight context %q: %v", value, err)
}
}
}
func TestOfflinePreflightChecksApplicableProfilesButNotExecutionReadiness(t *testing.T) {
engine := inspectionEngine(t)
for _, test := range []struct {
prompt, profile string
want error
}{
{"unknown-default", "", promptkit.ErrProfileNotFound},
{"no-default", "", nil},
{"unknown-default", "partial", nil},
{"no-default", "partial", nil},
} {
resolved := resolvedFixture()
lane := &resolved.Steps[0].ArtifactLanes[0]
lane.ExtractPrompts, lane.Extract.LLMProfile = []string{test.prompt}, test.profile
err := Validate(context.Background(), resolved, pipeline.ModuleCatalog{}, engine)
if test.want == nil && err != nil || test.want != nil && !errors.Is(err, test.want) {
t.Fatalf("prompt=%q profile=%q: %v", test.prompt, test.profile, err)
}
}
}
func TestOfflinePreflightLeavesDynamicSelectionsForRuntime(t *testing.T) {
resolved := resolvedFixture()
resolved.Steps[0].ArtifactLanes[0].ExtractPrompts = nil
resolved.ValidatorChains = nil
if err := Validate(context.Background(), resolved, pipeline.ModuleCatalog{}, inspectionEngine(t)); err != nil {
t.Fatal(err)
}
}
func TestOfflinePreflightIncludesInputOutputAndNestedValidatorsWithoutConstruction(t *testing.T) {
catalog := pipeline.ModuleCatalog{Inputs: pipeline.NewInputAdapterRegistry(), Outputs: pipeline.NewOutputEncoderRegistry()}
for _, stage := range []pipeline.ModuleStage{pipeline.StageInput, pipeline.StageOutput} {
spec := pipeline.ModuleSpec{Key: string(stage), Stage: stage, ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{"missing-" + string(stage)}}
var err error
if stage == pipeline.StageInput {
err = catalog.Inputs.RegisterWithSpec(spec, func() (contracts.InputAdapter, error) { t.Fatal("preflight constructed input"); return nil, nil })
} else {
err = catalog.Outputs.RegisterWithSpec(spec, func() (contracts.OutputEncoder, error) { t.Fatal("preflight constructed output"); return nil, nil })
}
if err != nil {
t.Fatal(err)
}
}
resolved := pipeline.ResolvedPipeline{ID: "io", Input: pipeline.Binding("input"), InputExecutionClass: contracts.ExecutionClassLLMBacked, Output: pipeline.Binding("output"), OutputExecutionClass: contracts.ExecutionClassLLMBacked}
err := Validate(context.Background(), resolved, catalog, inspectionEngine(t))
if !errors.Is(err, promptkit.ErrPromptNotFound) || !strings.Contains(err.Error(), "missing-input") || !strings.Contains(err.Error(), "missing-output") {
t.Fatalf("input/output declarations were missed: %v", err)
}
resolved = resolvedFixture()
resolved.Steps[0].ArtifactLanes[0].ExtractExecutionClass = contracts.ExecutionClassDeterministic
if !RequiresEngine(resolved) {
t.Fatal("nested validator omitted from engine selection")
}
}
func TestDeterministicPreflightNeedsNoEngineAndCancellationStopsInspection(t *testing.T) {
if RequiresEngine(pipeline.ResolvedPipeline{}) {
t.Fatal("deterministic composition requires engine")
}
if err := Validate(context.Background(), pipeline.ResolvedPipeline{}, pipeline.ModuleCatalog{}, nil); err != nil {
t.Fatal(err)
}
if err := Validate(context.Background(), resolvedFixture(), pipeline.ModuleCatalog{}, nil); !errors.Is(err, promptkit.ErrInvalidConfig) {
t.Fatalf("missing engine: %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if err := Validate(ctx, resolvedFixture(), pipeline.ModuleCatalog{}, nil); !errors.Is(err, context.Canceled) {
t.Fatalf("cancellation lost: %v", err)
}
}