From 936fc1e72a56b8182523cba4011966a8d5af3e56 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 22 Nov 2026 05:24:30 +0000 Subject: [PATCH] Migrate run state and reference contracts to native execution --- docs/roadmap/implementation.md | 12 + internal/cli/cache_contract_test.go | 59 +- internal/cli/example_contract_test.go | 56 + internal/cli/reference_contract_test.go | 148 +-- internal/cli/run_contract_test.go | 794 ++----------- internal/cli/state_hardening_test.go | 1118 ++++--------------- internal/cli/state_permissions_unix_test.go | 28 +- 7 files changed, 450 insertions(+), 1765 deletions(-) diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index f61e0e2c..30386cf3 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1729,6 +1729,18 @@ smoke-test kind rather than a command spelling. Focused native command/configura tests, vet, and repeated race checks pass. Whole CLI compilation still awaits the remaining run/state, enemy-event, reference, and example fixture cleanup. +Run/state contracts now use real native execution, domain files, and the unified +accepted-artifact cache rather than a synthetic completion/checkpoint harness. +Retained root independence, all cache modes, group-accessible reuse, narrow run +identities, output/debug collision protection, selected-data capture, failure +retention, discovery, and overlay origins. Reference-selector contracts retain +scope/specificity, unbinding, type variants, and generated-reference protection +without constructing modules. Moved the complete-example selector contract beside +the other example tests; maintained example migration remains Stage 36. +Focused native run/state/reference tests and vet pass; repeated race checks for +the run/state and command/configuration set pass. Remaining enemy-event/example +fixtures, dependency tidy/notices, and whole-tree compilation are still required. + **Goal:** delete the old architecture rather than leave dormant alternatives. **Inspect/change:** all production/test callers of the feature roadmap's removal diff --git a/internal/cli/cache_contract_test.go b/internal/cli/cache_contract_test.go index 908d0383..832a9a6c 100644 --- a/internal/cli/cache_contract_test.go +++ b/internal/cli/cache_contract_test.go @@ -2,54 +2,25 @@ package cli import ( "bytes" - "errors" - "os" - "strings" "testing" ) -func TestConfigCommandsDoNotResolveRunState(t *testing.T) { +func TestConfigCommandsDoNotAllocateRunState(t *testing.T) { for _, args := range [][]string{ - {"config", "validate", "--config"}, - {"pipelines", "list", "--config"}, + {"config", "validate"}, {"config", "show"}, {"pipeline", "list"}, {"pipeline", "validate", "dnd-session"}, } { - name := strings.Join(args[:2], "-") - t.Run(name, func(t *testing.T) { - roots := newStateTestRoots(t) - opts := newStateTestHarness().options() - opts.UserCacheDir = func() (string, error) { return "", errors.New("state root must not be resolved") } - command := append([]string(nil), args...) - command = append(command, roots.config) - var stdout, stderr bytes.Buffer - code := RunWithOptions(command, &stdout, &stderr, opts) - if code != 0 { - t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) - } - assertNoRunState(t, roots) - }) + fixture := newStateRunFixture(t) + var stdout, stderr bytes.Buffer + command := append(append([]string(nil), args...), "--config", fixture.configPath) + code := RunWithOptions(command, &stdout, &stderr, fixture.options) + if code != 0 { + t.Fatalf("%v status=%d stdout=%q stderr=%q", args, code, stdout.String(), stderr.String()) + } + assertAbsent(t, fixture.outputRoot) + assertAbsent(t, fixture.cacheRoot) + assertAbsent(t, fixture.debugRoot) + if len(fixture.client.requestsSnapshot()) != 0 { + t.Fatal("inspection invoked provider") + } } } - -func replaceStateTestConfigLine(t *testing.T, path, old, new string) { - t.Helper() - data, err := os.ReadFile(path) - if err != nil { - t.Fatal(err) - } - text := replaceRequiredOnce(t, string(data), old, new) - if err := os.WriteFile(path, []byte(text), 0o600); err != nil { - t.Fatal(err) - } -} - -func removeStateTestConfigLine(t *testing.T, path, line string) { - replaceStateTestConfigLine(t, path, line, "") -} - -func assertNoRunState(t *testing.T, roots stateTestRoots) { - t.Helper() - assertAbsent(t, roots.output) - assertAbsent(t, roots.plans) - assertAbsent(t, roots.checkpoints) - assertAbsent(t, roots.debug) -} diff --git a/internal/cli/example_contract_test.go b/internal/cli/example_contract_test.go index d7a95f0a..b8f5ac34 100644 --- a/internal/cli/example_contract_test.go +++ b/internal/cli/example_contract_test.go @@ -404,3 +404,59 @@ type exampleOutputIndexEntry struct { func resolveInputForMaintainedExample(components productionComponents, pipelineID string) config.ResolveInput { return config.ResolveInput{PipelineID: pipelineID, Catalog: catalogFromRegistries(components.registries)} } + +func TestCompleteDNDSharedCLIReferencesExpandAcrossCompatibleTargets(t *testing.T) { + cfg := loadMaintainedExample(t, repositoryPath("examples", "dnd-complete.config.yml")) + catalog := catalogFromRegistries(productionTestComponents(t).registries) + sources := map[string]string{ + "party": "/references/party.txt", + "players": "/references/players.txt", + "glossary": "/references/glossary.txt", + "spell_catalog": "/references/spells.json", + } + requests := make([]cliReferenceRequest, 0, len(sources)) + for _, slot := range []string{"party", "players", "glossary", "spell_catalog"} { + requests = append(requests, cliReferenceRequest{ + Selector: mustParseReferenceSelector(t, slot, "--reference"), + Source: sources[slot], + }) + } + overrides, unbinds, err := resolveCLIReferenceRequests(cfg, "dnd-session", nil, catalog, requests, nil) + if err != nil { + t.Fatalf("expand complete D&D references: %v", err) + } + if len(unbinds) != 0 { + t.Fatalf("unbinds = %#v, want none", unbinds) + } + + actual := make(map[string]pipeline.ReferenceBinding, len(overrides)) + for _, binding := range overrides { + actual[referenceContractBindingLabel(binding)] = binding + } + targets, err := selectedReferenceTargets(cfg, "dnd-session", nil, catalog) + if err != nil { + t.Fatal(err) + } + matched := make(map[string]int, len(sources)) + for _, target := range targets { + for slot, sourcePath := range sources { + if _, ok := target.slots[slot]; !ok { + continue + } + matched[slot]++ + label := targetLabel(target) + "." + slot + binding, ok := actual[label] + if !ok || binding.Source != sourcePath || binding.BindingSource != contracts.ReferenceBindingSourceCLI { + t.Fatalf("binding %q = %#v, want CLI source %q", label, binding, sourcePath) + } + } + } + for slot := range sources { + if matched[slot] < 2 { + t.Fatalf("reference %q matched %d target(s), want a shared D&D reference", slot, matched[slot]) + } + } + if _, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalog, ReferenceOverrides: overrides}); err != nil { + t.Fatalf("resolve complete D&D CLI references: %v", err) + } +} diff --git a/internal/cli/reference_contract_test.go b/internal/cli/reference_contract_test.go index d01a8ba1..762dd9a6 100644 --- a/internal/cli/reference_contract_test.go +++ b/internal/cli/reference_contract_test.go @@ -86,62 +86,6 @@ func TestReferenceSelectorSpecificityAndFinalOccurrenceChooseConcreteBindings(t } } -func TestCompleteDNDSharedCLIReferencesExpandAcrossCompatibleTargets(t *testing.T) { - cfg := loadMaintainedExample(t, repositoryPath("examples", "dnd-complete.config.yml")) - catalog := catalogFromRegistries(productionTestComponents(t).registries) - sources := map[string]string{ - "party": "/references/party.txt", - "players": "/references/players.txt", - "glossary": "/references/glossary.txt", - "spell_catalog": "/references/spells.json", - } - requests := make([]cliReferenceRequest, 0, len(sources)) - for _, slot := range []string{"party", "players", "glossary", "spell_catalog"} { - requests = append(requests, cliReferenceRequest{ - Selector: mustParseReferenceSelector(t, slot, "--reference"), - Source: sources[slot], - }) - } - overrides, unbinds, err := resolveCLIReferenceRequests(cfg, "dnd-session", nil, catalog, requests, nil) - if err != nil { - t.Fatalf("expand complete D&D references: %v", err) - } - if len(unbinds) != 0 { - t.Fatalf("unbinds = %#v, want none", unbinds) - } - - actual := make(map[string]pipeline.ReferenceBinding, len(overrides)) - for _, binding := range overrides { - actual[referenceContractBindingLabel(binding)] = binding - } - targets, err := selectedReferenceTargets(cfg, "dnd-session", nil, catalog) - if err != nil { - t.Fatal(err) - } - matched := make(map[string]int, len(sources)) - for _, target := range targets { - for slot, sourcePath := range sources { - if _, ok := target.slots[slot]; !ok { - continue - } - matched[slot]++ - label := targetLabel(target) + "." + slot - binding, ok := actual[label] - if !ok || binding.Source != sourcePath || binding.BindingSource != contracts.ReferenceBindingSourceCLI { - t.Fatalf("binding %q = %#v, want CLI source %q", label, binding, sourcePath) - } - } - } - for slot := range sources { - if matched[slot] < 2 { - t.Fatalf("reference %q matched %d target(s), want a shared D&D reference", slot, matched[slot]) - } - } - if _, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalog, ReferenceOverrides: overrides}); err != nil { - t.Fatalf("resolve complete D&D CLI references: %v", err) - } -} - func TestReferenceSelectorsRejectInvalidOrUnselectedScopesBeforeMaterialization(t *testing.T) { cfg := referenceContractConfig() catalog := referenceContractCatalog(t, true, true) @@ -173,14 +117,14 @@ func TestReferenceSyntaxErrorsReturnTwo(t *testing.T) { name string args []string }{ - {name: "reference missing value", args: []string{"run", "demo", "--config", "missing.yml", "--input", "input.txt", "--reference"}}, - {name: "reference missing selector", args: []string{"run", "demo", "--config", "missing.yml", "--input", "input.txt", "--reference", "=path.txt"}}, - {name: "reference missing separator", args: []string{"run", "demo", "--config", "missing.yml", "--input", "input.txt", "--reference", "slot"}}, - {name: "reference missing path", args: []string{"run", "demo", "--config", "missing.yml", "--input", "input.txt", "--reference", "slot="}}, - {name: "reference excess segments", args: []string{"run", "demo", "--config", "missing.yml", "--input", "input.txt", "--reference", "a.b.c.d=path.txt"}}, - {name: "unbind with path", args: []string{"run", "demo", "--config", "missing.yml", "--input", "input.txt", "--without-reference", "slot=path.txt"}}, - {name: "unbind excess segments", args: []string{"run", "demo", "--config", "missing.yml", "--input", "input.txt", "--without-reference", "a.b.c.d"}}, - {name: "unbind missing value", args: []string{"run", "demo", "--config", "missing.yml", "--input", "input.txt", "--without-reference"}}, + {name: "reference missing value", args: []string{"run", "demo", "--config", "missing.yml", "--input-file", "input.txt", "--reference"}}, + {name: "reference missing selector", args: []string{"run", "demo", "--config", "missing.yml", "--input-file", "input.txt", "--reference", "=path.txt"}}, + {name: "reference missing separator", args: []string{"run", "demo", "--config", "missing.yml", "--input-file", "input.txt", "--reference", "slot"}}, + {name: "reference missing path", args: []string{"run", "demo", "--config", "missing.yml", "--input-file", "input.txt", "--reference", "slot="}}, + {name: "reference excess segments", args: []string{"run", "demo", "--config", "missing.yml", "--input-file", "input.txt", "--reference", "a.b.c.d=path.txt"}}, + {name: "unbind with path", args: []string{"run", "demo", "--config", "missing.yml", "--input-file", "input.txt", "--without-reference", "slot=path.txt"}}, + {name: "unbind excess segments", args: []string{"run", "demo", "--config", "missing.yml", "--input-file", "input.txt", "--without-reference", "a.b.c.d"}}, + {name: "unbind missing value", args: []string{"run", "demo", "--config", "missing.yml", "--input-file", "input.txt", "--without-reference"}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -289,7 +233,7 @@ func TestReferenceMaterializationSeparatesCLIAndConfigPathOrigins(t *testing.T) workingDir := t.TempDir() cfg := referenceContractConfig() configPath := filepath.Join(configDir, "config.yml") - if err := os.WriteFile(configPath, []byte("version: 5\n"), 0o600); err != nil { + if err := os.WriteFile(configPath, []byte("version: 6\n"), 0o600); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(configDir, "required.txt"), []byte("config reference"), 0o600); err != nil { @@ -348,10 +292,9 @@ func TestPipelineScopedCLIReferenceProtectsGeneratedHandoff(t *testing.T) { Selector: mustParseReferenceSelector(t, "shared", "--reference"), Source: filepath.Join(t.TempDir(), "never-read.json"), }}, nil) - if err != nil { - t.Fatalf("expand CLI reference: %v", err) + if err == nil { + _, err = cfg.Resolve(config.ResolveInput{PipelineID: "demo", Catalog: catalog, ReferenceOverrides: overrides}) } - _, err = cfg.Resolve(config.ResolveInput{PipelineID: "demo", Catalog: catalog, ReferenceOverrides: overrides}) if err == nil || !strings.Contains(err.Error(), "conflicting generated and external bindings") || strings.Contains(err.Error(), "never-read.json") { t.Fatalf("resolve error = %v, want generated/external conflict before file access", err) } @@ -482,24 +425,55 @@ func referenceContractCatalog(t *testing.T, includeBetaMerger, includeBetaNormal t.Fatal(err) } } - register(registries.Inputs.RegisterBuilderWithSpec(pipeline.ModuleSpec{Key: "reference/input", Stage: pipeline.StageInput, ExecutionClass: contracts.ExecutionClassDeterministic, Provides: []string{"source"}}, func(map[string]any) error { return nil }, func(pipeline.BuildRequest) (contracts.InputAdapter, error) { return stateTestInput{}, nil })) - register(registries.Chunkers.RegisterWithSpec(pipeline.ModuleSpec{Key: "reference/chunk", Stage: pipeline.StageChunk, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"source"}, Provides: []string{"chunks"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "chunk-slot"}, {Name: "required-chunk", Required: true}}}, func() (contracts.Chunker, error) { return stateTestChunker{}, nil })) + register(registries.Inputs.RegisterBuilderWithSpec(pipeline.ModuleSpec{Key: "reference/input", Stage: pipeline.StageInput, ExecutionClass: contracts.ExecutionClassDeterministic, Provides: []string{"source"}}, func(map[string]any) error { return nil }, func(pipeline.BuildRequest) (contracts.InputAdapter, error) { + t.Fatal("reference selection constructed a module") + return nil, nil + })) + register(registries.Chunkers.RegisterWithSpec(pipeline.ModuleSpec{Key: "reference/chunk", Stage: pipeline.StageChunk, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"source"}, Provides: []string{"chunks"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "chunk-slot"}, {Name: "required-chunk", Required: true}}}, func() (contracts.Chunker, error) { + t.Fatal("reference selection constructed a module") + return nil, nil + })) register(pipeline.RegisterArtifactCodec(registries.ArtifactCodecs, referenceContractCodecA{})) register(pipeline.RegisterArtifactCodec(registries.ArtifactCodecs, referenceContractCodecB{})) - register(pipeline.RegisterExtractor(registries.Extractors, pipeline.ModuleSpec{Key: "reference/extract-alpha", Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: referenceContractKindAlpha, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared", AcceptedArtifactKinds: []contracts.ArtifactKind{referenceContractKindBeta}, AcceptedMediaTypes: []string{"application/json"}}, {Name: "alpha-slot"}, {Name: "required-extract", Required: true}}}, func() (contracts.Extractor[stateTestArtifact], error) { return stateTestExtractor{}, nil })) - register(pipeline.RegisterExtractor(registries.Extractors, pipeline.ModuleSpec{Key: "reference/extract-beta", Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: referenceContractKindBeta, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "beta-slot"}, {Name: "required-extract", Required: true}}}, func() (contracts.Extractor[stateTestArtifact], error) { return stateTestExtractor{}, nil })) - register(pipeline.RegisterMerger(registries.Mergers, pipeline.ModuleSpec{Key: "reference/shared-merge", Stage: pipeline.StageMerge, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"artifact"}, Provides: []string{"merged"}, ArtifactKind: referenceContractKindAlpha, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "alpha-merge"}, {Name: "required-merge", Required: true}}}, func() (contracts.Merger[stateTestArtifact], error) { return stateTestMerger{}, nil })) + register(pipeline.RegisterExtractor(registries.Extractors, pipeline.ModuleSpec{Key: "reference/extract-alpha", Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: referenceContractKindAlpha, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared", AcceptedArtifactKinds: []contracts.ArtifactKind{referenceContractKindBeta}, AcceptedMediaTypes: []string{"application/json"}}, {Name: "alpha-slot"}, {Name: "required-extract", Required: true}}}, func() (contracts.Extractor[referenceContractArtifact], error) { + t.Fatal("reference selection constructed a module") + return nil, nil + })) + register(pipeline.RegisterExtractor(registries.Extractors, pipeline.ModuleSpec{Key: "reference/extract-beta", Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: referenceContractKindBeta, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "beta-slot"}, {Name: "required-extract", Required: true}}}, func() (contracts.Extractor[referenceContractArtifact], error) { + t.Fatal("reference selection constructed a module") + return nil, nil + })) + register(pipeline.RegisterMerger(registries.Mergers, pipeline.ModuleSpec{Key: "reference/shared-merge", Stage: pipeline.StageMerge, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"artifact"}, Provides: []string{"merged"}, ArtifactKind: referenceContractKindAlpha, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "alpha-merge"}, {Name: "required-merge", Required: true}}}, func() (contracts.Merger[referenceContractArtifact], error) { + t.Fatal("reference selection constructed a module") + return nil, nil + })) if includeBetaMerger { - register(pipeline.RegisterMerger(registries.Mergers, pipeline.ModuleSpec{Key: "reference/shared-merge", Stage: pipeline.StageMerge, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"artifact"}, Provides: []string{"merged"}, ArtifactKind: referenceContractKindBeta, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "beta-merge"}, {Name: "required-merge", Required: true}}}, func() (contracts.Merger[stateTestArtifact], error) { return stateTestMerger{}, nil })) + register(pipeline.RegisterMerger(registries.Mergers, pipeline.ModuleSpec{Key: "reference/shared-merge", Stage: pipeline.StageMerge, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"artifact"}, Provides: []string{"merged"}, ArtifactKind: referenceContractKindBeta, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "beta-merge"}, {Name: "required-merge", Required: true}}}, func() (contracts.Merger[referenceContractArtifact], error) { + t.Fatal("reference selection constructed a module") + return nil, nil + })) } - register(pipeline.RegisterNormalizer(registries.Normalizers, pipeline.ModuleSpec{Key: "reference/shared-normalize", Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: referenceContractKindAlpha, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "alpha-normalize"}, {Name: "required-normalize", Required: true}}}, func() (contracts.Normalizer[stateTestArtifact], error) { return stateTestNormalizer{}, nil })) + register(pipeline.RegisterNormalizer(registries.Normalizers, pipeline.ModuleSpec{Key: "reference/shared-normalize", Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: referenceContractKindAlpha, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "alpha-normalize"}, {Name: "required-normalize", Required: true}}}, func() (contracts.Normalizer[referenceContractArtifact], error) { + t.Fatal("reference selection constructed a module") + return nil, nil + })) if includeBetaNormalizer { - register(pipeline.RegisterNormalizer(registries.Normalizers, pipeline.ModuleSpec{Key: "reference/shared-normalize", Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: referenceContractKindBeta, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "beta-normalize"}, {Name: "required-normalize", Required: true}}}, func() (contracts.Normalizer[stateTestArtifact], error) { return stateTestNormalizer{}, nil })) + register(pipeline.RegisterNormalizer(registries.Normalizers, pipeline.ModuleSpec{Key: "reference/shared-normalize", Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: referenceContractKindBeta, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "beta-normalize"}, {Name: "required-normalize", Required: true}}}, func() (contracts.Normalizer[referenceContractArtifact], error) { + t.Fatal("reference selection constructed a module") + return nil, nil + })) } - register(registries.Outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "reference/output", Stage: pipeline.StageOutput, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"normalized"}, Provides: []string{"output"}}, func() (contracts.OutputEncoder, error) { return stateTestOutput{}, nil })) + register(registries.Outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "reference/output", Stage: pipeline.StageOutput, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"normalized"}, Provides: []string{"output"}}, func() (contracts.OutputEncoder, error) { + t.Fatal("reference selection constructed a module") + return nil, nil + })) return catalogFromRegistries(registries) } +type referenceContractArtifact struct { + Value string `json:"value"` +} + type referenceContractCodecB struct{} type referenceContractCodecA struct{} @@ -509,16 +483,16 @@ func (referenceContractCodecA) Schema() contracts.ArtifactSchema { return contracts.ArtifactSchema{ID: "reference.alpha", Name: "reference_alpha", Version: "v1", JSONSchema: []byte(`{"type":"object"}`)} } func (referenceContractCodecA) MediaType() string { return "application/json" } -func (referenceContractCodecA) EncodeCandidate(stateTestArtifact) ([]byte, error) { +func (referenceContractCodecA) EncodeCandidate(referenceContractArtifact) ([]byte, error) { return []byte(`{"value":"ok"}`), nil } -func (referenceContractCodecA) Encode(stateTestArtifact) ([]byte, error) { +func (referenceContractCodecA) Encode(referenceContractArtifact) ([]byte, error) { return []byte(`{"value":"ok"}`), nil } -func (referenceContractCodecA) Decode([]byte) (stateTestArtifact, error) { - return stateTestArtifact{Value: "ok"}, nil +func (referenceContractCodecA) Decode([]byte) (referenceContractArtifact, error) { + return referenceContractArtifact{Value: "ok"}, nil } -func (codec referenceContractCodecA) DecodeCandidate(content []byte) (stateTestArtifact, error) { +func (codec referenceContractCodecA) DecodeCandidate(content []byte) (referenceContractArtifact, error) { return codec.Decode(content) } @@ -527,16 +501,16 @@ func (referenceContractCodecB) Schema() contracts.ArtifactSchema { return contracts.ArtifactSchema{ID: "reference.beta", Name: "reference_beta", Version: "v1", JSONSchema: []byte(`{"type":"object"}`)} } func (referenceContractCodecB) MediaType() string { return "application/json" } -func (referenceContractCodecB) EncodeCandidate(stateTestArtifact) ([]byte, error) { +func (referenceContractCodecB) EncodeCandidate(referenceContractArtifact) ([]byte, error) { return []byte(`{"value":"ok"}`), nil } -func (referenceContractCodecB) Encode(stateTestArtifact) ([]byte, error) { +func (referenceContractCodecB) Encode(referenceContractArtifact) ([]byte, error) { return []byte(`{"value":"ok"}`), nil } -func (referenceContractCodecB) Decode([]byte) (stateTestArtifact, error) { - return stateTestArtifact{Value: "ok"}, nil +func (referenceContractCodecB) Decode([]byte) (referenceContractArtifact, error) { + return referenceContractArtifact{Value: "ok"}, nil } -func (codec referenceContractCodecB) DecodeCandidate(content []byte) (stateTestArtifact, error) { +func (codec referenceContractCodecB) DecodeCandidate(content []byte) (referenceContractArtifact, error) { return codec.Decode(content) } diff --git a/internal/cli/run_contract_test.go b/internal/cli/run_contract_test.go index e31a7a13..d7c2ea98 100644 --- a/internal/cli/run_contract_test.go +++ b/internal/cli/run_contract_test.go @@ -2,750 +2,110 @@ package cli import ( "bytes" - "context" - "errors" - "fmt" "os" "path/filepath" "strings" "testing" - - "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" - "gitea.maximumdirect.net/eric/notarius/internal/core/config" - "gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle" - "gitea.maximumdirect.net/eric/notarius/internal/framework/checkpoint" - "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" - "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" ) -func TestRunControlsRejectSyntaxWithoutAllocatingState(t *testing.T) { - tests := []struct { - name string - args func(stateTestRoots) []string - }{ - {name: "missing pipeline", args: func(roots stateTestRoots) []string { - return []string{"run", "--config", roots.config, "--input", roots.input} - }}, - {name: "missing input", args: func(roots stateTestRoots) []string { - return []string{"run", "sample", "--config", roots.config} - }}, - {name: "unknown flag", args: func(roots stateTestRoots) []string { - return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--unknown"} - }}, - {name: "blank output directory", args: func(roots stateTestRoots) []string { - return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--output-dir", ""} - }}, - {name: "blank debug directory", args: func(roots stateTestRoots) []string { - return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--debug-dir", ""} - }}, - {name: "missing configuration overlay", args: func(roots stateTestRoots) []string { - return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--config-overlay"} - }}, - {name: "debug directory without debug", args: func(roots stateTestRoots) []string { - return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--debug-dir", filepath.Join(filepath.Dir(roots.debug), "requested-debug")} - }}, - {name: "blank session ID", args: func(roots stateTestRoots) []string { - return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--session-id", ""} - }}, - {name: "blank LLM profile", args: func(roots stateTestRoots) []string { - return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--llm-profile", ""} - }}, - {name: "whitespace LLM profile", args: func(roots stateTestRoots) []string { - return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--llm-profile", " \t "} - }}, - {name: "multiple pipeline IDs", args: func(roots stateTestRoots) []string { - return []string{"run", "sample", "extra", "--config", roots.config, "--input", roots.input} - }}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - roots := newStateTestRoots(t) - var stdout, stderr bytes.Buffer - code := RunWithOptions(tt.args(roots), &stdout, &stderr, newStateTestHarness().options()) - if code != 2 || stdout.Len() != 0 || stderr.Len() == 0 { - t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) - } - assertAbsent(t, roots.output) - assertAbsent(t, roots.debug) - }) - } -} - -func TestRecomputeStepCLIContract(t *testing.T) { - tests := []struct { - name string - configure func(*testing.T, stateTestRoots) - flags []string - wantCode int - wantOutput string - wantError string - }{ - { - name: "explicit step", - configure: func(t *testing.T, roots stateTestRoots) { - replaceStateTestConfigLine(t, roots.config, " artifacts:\n items:\n extract:\n module: test/extract\n merge:\n module: test/merge\n normalize:\n module: test/normalize\n", " steps:\n - id: chosen\n artifacts:\n items:\n extract:\n module: test/extract\n merge:\n module: test/merge\n normalize:\n module: test/normalize\n") - }, - flags: []string{"--resume", "--recompute-step", "chosen"}, - wantCode: 0, - wantOutput: "outputs=1", - }, - {name: "implicit default step", flags: []string{"--resume", "--recompute-step", "default"}, wantCode: 0, wantOutput: "outputs=1"}, - {name: "repeated flag", flags: []string{"--resume", "--recompute-step", "default", "--recompute-step", "default"}, wantCode: 2, wantError: "specified only once"}, - {name: "empty step", flags: []string{"--resume", "--recompute-step", ""}, wantCode: 2, wantError: "must not be empty"}, - {name: "unknown step", flags: []string{"--resume", "--recompute-step", "missing"}, wantCode: 1, wantError: "unknown pipeline step"}, - {name: "without resume", flags: []string{"--recompute-step", "default"}, wantCode: 2, wantError: "requires --resume"}, - { - name: "checkpoint recording disabled", - configure: func(t *testing.T, roots stateTestRoots) { - replaceStateTestConfigLine(t, roots.config, " enabled: true\n", " enabled: false\n") - }, - flags: []string{"--resume", "--recompute-step", "default"}, wantCode: 1, wantError: "cache.checkpoints.enabled", - }, - {name: "with only", flags: []string{"--resume", "--recompute-step", "default", "--only", "items"}, wantCode: 2, wantError: "cannot be combined with --only"}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - roots := newStateTestRoots(t) - if tt.configure != nil { - tt.configure(t, roots) - } - args := []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"} - args = append(args, tt.flags...) - var stdout, stderr bytes.Buffer - code := RunWithOptions(args, &stdout, &stderr, newStateTestHarness().options()) - if code != tt.wantCode || (tt.wantOutput != "" && !strings.Contains(stdout.String(), tt.wantOutput)) || (tt.wantError != "" && !strings.Contains(stderr.String(), tt.wantError)) { - t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) - } - }) - } -} - -func TestRunValidFailuresClassifyAndReportDebug(t *testing.T) { - tests := []struct { - name string - args func(stateTestRoots) []string - wantError string - wantDebug bool - }{ - {name: "unknown pipeline", args: func(roots stateTestRoots) []string { - return []string{"run", "missing", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"} - }, wantError: `pipeline "missing"`}, - {name: "unknown lane", args: func(roots stateTestRoots) []string { - return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--only", "missing", "--chunk_cache", "bypass", "--debug"} - }, wantError: `lane "missing"`, wantDebug: true}, - {name: "unreadable input", args: func(roots stateTestRoots) []string { - return []string{"run", "sample", "--config", roots.config, "--input", filepath.Join(filepath.Dir(roots.input), "unreadable.txt"), "--chunk_cache", "bypass", "--debug"} - }, wantError: "read input", wantDebug: true}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - roots := newStateTestRoots(t) - var stdout, stderr bytes.Buffer - code := RunWithOptions(tt.args(roots), &stdout, &stderr, newStateTestHarness().options()) - if code != 1 || stdout.Len() != 0 || !strings.Contains(stderr.String(), tt.wantError) { - t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) - } - if tt.wantDebug { - if !strings.Contains(stderr.String(), "debug=") { - t.Fatalf("stderr=%q, want debug path", stderr.String()) - } - onlyChildDir(t, roots.debug) - } else { - assertAbsent(t, roots.debug) - } - assertAbsent(t, roots.output) - }) - } -} - -func TestRunOnlyExecutesSelectedLanes(t *testing.T) { - roots := newStateTestRoots(t) - data, err := os.ReadFile(roots.config) - if err != nil { - t.Fatal(err) - } - data = []byte(replaceRequiredOnce(t, string(data), " output:\n module: test/output\n", " other:\n extract:\n module: test/extract\n output:\n module: test/output\n")) - if err := os.WriteFile(roots.config, data, 0o600); err != nil { - t.Fatal(err) - } - harness := newStateTestHarness() - var stdout, stderr bytes.Buffer - code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--only", "items", "--chunk_cache", "bypass"}, &stdout, &stderr, harness.options()) - if code != 0 || !strings.Contains(stdout.String(), "outputs=1") || stderr.Len() != 0 { - t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) - } - harness.mu.Lock() - extractCalls := harness.extractCalls - harness.mu.Unlock() - if extractCalls != 1 { - t.Fatalf("extract calls = %d, want only the selected lane", extractCalls) - } -} - -func TestRunStateRootsHonorEnvironmentFlagsAndDefaults(t *testing.T) { - t.Run("retired environment roots have no effect", func(t *testing.T) { - roots := newStateTestRoots(t) - environmentOutput := filepath.Join(t.TempDir(), "environment-output") - environmentDebug := filepath.Join(t.TempDir(), "environment-debug") - opts := newStateTestHarness().options() - opts.LookupEnv = lookupRunContractEnv(map[string]string{ - "NOTARIUS_OUTPUT_DIR": environmentOutput, - "NOTARIUS_DEBUG_DIR": environmentDebug, - }) - result := runWithStateRoots(t, roots, opts, nil) - if result.code != 0 { - t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr) - } - assertFile(t, filepath.Join(roots.output, filepath.Base(onlyChildDir(t, roots.output)), "result.json")) - onlyChildDir(t, roots.debug) - assertAbsent(t, environmentOutput) - assertAbsent(t, environmentDebug) - }) - - t.Run("command flags override environment", func(t *testing.T) { - roots := newStateTestRoots(t) - environmentOutput := filepath.Join(t.TempDir(), "environment-output") - environmentDebug := filepath.Join(t.TempDir(), "environment-debug") - flagOutput := filepath.Join(t.TempDir(), "flag-output") - flagDebug := filepath.Join(t.TempDir(), "flag-debug") - opts := newStateTestHarness().options() - opts.LookupEnv = lookupRunContractEnv(map[string]string{ - "NOTARIUS_OUTPUT_DIR": environmentOutput, - "NOTARIUS_DEBUG_DIR": environmentDebug, - }) - result := runWithStateRoots(t, roots, opts, []string{"--output-dir", flagOutput, "--debug-dir", flagDebug}) - if result.code != 0 { - t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr) - } - assertFile(t, filepath.Join(flagOutput, filepath.Base(onlyChildDir(t, flagOutput)), "result.json")) - onlyChildDir(t, flagDebug) - assertAbsent(t, environmentOutput) - assertAbsent(t, environmentDebug) - }) - - t.Run("built-in roots", func(t *testing.T) { - roots := newStateTestRoots(t) - data, err := os.ReadFile(roots.config) - if err != nil { - t.Fatal(err) - } - text := string(data) - text = replaceRequiredOnce(t, text, fmt.Sprintf("output:\n directory: %q\n", roots.output), "") - text = replaceRequiredOnce(t, text, fmt.Sprintf("debug:\n directory: %q\n", roots.debug), "") - if err := os.WriteFile(roots.config, []byte(text), 0o600); err != nil { - t.Fatal(err) - } - opts := newStateTestHarness().options() - result := runWithStateRoots(t, roots, opts, nil) - if result.code != 0 { - t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr) - } - configDirectory := filepath.Dir(roots.config) - assertFile(t, filepath.Join(configDirectory, "notarius-output", filepath.Base(onlyChildDir(t, filepath.Join(configDirectory, "notarius-output"))), "result.json")) - onlyChildDir(t, filepath.Join(configDirectory, "notarius-debug")) - }) -} - -func TestRunAppliesConfigurationOverlayAndRecordsSelection(t *testing.T) { - roots := newStateTestRoots(t) - overlayDirectory := filepath.Join(filepath.Dir(roots.config), "overlays") - overlayOutput := filepath.Join(overlayDirectory, "overlay-output") - overlayDebug := filepath.Join(overlayDirectory, "overlay-debug") - writeConfigKitFile(t, filepath.Join(overlayDirectory, "development.yml"), "output:\n directory: ./overlay-output\ndebug:\n directory: ./overlay-debug\n") - - var stdout, stderr bytes.Buffer - code := RunWithOptions([]string{ - "run", "sample", "--config", roots.config, "--config-overlay", "development", - "--input", roots.input, "--chunk_cache", "bypass", "--debug", - }, &stdout, &stderr, newStateTestHarness().options()) - if code != 0 || stderr.Len() != 0 { - t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) - } - assertFile(t, filepath.Join(overlayOutput, filepath.Base(onlyChildDir(t, overlayOutput)), "result.json")) - bundle := onlyChildDir(t, overlayDebug) - var invocation debugbundle.Invocation - readStateTestSummaryJSON(t, bundle, "invocation.json", &invocation) - if invocation.ConfigOverlay != "development" { - t.Fatalf("config overlay = %q, want development", invocation.ConfigOverlay) - } -} - -func TestRunRecordsDiscoveredConfigurationPath(t *testing.T) { - roots := newStateTestRoots(t) - work := t.TempDir() - selected := filepath.Join(work, "notarius.yml") - data, err := os.ReadFile(roots.config) - if err != nil { - t.Fatal(err) - } - if err := os.WriteFile(selected, data, 0600); err != nil { - t.Fatal(err) - } - previous, err := os.Getwd() - if err != nil { - t.Fatal(err) - } - if err := os.Chdir(work); err != nil { - t.Fatal(err) - } - t.Cleanup(func() { - if err := os.Chdir(previous); err != nil { - t.Error(err) - } - }) - var stdout, stderr bytes.Buffer - code := RunWithOptions([]string{"run", "sample", "--input", roots.input, "--chunk_cache", "bypass", "--debug"}, &stdout, &stderr, newStateTestHarness().options()) - if code != 0 { - t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) - } - var invocation debugbundle.Invocation - readStateTestSummaryJSON(t, onlyChildDir(t, roots.debug), "invocation.json", &invocation) - if invocation.ConfigPath != selected { - t.Fatalf("recorded config path = %q, want %q", invocation.ConfigPath, selected) - } -} - -func TestEffectiveConfigOverlayOmitsBaseSelection(t *testing.T) { - for _, overlay := range []string{"", "default"} { - if got := effectiveConfigOverlay(overlay); got != "" { - t.Fatalf("effectiveConfigOverlay(%q) = %q, want empty", overlay, got) - } - } - if got := effectiveConfigOverlay("development"); got != "development" { - t.Fatalf("effectiveConfigOverlay(development) = %q", got) - } -} - -func TestRunLLMProfileOverrideAndValidationUseInjectedBoundaries(t *testing.T) { - t.Run("one effective profile reaches the factory and modules", func(t *testing.T) { - roots := newStateTestRoots(t) - profileDir := writeRunContractProfiles(t, "override-profile") - prependRunContractConfig(t, roots, fmt.Sprintf("promptkit:\n profile_dir: %q\n", profileDir)) - harness := newStateTestHarness() - var factoryProfiles []string - opts := harness.options() - var factoryOverrides []LLMRuntimeOverrides - opts.LLMClientFactory = func(_ context.Context, _ config.Config, profileID string, overrides LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { - factoryProfiles = append(factoryProfiles, profileID) - factoryOverrides = append(factoryOverrides, overrides) - return nil, nil, nil - } - var stdout, stderr bytes.Buffer - code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--llm-profile", " override-profile "}, &stdout, &stderr, opts) - if code != 0 || stderr.Len() != 0 { - t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) - } - if len(factoryProfiles) != 1 || factoryProfiles[0] != "override-profile" { - t.Fatalf("factory profiles = %#v, want one override profile", factoryProfiles) - } - if len(factoryOverrides) != 1 || factoryOverrides[0].ReasoningEffort != nil { - t.Fatalf("factory overrides = %#v, want inherited reasoning", factoryOverrides) - } - harness.mu.Lock() - profiles := append([]string(nil), harness.moduleProfiles...) - harness.mu.Unlock() - if len(profiles) < 4 { - t.Fatalf("module profiles = %#v, want chunk and lane stage requests", profiles) - } - for _, profile := range profiles { - if profile != "override-profile" { - t.Fatalf("module profiles = %#v, want override on every request", profiles) - } - } - }) - - t.Run("runtime override applies to validators", func(t *testing.T) { - roots := newStateTestRoots(t) - profileDir := writeRunContractProfiles(t, "override-profile", "validator-profile") - prependRunContractConfig(t, roots, fmt.Sprintf("promptkit:\n profile_dir: %q\n", profileDir)) - harness := newStateTestHarness() - var validatorProfiles []string - opts := harness.options() - registerRunContractValidator(t, &opts, &validatorProfiles) - factoryProfiles := []string{} - opts.LLMClientFactory = func(_ context.Context, _ config.Config, profileID string, _ LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { - factoryProfiles = append(factoryProfiles, profileID) - return nil, nil, nil - } - var stdout, stderr bytes.Buffer - code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--llm-profile", "override-profile"}, &stdout, &stderr, opts) - if code != 0 || stderr.Len() != 0 { - t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) - } - if len(factoryProfiles) != 1 || factoryProfiles[0] != "override-profile" { - t.Fatalf("factory profiles = %#v, want one override profile", factoryProfiles) - } - if len(validatorProfiles) != 1 || validatorProfiles[0] != "override-profile" { - t.Fatalf("validator profiles = %#v, want runtime override", validatorProfiles) - } - }) - - t.Run("unknown profile is rejected without factory access", func(t *testing.T) { - roots := newStateTestRoots(t) - profileDir := writeRunContractProfiles(t, "override-profile") - prependRunContractConfig(t, roots, fmt.Sprintf("promptkit:\n profile_dir: %q\n", profileDir)) - factoryCalls := 0 - opts := newStateTestHarness().options() - opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { - factoryCalls++ - return nil, nil, nil - } - var stdout, stderr bytes.Buffer - code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--llm-profile", "missing-profile"}, &stdout, &stderr, opts) - if code != 1 || !strings.Contains(stderr.String(), "not configured") || factoryCalls != 0 || stdout.Len() != 0 { - t.Fatalf("code=%d stdout=%q stderr=%q factoryCalls=%d", code, stdout.String(), stderr.String(), factoryCalls) - } - }) - - t.Run("pipeline default is rejected before factory access", func(t *testing.T) { - roots := newStateTestRoots(t) - profileDir := writeRunContractProfiles(t, "configured-profile") - prependRunContractConfig(t, roots, fmt.Sprintf("promptkit:\n profile_dir: %q\n", profileDir)) - replaceStateTestConfigLine(t, roots.config, " sample:\n", " sample:\n llm_profile: missing-profile\n") - factoryCalls := 0 - opts := newStateTestHarness().options() - opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { - factoryCalls++ - return nil, nil, nil - } - var stdout, stderr bytes.Buffer - code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}, &stdout, &stderr, opts) - if code != 1 || !strings.Contains(stderr.String(), "not configured") || factoryCalls != 0 || stdout.Len() != 0 { - t.Fatalf("code=%d stdout=%q stderr=%q factoryCalls=%d", code, stdout.String(), stderr.String(), factoryCalls) - } - }) -} - -func TestRunReasoningEffortOverrideReachesFactory(t *testing.T) { - tests := []struct { - name string - flags []string - wantValue string - wantSet bool - }{ - {name: "inherit"}, - {name: "replace", flags: []string{"--reasoning-effort", " focused "}, wantValue: "focused", wantSet: true}, - {name: "clear", flags: []string{"--clear-reasoning-effort"}, wantSet: true}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - roots := newStateTestRoots(t) - opts := newStateTestHarness().options() - var got []LLMRuntimeOverrides - opts.LLMClientFactory = func(_ context.Context, _ config.Config, _ string, overrides LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { - got = append(got, overrides) - return nil, nil, nil - } - args := append([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}, tt.flags...) - var stdout, stderr bytes.Buffer - if code := RunWithOptions(args, &stdout, &stderr, opts); code != 0 || stderr.Len() != 0 { - t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) - } - if len(got) != 1 { - t.Fatalf("factory overrides = %#v, want one call", got) - } - if !tt.wantSet { - if got[0].ReasoningEffort != nil { - t.Fatalf("reasoning effort = %q, want inherit", *got[0].ReasoningEffort) - } - return - } - if got[0].ReasoningEffort == nil || *got[0].ReasoningEffort != tt.wantValue { - t.Fatalf("reasoning effort = %#v, want %q", got[0].ReasoningEffort, tt.wantValue) - } - }) - } -} - -func TestRunReasoningEffortOverrideRejectsInvalidSyntax(t *testing.T) { - tests := []struct { - name string - flags []string - wantError string - }{ - { - name: "mutually exclusive controls", - flags: []string{"--reasoning-effort", "focused", "--clear-reasoning-effort"}, - wantError: "cannot be combined", - }, - { - name: "empty replacement", - flags: []string{"--reasoning-effort", " "}, - wantError: "must not be empty", - }, - { - name: "duplicate replacement", - flags: []string{"--reasoning-effort", "low", "--reasoning-effort", "high"}, - wantError: "may be specified only once", - }, - { - name: "missing replacement", - flags: []string{"--reasoning-effort"}, - wantError: "flag needs an argument", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - roots := newStateTestRoots(t) - args := append([]string{"run", "sample", "--config", roots.config, "--input", roots.input}, tt.flags...) - var stdout, stderr bytes.Buffer - code := RunWithOptions(args, &stdout, &stderr, newStateTestHarness().options()) - if code != 2 || stdout.Len() != 0 || !strings.Contains(stderr.String(), tt.wantError) { - t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) - } - assertNoRunState(t, roots) - }) - } -} - -func TestReasoningEffortOverrideSeparatesCheckpointIdentities(t *testing.T) { - replacement := " focused " - cleared := "" - states := []struct { - name string - overrides LLMRuntimeOverrides - wantValue string - wantSet bool - }{ - {name: "inherit"}, - {name: "replace", overrides: LLMRuntimeOverrides{ReasoningEffort: &replacement}, wantValue: "focused", wantSet: true}, - {name: "clear", overrides: LLMRuntimeOverrides{ReasoningEffort: &cleared}, wantValue: "", wantSet: true}, - } - digests := make(map[string]string, len(states)) - for _, state := range states { - fingerprints := runtimeOverrideFingerprints("", "", state.overrides) - var value string - var found bool - for _, fingerprint := range fingerprints { - if fingerprint.Name == "reasoning_effort_override" { - value, found = fingerprint.Value, true - } - } - if found != state.wantSet || (found && value != state.wantValue) { - t.Fatalf("%s fingerprint found=%t value=%q, want found=%t value=%q", state.name, found, value, state.wantSet, state.wantValue) - } - identity, err := checkpoint.NewIdentity(checkpoint.IdentityInput{ - Pipeline: pipeline.ResolvedPipeline{ID: "sample", Digest: "sha256:pipeline", Input: pipeline.Binding("test/input")}, - RawInputDigest: "sha256:input", - RuntimeOverrides: fingerprints, - }) - if err != nil { - t.Fatal(err) - } - digests[state.name] = identity.Digest - } - if digests["inherit"] == digests["replace"] || digests["inherit"] == digests["clear"] || digests["replace"] == digests["clear"] { - t.Fatalf("checkpoint identity digests are not distinct: %#v", digests) - } -} - -func TestEffectiveLLMProfileIDsAreSortedDeduplicatedAndLLMOnly(t *testing.T) { - resolved := pipeline.ResolvedPipeline{ - Input: pipeline.ModuleBinding{LLMProfile: "input-profile"}, - InputExecutionClass: contracts.ExecutionClassLLMBacked, - Chunk: pipeline.ModuleBinding{LLMProfile: " zeta "}, - ChunkExecutionClass: contracts.ExecutionClassLLMBacked, - Steps: []pipeline.ResolvedPipelineStep{{ - ID: "default", - ArtifactLanes: []pipeline.ResolvedArtifactLane{{ - Extract: pipeline.ModuleBinding{LLMProfile: "alpha"}, - ExtractExecutionClass: contracts.ExecutionClassLLMBacked, - Merge: pipeline.ModuleBinding{LLMProfile: "deterministic-merge"}, - MergeExecutionClass: contracts.ExecutionClassDeterministic, - Normalize: pipeline.ModuleBinding{LLMProfile: " gamma "}, - NormalizeExecutionClass: contracts.ExecutionClassLLMBacked, - }}, - }}, - ValidatorChains: []pipeline.ResolvedValidatorChain{{Validators: []pipeline.ResolvedValidator{ - {Binding: pipeline.ModuleBinding{LLMProfile: "deterministic-profile"}, ExecutionClass: contracts.ExecutionClassDeterministic}, - {Binding: pipeline.ModuleBinding{LLMProfile: "beta"}, ExecutionClass: contracts.ExecutionClassLLMBacked}, - }}}, - Output: pipeline.ModuleBinding{LLMProfile: "output-profile"}, - OutputExecutionClass: contracts.ExecutionClassLLMBacked, - } - got := effectiveLLMProfileIDs(resolved) - want := []string{"alpha", "beta", "gamma", "input-profile", "output-profile", "zeta"} - if strings.Join(got, ",") != strings.Join(want, ",") { - t.Fatalf("effective profiles = %#v, want %#v", got, want) - } -} - -func TestRunSessionIDUsesEffectiveValueForPromptRequests(t *testing.T) { - for _, tt := range []struct { - name string - args []string - want string - }{ - {name: "derived default"}, - {name: "explicit trimmed value", args: []string{"--session-id", " explicit-session "}, want: "explicit-session"}, +func TestRunInvalidSyntaxCreatesNoRunState(t *testing.T) { + for _, flags := range [][]string{ + {"--unknown"}, + {"--output-dir="}, + {"--overlay"}, + {"--llm-temperature=invalid"}, + {"--input-file="}, } { - t.Run(tt.name, func(t *testing.T) { - roots := newStateTestRoots(t) - want := tt.want - if want == "" { - rawInput, err := os.ReadFile(roots.input) - if err != nil { - t.Fatal(err) - } - want, err = resolvePromptSessionID("", "test/input", rawInput) - if err != nil { - t.Fatal(err) - } + t.Run(strings.Join(flags, " "), func(t *testing.T) { + fixture := newStateRunFixture(t) + result := fixture.run(t, flags...) + if result.status != 2 || result.stdout != "" || result.stderr == "" { + t.Fatalf("syntax error=%#v", result) } - harness := newStateTestHarness() - args := append([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--debug"}, tt.args...) - var stdout, stderr bytes.Buffer - code := RunWithOptions(args, &stdout, &stderr, harness.options()) - if code != 0 || stderr.Len() != 0 { - t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) - } - harness.mu.Lock() - sessions := append([]string(nil), harness.sessionIDs...) - harness.mu.Unlock() - if len(sessions) < 4 { - t.Fatalf("session IDs = %#v, want all prompt-facing module requests", sessions) - } - for _, session := range sessions { - if session != want { - t.Fatalf("session IDs = %#v, want %q", sessions, want) - } - } - var manifest artifacts.RunManifest - readStateTestSummaryJSON(t, onlyChildDir(t, roots.debug), "run-manifest.json", &manifest) - if session, ok := manifest.Metadata["session_id"]; !ok || session != want { - t.Fatalf("manifest session = %#v, want %q; metadata = %#v", session, want, manifest.Metadata) + assertAbsent(t, fixture.outputRoot) + assertAbsent(t, fixture.cacheRoot) + assertAbsent(t, fixture.debugRoot) + if len(fixture.client.requestsSnapshot()) != 0 { + t.Fatal("syntax error reached provider") } }) } } -func TestRunFactoryAndPreparationFailuresAreProcessFailures(t *testing.T) { - t.Run("LLM factory", func(t *testing.T) { - roots := newStateTestRoots(t) - opts := newStateTestHarness().options() - opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { - return nil, nil, errors.New("injected LLM factory failure") +func TestRunStateRootsIgnoreRetiredEnvironmentAndHonorNativeFlags(t *testing.T) { + fixture := newStateRunFixture(t) + environmentOutput, environmentDebug := filepath.Join(t.TempDir(), "environment-output"), filepath.Join(t.TempDir(), "environment-debug") + fixture.options.LookupEnv = func(name string) (string, bool) { + switch name { + case "NOTARIUS_OUTPUT_DIR": + return environmentOutput, true + case "NOTARIUS_DEBUG_DIR": + return environmentDebug, true } - var stdout, stderr bytes.Buffer - code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}, &stdout, &stderr, opts) - if code != 1 || !strings.Contains(stderr.String(), "injected LLM factory failure") || stdout.Len() != 0 { - t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) - } - }) - - t.Run("pipeline preparation", func(t *testing.T) { - roots := newStateTestRoots(t) - data, err := os.ReadFile(roots.config) - if err != nil { - t.Fatal(err) - } - data = []byte(replaceRequiredOnce(t, string(data), "module: test/extract", "module: test/failing-extract")) - if err := os.WriteFile(roots.config, data, 0o600); err != nil { - t.Fatal(err) - } - opts := newStateTestHarness().options() - if err := pipeline.RegisterExtractorBuilder(opts.Registries.Extractors, pipeline.ModuleSpec{Key: "test/failing-extract", Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: stateTestArtifactKind}, func(map[string]any) error { return nil }, func(pipeline.BuildRequest) (contracts.Extractor[stateTestArtifact], error) { - return nil, errors.New("injected extractor construction failure") - }); err != nil { - t.Fatal(err) - } - opts.Catalog = catalogFromRegistries(opts.Registries) - var stdout, stderr bytes.Buffer - code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}, &stdout, &stderr, opts) - if code != 1 || !strings.Contains(stderr.String(), "injected extractor construction failure") || stdout.Len() != 0 { - t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) - } - }) + return "", false + } + first := fixture.run(t, "--debug", "--cache", "bypass") + if first.status != 0 { + t.Fatalf("configured roots=%#v", first) + } + assertFile(t, filepath.Join(first.report.OutputDirectory, "lanes", "spells.json")) + assertFile(t, filepath.Join(first.report.DebugDirectory, "outcome.json")) + flagOutput, flagDebug := filepath.Join(t.TempDir(), "flag-output"), filepath.Join(t.TempDir(), "flag-debug") + second := fixture.run(t, "--debug", "--cache", "bypass", "--output-dir", flagOutput, "--debug-dir", flagDebug) + if second.status != 0 || !strings.HasPrefix(second.report.OutputDirectory, flagOutput+string(os.PathSeparator)) || !strings.HasPrefix(second.report.DebugDirectory, flagDebug+string(os.PathSeparator)) { + t.Fatalf("flag roots=%#v", second) + } + assertAbsent(t, environmentOutput) + assertAbsent(t, environmentDebug) } -func TestRunWarningsRemainSuccessfulAndReachDurableSurfaces(t *testing.T) { - roots := newStateTestRoots(t) - harness := newStateTestHarness() - harness.includeWarnings = true - harness.includeWarningFile = true - harness.chunkDiagnostics = []contracts.ProducerDiagnostic{stateTestDiagnostic("chunk", "contract-warning", "warning retained")} - var stdout, stderr bytes.Buffer - code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--debug"}, &stdout, &stderr, harness.options()) - if code != 0 || !strings.Contains(stdout.String(), "outputs=1") || !strings.Contains(stderr.String(), "1 warning group(s), 1 occurrence(s)") || !strings.Contains(stderr.String(), "warnings.json") { - t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) - } - outputPath := filepath.Join(onlyChildDir(t, roots.output), "result.json") - output, err := os.ReadFile(outputPath) - if err != nil || !strings.Contains(string(output), "contract-warning") { - t.Fatalf("durable output = %q, %v", output, err) - } - assertFile(t, filepath.Join(filepath.Dir(outputPath), "warnings.json")) - bundle := onlyChildDir(t, roots.debug) - var diagnostics contracts.DiagnosticCollection - readStateTestSummaryJSON(t, bundle, "final-diagnostics.json", &diagnostics) - if len(diagnostics.Groups) != 1 || diagnostics.Groups[0].ReasonCode != "contract-warning" { - t.Fatalf("debug diagnostics = %#v", diagnostics) +func TestRunDebugDirectoryAloneDoesNotEnableCapture(t *testing.T) { + fixture := newStateRunFixture(t) + debug := filepath.Join(t.TempDir(), "explicit-debug") + result := fixture.run(t, "--cache", "bypass", "--debug-dir", debug) + if result.status != 0 || result.report.DebugDirectory != "" { + t.Fatalf("run=%#v", result) } + assertAbsent(t, debug) + assertAbsent(t, fixture.debugRoot) } -func runWithStateRoots(t *testing.T, roots stateTestRoots, opts Options, extra []string) stateTestResult { - t.Helper() - args := []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--debug"} - args = append(args, extra...) - var stdout, stderr bytes.Buffer - return stateTestResult{code: RunWithOptions(args, &stdout, &stderr, opts), stdout: stdout.String(), stderr: stderr.String()} -} - -func lookupRunContractEnv(values map[string]string) func(string) (string, bool) { - return func(name string) (string, bool) { - value, ok := values[name] - return value, ok +func TestRunUsesDeclarationRelativeOverlayRootsAndCapturesSelection(t *testing.T) { + fixture := newStateRunFixture(t) + directory := filepath.Join(filepath.Dir(fixture.configPath), "overlays") + path := filepath.Join(directory, "development.yml") + writeConfigKitFile(t, path, "output: {directory: ./overlay-output}\ndebug: {directory: ./overlay-debug}\n") + result := fixture.run(t, "--debug", "--cache", "bypass", "--overlay", "development") + if result.status != 0 || filepath.Dir(result.report.OutputDirectory) != filepath.Join(directory, "overlay-output") || filepath.Dir(result.report.DebugDirectory) != filepath.Join(directory, "overlay-debug") { + t.Fatalf("overlay roots=%#v", result) } + content, err := os.ReadFile(filepath.Join(result.report.DebugDirectory, "configuration", "sources.json")) + if err != nil || !strings.Contains(string(content), path) { + t.Fatalf("captured overlay selection=%s %v", content, err) + } + assertAbsent(t, fixture.outputRoot) + assertAbsent(t, fixture.debugRoot) } -func prependRunContractConfig(t *testing.T, roots stateTestRoots, prefix string) { - t.Helper() - data, err := os.ReadFile(roots.config) +func TestRunCapturesDiscoveredConfigurationPath(t *testing.T) { + fixture := newStateRunFixture(t) + root := filepath.Dir(fixture.configPath) + selected := filepath.Join(root, "notarius.yml") + content, err := os.ReadFile(fixture.configPath) if err != nil { t.Fatal(err) } - if err := os.WriteFile(roots.config, append([]byte(prefix), data...), 0o600); err != nil { + if err := os.WriteFile(selected, content, 0600); err != nil { t.Fatal(err) } -} - -func writeRunContractProfiles(t *testing.T, ids ...string) string { - t.Helper() - dir := t.TempDir() - for _, id := range ids { - profile := fmt.Sprintf("profiles:\n %s:\n endpoint: http://127.0.0.1:1/v1\n model: %s-model\n", id, id) - if err := os.WriteFile(filepath.Join(dir, id+".yaml"), []byte(profile), 0o600); err != nil { - t.Fatal(err) - } + t.Chdir(root) + var stdout, stderr bytes.Buffer + code := RunWithOptions([]string{"run", "dnd-session", "--input-file", fixture.inputPath, "--debug", "--cache", "bypass", "--format", "json"}, &stdout, &stderr, fixture.options) + if code != 0 { + t.Fatalf("status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } - return dir -} - -func registerRunContractValidator(t *testing.T, opts *Options, profiles *[]string) { - t.Helper() - if err := pipeline.RegisterTypedValidatorBuilder(opts.Registries.Validators, stateTestArtifactKind, pipeline.ValidatorSpec{Key: "run-contract-validator", ExecutionClass: contracts.ExecutionClassLLMBacked}, func(map[string]any) error { return nil }, func(pipeline.BuildRequest) (contracts.TypedValidator[stateTestArtifact], error) { - return runContractValidator{profiles: profiles}, nil - }); err != nil { - t.Fatal(err) + sourceBytes, err := os.ReadFile(filepath.Join(onlyChildDir(t, fixture.debugRoot), "configuration", "sources.json")) + if err != nil || !strings.Contains(string(sourceBytes), selected) { + t.Fatalf("selected configuration=%s %v", sourceBytes, err) } - if err := opts.Registries.ValidatorChains.Register(pipeline.ValidatorChainMapping{Stage: pipeline.StageExtract, Module: "test/extract", Validators: []pipeline.ModuleBinding{{Module: "run-contract-validator", LLMProfile: "validator-profile"}}}); err != nil { - t.Fatal(err) - } - opts.Catalog = catalogFromRegistries(opts.Registries) -} - -type runContractValidator struct { - profiles *[]string -} - -func (v runContractValidator) Name() string { return "run-contract-validator" } - -func (v runContractValidator) ExecutionClass() contracts.ExecutionClass { - return contracts.ExecutionClassLLMBacked -} - -func (v runContractValidator) Validate(_ context.Context, req contracts.TypedValidationRequest[stateTestArtifact]) (contracts.ValidationResult, error) { - *v.profiles = append(*v.profiles, req.LLMProfile) - return contracts.ValidationResult{Approved: true}, nil } diff --git a/internal/cli/state_hardening_test.go b/internal/cli/state_hardening_test.go index b5b2f609..33472a57 100644 --- a/internal/cli/state_hardening_test.go +++ b/internal/cli/state_hardening_test.go @@ -8,717 +8,275 @@ import ( "fmt" "os" "path/filepath" - "runtime" + "reflect" "strings" - "sync" "testing" "time" - "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" - "gitea.maximumdirect.net/eric/notarius/internal/core/config" - "gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle" - "gitea.maximumdirect.net/eric/notarius/internal/core/source" - "gitea.maximumdirect.net/eric/notarius/internal/framework/chunkplan" - "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" - frameworkdebug "gitea.maximumdirect.net/eric/notarius/internal/framework/debug" - "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" + "go.maximumdirect.net/promptkit" ) -const stateTestDigest = "sha256:e511d8906649b78eb639b11215fa57a9652a1a64f4aefa3ed68320dbda46f439" +type stateRunFixture struct { + configPath, inputPath, outputRoot, cacheRoot, debugRoot string + client *productionFakeLLMClient + options Options +} -func TestRunStateSurfaceMatrix(t *testing.T) { +func newStateRunFixture(t *testing.T) *stateRunFixture { + t.Helper() + root := t.TempDir() + fixture := &stateRunFixture{ + configPath: filepath.Join(root, "config.yml"), + inputPath: repositoryPath("examples", "seriatim-minimal-transcript.json"), + outputRoot: filepath.Join(root, "output"), cacheRoot: filepath.Join(root, "cache"), debugRoot: filepath.Join(root, "debug"), + client: &productionFakeLLMClient{}, + } + contents := fmt.Sprintf(`version: 6 +output: {directory: %q} +cache_directory: %q +debug: {directory: %q} +pipelines: + dnd-session: + input: {module: seriatim} + chunk: {module: generic} + artifacts: + spells: + extract: {module: dnd/spells} +`, fixture.outputRoot, fixture.cacheRoot, fixture.debugRoot) + productionProviderConfiguration + if err := os.WriteFile(fixture.configPath, []byte(contents), 0600); err != nil { + t.Fatal(err) + } + fixture.options = productionRunOptions(t, fixture.client) + nextID := 0 + fixture.options.RunIDGenerator = func(time.Time) (string, error) { nextID++; return fmt.Sprintf("run-%d", nextID), nil } + return fixture +} + +type stateRunResult struct { + status int + stdout, stderr string + report runResult +} + +func (fixture *stateRunFixture) run(t *testing.T, flags ...string) stateRunResult { + t.Helper() + args := append([]string{"run", "dnd-session", "--config", fixture.configPath, "--input-file", fixture.inputPath, "--format", "json"}, flags...) + var stdout, stderr bytes.Buffer + result := stateRunResult{status: RunWithOptions(args, &stdout, &stderr, fixture.options), stdout: stdout.String(), stderr: stderr.String()} + if result.status == 0 { + if err := json.Unmarshal(stdout.Bytes(), &result.report); err != nil { + t.Fatalf("completion=%q: %v", stdout.String(), err) + } + } + return result +} + +func TestRunStateSurfacesFollowDebugAndNativeCacheModes(t *testing.T) { for _, debug := range []bool{false, true} { - for _, resume := range []bool{false, true} { - for _, mode := range []string{"auto", "bypass", "refresh"} { - name := fmt.Sprintf("debug=%t/resume=%t/cache=%s", debug, resume, mode) - t.Run(name, func(t *testing.T) { - roots := newStateTestRoots(t) - harness := newStateTestHarness() - opts := harness.options() - var storeRoots []string - opts.ChunkPlanStoreFactory = func(root string) (pipeline.ChunkPlanStore, error) { - storeRoots = append(storeRoots, root) - return chunkplan.NewFilesystemStore(root) + for _, mode := range []string{"auto", "bypass", "refresh"} { + t.Run(fmt.Sprintf("debug=%t/cache=%s", debug, mode), func(t *testing.T) { + fixture := newStateRunFixture(t) + result := fixture.run(t, "--cache", mode, fmt.Sprintf("--debug=%t", debug)) + if result.status != 0 || result.report.ArtifactCount != 1 || result.report.FallbackCount != 0 { + t.Fatalf("run=%#v", result) + } + assertFile(t, filepath.Join(result.report.OutputDirectory, "lanes", "spells.json")) + if !strings.HasPrefix(result.report.OutputDirectory, fixture.outputRoot+string(os.PathSeparator)) { + t.Fatalf("output escaped selected root: %#v", result.report) + } + if mode == "bypass" { + assertAbsent(t, fixture.cacheRoot) + } else { + files := readTree(t, fixture.cacheRoot) + if len(files) != 1 { + t.Fatalf("accepted cache files=%#v", files) } - result := runStateTest(t, roots, opts, debug, resume, mode) - if result.code != 0 { - t.Fatalf("code=%d stderr=%q", result.code, result.stderr) - } - assertStateTestOutput(t, roots.output) - if mode == "bypass" { - assertAbsent(t, roots.plans) - if len(storeRoots) != 0 { - t.Fatalf("chunk plan store roots = %v, want none", storeRoots) - } - } else { - assertFile(t, filepath.Join(roots.plans, strings.TrimPrefix(stateTestDigest, "sha256:"), "plan.json")) - if len(storeRoots) != 1 || storeRoots[0] != roots.plans { - t.Fatalf("chunk plan store roots = %v, want [%q]", storeRoots, roots.plans) + for name, content := range files { + if !strings.HasSuffix(name, ".json") || !json.Valid(content) { + t.Fatalf("invalid accepted cache entry %s", name) } } - assertAnyFile(t, roots.checkpoints) - if debug { - bundle := onlyChildDir(t, roots.debug) - assertFile(t, filepath.Join(bundle, "summary", "invocation.json")) - assertAnyFile(t, filepath.Join(bundle, "trace")) - assertRestrictedTree(t, roots.debug) - } else { - assertAbsent(t, roots.debug) + } + if debug { + if result.report.DebugDirectory != filepath.Join(fixture.debugRoot, filepath.Base(result.report.OutputDirectory)) { + t.Fatalf("run identity differs between roots: %#v", result.report) } - }) - } + assertFile(t, filepath.Join(result.report.DebugDirectory, "outcome.json")) + } else { + assertAbsent(t, fixture.debugRoot) + if result.report.DebugDirectory != "" { + t.Fatalf("disabled debug path reported: %#v", result.report) + } + } + }) } } } -func TestRunKeepsStateRootsIndependentAndReusesSelectedCheckpointRoot(t *testing.T) { - roots := newStateTestRoots(t) - harness := newStateTestHarness() - first := runStateTest(t, roots, harness.options(), true, false, "auto") - if first.code != 0 { - t.Fatalf("first run code=%d stderr=%q", first.code, first.stderr) +func TestRunKeepsStateRootsIndependentWhenReusingAcceptedArtifacts(t *testing.T) { + fixture := newStateRunFixture(t) + first := fixture.run(t, "--debug", "--cache", "auto") + if first.status != 0 { + t.Fatalf("first=%#v", first) } - planPath := filepath.Join(roots.plans, strings.TrimPrefix(stateTestDigest, "sha256:"), "plan.json") - initialPlan, err := os.ReadFile(planPath) - if err != nil { - t.Fatal(err) + entries := readTree(t, fixture.cacheRoot) + calls := len(fixture.client.requestsSnapshot()) + second := fixture.run(t, "--cache", "auto") + if second.status != 0 || len(fixture.client.requestsSnapshot()) != calls { + t.Fatalf("reused=%#v provider calls=%d/%d", second, len(fixture.client.requestsSnapshot()), calls) } - firstBundle := onlyChildDir(t, roots.debug) - - second := runStateTest(t, roots, harness.options(), false, false, "auto") - if second.code != 0 { - t.Fatalf("second run code=%d stderr=%q", second.code, second.stderr) + if !sameFiles(entries, readTree(t, fixture.cacheRoot)) { + t.Fatal("accepted hit rewrote cached artifact") } - if harness.chunkCalls != 1 { - t.Fatalf("chunk calls after debug toggle = %d, want 1", harness.chunkCalls) + if first.report.OutputDirectory == second.report.OutputDirectory { + t.Fatal("rerun did not get its own output directory") } - if harness.extractCalls != 2 { - t.Fatalf("extract calls after two recording-only runs = %d, want 2", harness.extractCalls) - } - if got, err := os.ReadFile(planPath); err != nil || !bytes.Equal(got, initialPlan) { - t.Fatalf("chunk plan changed after debug toggle: %v", err) - } - if _, err := os.Stat(firstBundle); err != nil { - t.Fatalf("initial debug bundle was removed: %v", err) - } - - checkpointRoot := roots.checkpoints - extractCallsBeforeResume := harness.extractCalls - seed := runStateTest(t, roots, harness.options(), false, true, "auto") - if seed.code != 0 { - t.Fatalf("checkpoint seed code=%d stderr=%q", seed.code, seed.stderr) - } - if harness.extractCalls != extractCallsBeforeResume { - t.Fatalf("extract calls after reusing recording-only checkpoint = %d, want %d", harness.extractCalls, extractCallsBeforeResume) - } - extractCalls := harness.extractCalls - checkpointFiles := readTree(t, checkpointRoot) - reused := runStateTest(t, roots, harness.options(), false, true, "auto") - if reused.code != 0 { - t.Fatalf("checkpoint reuse code=%d stderr=%q", reused.code, reused.stderr) - } - if harness.extractCalls != extractCalls { - t.Fatalf("extract calls after checkpoint reuse = %d, want %d", harness.extractCalls, extractCalls) - } - if got := readTree(t, checkpointRoot); !sameFiles(got, checkpointFiles) { - t.Fatal("reused checkpoint was rewritten") + assertFile(t, filepath.Join(first.report.OutputDirectory, "lanes", "spells.json")) + assertFile(t, filepath.Join(second.report.OutputDirectory, "lanes", "spells.json")) + assertFile(t, filepath.Join(first.report.DebugDirectory, "outcome.json")) + target, err := os.Readlink(filepath.Join(fixture.debugRoot, "latest")) + if err != nil || target != filepath.Base(first.report.DebugDirectory) { + t.Fatalf("disabled capture changed latest: %q %v", target, err) } } -func TestRunRecomputesOnlyAfterExplicitChunkPlanRemoval(t *testing.T) { - roots := newStateTestRoots(t) - harness := newStateTestHarness() - first := runStateTest(t, roots, harness.options(), true, false, "auto") - if first.code != 0 { - t.Fatalf("first run code=%d stderr=%q", first.code, first.stderr) - } - firstOutput := onlyChildDir(t, roots.output) - firstBundle := onlyChildDir(t, roots.debug) - entry := filepath.Join(roots.plans, strings.TrimPrefix(stateTestDigest, "sha256:")) - if err := os.RemoveAll(entry); err != nil { - t.Fatal(err) - } - second := runStateTest(t, roots, harness.options(), false, false, "auto") - if second.code != 0 { - t.Fatalf("second run code=%d stderr=%q", second.code, second.stderr) - } - if harness.chunkCalls != 2 { - t.Fatalf("chunk calls = %d, want 2 after removing exact cache entry", harness.chunkCalls) - } - assertFile(t, filepath.Join(firstOutput, "result.json")) - assertFile(t, filepath.Join(firstBundle, "summary", "run-report.json")) -} - -func TestRunRetainsDebugBundlesAcrossFailures(t *testing.T) { - t.Run("configuration failure precedes allocation", func(t *testing.T) { - root := filepath.Join(t.TempDir(), "debug") - var stdout, stderr bytes.Buffer - code := RunWithOptions([]string{"run", "sample", "--config", filepath.Join(t.TempDir(), "missing.yml"), "--input", "missing", "--debug", "--debug-dir", root}, &stdout, &stderr, newStateTestHarness().options()) - if code != 1 || !strings.Contains(stderr.String(), "source.missing") { - t.Fatalf("code=%d stderr=%q", code, stderr.String()) - } - assertAbsent(t, root) - }) - - for _, failure := range []struct { - name string - expected string - setup func(*testing.T, stateTestRoots, *stateTestHarness) Options - }{ - {"resolution", "pipeline \"missing\"", func(t *testing.T, roots stateTestRoots, h *stateTestHarness) Options { return h.options() }}, - {"pipeline", "synthetic extraction failure", func(t *testing.T, roots stateTestRoots, h *stateTestHarness) Options { - h.extractErr = errors.New("synthetic extraction failure") - return h.options() - }}, - {"output", "create output parent", func(t *testing.T, roots stateTestRoots, h *stateTestHarness) Options { - if err := os.WriteFile(roots.output, []byte("not a directory"), 0o600); err != nil { - t.Fatal(err) +func TestRunRetainsCaptureAcrossResolutionProviderAndPublicationFailures(t *testing.T) { + for _, failure := range []string{"resolution", "provider", "publication"} { + t.Run(failure, func(t *testing.T) { + fixture := newStateRunFixture(t) + flags := []string{"--debug", "--cache", "bypass", "--llm-transient-retries=0"} + fragment := "" + switch failure { + case "resolution": + flags = append(flags, "--lane", "missing") + fragment = "missing" + case "provider": + fixture.options.LLMClient = stateFailureClient{cause: errors.New("native provider failure")} + fragment = "native provider failure" + case "publication": + if err := os.WriteFile(fixture.outputRoot, []byte("not a directory"), 0600); err != nil { + t.Fatal(err) + } + fragment = fixture.outputRoot } - return h.options() - }}, - {"summary", "write debug invocation metadata", func(t *testing.T, roots stateTestRoots, h *stateTestHarness) Options { - opts := h.options() - opts.DebugRecorderFactory = func(traceRoot string) (pipeline.DebugRecorder, error) { - if err := os.RemoveAll(filepath.Join(filepath.Dir(traceRoot), "summary")); err != nil { - return nil, err - } - if err := os.WriteFile(filepath.Join(filepath.Dir(traceRoot), "summary"), []byte("blocked"), 0o600); err != nil { - return nil, err - } - return frameworkdebug.NewFilesystemRecorder(traceRoot) + result := fixture.run(t, flags...) + if result.status == 0 || !strings.Contains(result.stderr, fragment) || result.stdout != "" { + t.Fatalf("failed run=%#v", result) } - return opts - }}, - {"trace", "trace unavailable", func(t *testing.T, roots stateTestRoots, h *stateTestHarness) Options { - opts := h.options() - opts.DebugRecorderFactory = func(string) (pipeline.DebugRecorder, error) { return failingDebugRecorder{}, nil } - return opts - }}, - } { - t.Run(failure.name, func(t *testing.T) { - roots := newStateTestRoots(t) - harness := newStateTestHarness() - opts := failure.setup(t, roots, harness) - failureStderr := "" - if failure.name == "resolution" { - var stdout, stderr bytes.Buffer - code := RunWithOptions([]string{"run", "missing", "--config", roots.config, "--input", roots.input, "--debug"}, &stdout, &stderr, opts) - if code != 1 { - t.Fatalf("code=%d stderr=%q", code, stderr.String()) - } - failureStderr = stderr.String() + bundle := onlyChildDir(t, fixture.debugRoot) + if !strings.Contains(result.stderr, bundle) || !strings.Contains(readAllFiles(t, bundle), fragment) { + t.Fatalf("failure evidence or path missing: %#v", result) + } + assertFile(t, filepath.Join(bundle, "outcome.json")) + if failure != "publication" { + assertAbsent(t, fixture.outputRoot) } else { - result := runStateTest(t, roots, opts, true, false, "bypass") - if result.code != 1 { - t.Fatalf("code=%d stderr=%q", result.code, result.stderr) + data, err := os.ReadFile(fixture.outputRoot) + if err != nil || string(data) != "not a directory" { + t.Fatalf("blocked output parent changed: %s %v", data, err) } - failureStderr = result.stderr - } - if !strings.Contains(failureStderr, failure.expected) || !strings.Contains(failureStderr, "debug=") { - t.Fatalf("stderr=%q, want %q and debug path", failureStderr, failure.expected) - } - bundle := onlyChildDir(t, roots.debug) - if !strings.Contains(readAllFiles(t, bundle), "synthetic") && failure.name == "pipeline" { - t.Fatal("pipeline failure was not retained in debug bundle") } }) } } -func TestRunDebugArtifactsRedactSecretsButRetainApplicationData(t *testing.T) { - roots := newStateTestRoots(t) - t.Setenv("STATE_TEST_UNRELATED_ENV", "HOST_ONLY_SENTINEL") - if err := os.WriteFile(filepath.Join(filepath.Dir(roots.input), "unrelated.txt"), []byte("HOST_ONLY_FILE_SENTINEL"), 0o600); err != nil { +func TestRunCaptureContainsSelectedApplicationDataNotUnrelatedHostData(t *testing.T) { + fixture := newStateRunFixture(t) + t.Setenv("NOTARIUS_TEST_UNRELATED_ENV", "HOST_ONLY_SENTINEL") + if err := os.WriteFile(filepath.Join(filepath.Dir(fixture.configPath), "unrelated.txt"), []byte("HOST_ONLY_FILE_SENTINEL"), 0600); err != nil { t.Fatal(err) } - harness := newStateTestHarness() - result := runStateTest(t, roots, harness.options(), true, false, "bypass") - if result.code != 0 { - t.Fatalf("code=%d stderr=%q", result.code, result.stderr) + result := fixture.run(t, "--debug", "--cache", "bypass") + if result.status != 0 { + t.Fatalf("run=%#v", result) } - bundle := onlyChildDir(t, roots.debug) - summary := readAllFiles(t, filepath.Join(bundle, "summary")) - trace := readAllFiles(t, filepath.Join(bundle, "trace")) - for _, forbidden := range []string{"sk-secretvalue", "Bearer secretvalue", "HOST_ONLY_SENTINEL", "HOST_ONLY_FILE_SENTINEL"} { - if strings.Contains(summary, forbidden) || strings.Contains(trace, forbidden) { - t.Fatalf("debug bundle contains %q", forbidden) + evidence := readAllFiles(t, result.report.DebugDirectory) + if !strings.Contains(evidence, "Aria raises her holy symbol and casts Cure Wounds.") { + t.Fatal("capture omitted selected source") + } + for _, unrelated := range []string{"HOST_ONLY_SENTINEL", "HOST_ONLY_FILE_SENTINEL"} { + if strings.Contains(evidence, unrelated) { + t.Fatalf("capture included unrelated host data: %s", unrelated) } } - if strings.Contains(summary, "application content") { - t.Fatal("summary contains raw application input") - } - if !strings.Contains(trace, "application content") { - t.Fatal("trace does not retain expected application input") - } -} - -func TestRunRedactsSensitiveModuleOptionsFromConfigAndPipelineSummaries(t *testing.T) { - roots := newStateTestRoots(t) - data, err := os.ReadFile(roots.config) - if err != nil { - t.Fatal(err) - } - configText := replaceRequiredOnce(t, string(data), " input:\n module: test/input\n", ` input: - module: test/input - options: - api_key: CONFIG_SUMMARY_SECRET_SENTINEL - safe: SAFE_OPTION_SENTINEL - nested: - - - password: PIPELINE_SUMMARY_SECRET_SENTINEL - neighbor: SAFE_NESTED_OPTION_SENTINEL -`) - if err := os.WriteFile(roots.config, []byte(configText), 0o600); err != nil { - t.Fatal(err) - } - - result := runStateTest(t, roots, newStateTestHarness().options(), true, false, "bypass") - if result.code != 0 { - t.Fatalf("code=%d stderr=%q", result.code, result.stderr) - } - summaryRoot := filepath.Join(onlyChildDir(t, roots.debug), "summary") - for _, name := range []string{"effective-config.yml", "resolved-pipeline.json"} { - contents, err := os.ReadFile(filepath.Join(summaryRoot, name)) - if err != nil { - t.Fatal(err) - } - text := string(contents) - for _, secret := range []string{"CONFIG_SUMMARY_SECRET_SENTINEL", "PIPELINE_SUMMARY_SECRET_SENTINEL"} { - if strings.Contains(text, secret) { - t.Fatalf("%s contains %q: %s", name, secret, text) - } - } - for _, retained := range []string{"[REDACTED]", "SAFE_OPTION_SENTINEL", "SAFE_NESTED_OPTION_SENTINEL"} { - if !strings.Contains(text, retained) { - t.Fatalf("%s does not contain %q: %s", name, retained, text) - } - } - } -} - -func TestRunUsesOneInjectedIdentityForDebugOutputAndManifest(t *testing.T) { - roots := newStateTestRoots(t) - harness := newStateTestHarness() - opts := harness.options() - const runID = "run-1000000000-11111111111111111111111111111111" - opts.RunIDGenerator = func(time.Time) (string, error) { return runID, nil } - - result := runStateTest(t, roots, opts, true, false, "bypass") - if result.code != 0 { - t.Fatalf("code=%d stderr=%q", result.code, result.stderr) - } - outputPath := filepath.Join(roots.output, runID) - debugPath := filepath.Join(roots.debug, runID) - assertFile(t, filepath.Join(outputPath, "result.json")) - assertFile(t, filepath.Join(debugPath, "summary", "run-manifest.json")) - if !strings.Contains(result.stdout, "output="+outputPath) || !strings.Contains(result.stdout, "debug="+debugPath) { - t.Fatalf("stdout=%q, want shared run identity", result.stdout) - } - data, err := os.ReadFile(filepath.Join(debugPath, "summary", "run-manifest.json")) - if err != nil { - t.Fatal(err) - } - var manifest artifacts.RunManifest - if err := json.Unmarshal(data, &manifest); err != nil { - t.Fatal(err) - } - if manifest.RunID != runID { - t.Fatalf("manifest run ID = %q, want %q", manifest.RunID, runID) - } - wantStartedAt := time.Unix(1, 0).UTC() - if manifest.StartedAt == nil || !manifest.StartedAt.Equal(wantStartedAt) { - t.Fatalf("manifest started at = %v, want %v", manifest.StartedAt, wantStartedAt) - } - rawInput, err := os.ReadFile(roots.input) - if err != nil { - t.Fatal(err) - } - wantSessionID, err := resolvePromptSessionID("", "test/input", rawInput) - if err != nil { - t.Fatal(err) - } - if sessionID, ok := manifest.Metadata["session_id"]; !ok || sessionID != wantSessionID { - t.Fatalf("manifest session = %#v, want %q; metadata = %#v", sessionID, wantSessionID, manifest.Metadata) - } - var invocation debugbundle.Invocation - readStateTestSummaryJSON(t, debugPath, "invocation.json", &invocation) - if invocation.RunID != runID || !invocation.StartedAt.Equal(wantStartedAt) { - t.Fatalf("debug invocation identity = %#v, want run %q at %v", invocation, runID, wantStartedAt) - } - if invocation.SessionID != wantSessionID { - t.Fatalf("debug invocation session = %q, want %q", invocation.SessionID, wantSessionID) - } - report := readStateTestRunReport(t, debugPath) - if !report.Succeeded || report.RunID != runID || report.PipelineID != "sample" || report.OutputPath != outputPath || report.DebugPath != debugPath || report.OutputCount != 1 || report.RejectedCount != 0 || report.WarningGroupCount != 0 || report.WarningOccurrenceCount != 0 || report.DiagnosticGroupCount != 0 || report.DiagnosticOccurrenceCount != 0 || report.DiagnosticsTruncated || report.ValidationStatus != "approved" { - t.Fatalf("success report = %#v", report) - } - if !strings.Contains(result.stdout, "outputs=1 rejected=0") { - t.Fatalf("stdout=%q, want report counts", result.stdout) - } -} - -func TestRunCheckpointReuseRequiresSameEffectiveSession(t *testing.T) { - roots := newStateTestRoots(t) - harness := newStateTestHarness() - run := func(extra ...string) stateTestResult { - args := []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"} - args = append(args, extra...) - var stdout, stderr bytes.Buffer - return stateTestResult{code: RunWithOptions(args, &stdout, &stderr, harness.options()), stdout: stdout.String(), stderr: stderr.String()} - } - - if result := run(); result.code != 0 { - t.Fatalf("initial run code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr) - } - harness.mu.Lock() - initialExtractCalls := harness.extractCalls - harness.mu.Unlock() - if initialExtractCalls != 1 { - t.Fatalf("initial extract calls = %d, want 1", initialExtractCalls) - } - if result := run("--resume"); result.code != 0 { - t.Fatalf("same-session resume code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr) - } - harness.mu.Lock() - reusedExtractCalls := harness.extractCalls - harness.mu.Unlock() - if reusedExtractCalls != initialExtractCalls { - t.Fatalf("extract calls after same-session resume = %d, want %d", reusedExtractCalls, initialExtractCalls) - } - if result := run("--resume", "--session-id", "different-session"); result.code != 0 { - t.Fatalf("different-session resume code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr) - } - harness.mu.Lock() - differentSessionExtractCalls := harness.extractCalls - harness.mu.Unlock() - if differentSessionExtractCalls != initialExtractCalls+1 { - t.Fatalf("extract calls after different-session resume = %d, want %d", differentSessionExtractCalls, initialExtractCalls+1) - } -} - -func TestRunWritesTerminalArtifactsForResolutionPipelineAndOutputFailures(t *testing.T) { - for _, tc := range []struct { - name string - pipelineID string - wantError string - wantOutputs int - wantValidation string - configureFailure func(*testing.T, stateTestRoots, *stateTestHarness) - }{ - {name: "resolution", pipelineID: "missing", wantError: `pipeline "missing"`}, - {name: "pipeline", pipelineID: "sample", wantError: "synthetic extraction failure", wantValidation: "failed", configureFailure: func(_ *testing.T, _ stateTestRoots, h *stateTestHarness) { - h.extractErr = errors.New("synthetic extraction failure") - }}, - {name: "output", pipelineID: "sample", wantError: "create output parent", wantOutputs: 1, wantValidation: "approved", configureFailure: func(t *testing.T, roots stateTestRoots, _ *stateTestHarness) { - if err := os.WriteFile(roots.output, []byte("not a directory"), 0o600); err != nil { - t.Fatal(err) - } - }}, - } { - t.Run(tc.name, func(t *testing.T) { - roots := newStateTestRoots(t) - harness := newStateTestHarness() - if tc.configureFailure != nil { - tc.configureFailure(t, roots, harness) - } - opts := harness.options() - var stdout, stderr bytes.Buffer - args := []string{"run", tc.pipelineID, "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--debug"} - code := RunWithOptions(args, &stdout, &stderr, opts) - if code != 1 || !strings.Contains(stderr.String(), tc.wantError) { - t.Fatalf("code=%d stderr=%q", code, stderr.String()) - } - bundlePath := onlyChildDir(t, roots.debug) - runID := filepath.Base(bundlePath) - report := readStateTestRunReport(t, bundlePath) - if report.Succeeded || report.RunID != runID || report.PipelineID != tc.pipelineID || report.OutputPath != filepath.Join(roots.output, runID) || report.DebugPath != bundlePath || report.OutputCount != tc.wantOutputs || report.RejectedCount != 0 || report.WarningGroupCount != 0 || report.WarningOccurrenceCount != 0 || report.DiagnosticGroupCount != 0 || report.DiagnosticOccurrenceCount != 0 || report.DiagnosticsTruncated || report.ValidationStatus != tc.wantValidation { - t.Fatalf("failure report = %#v", report) - } - errorLog, err := os.ReadFile(filepath.Join(bundlePath, "summary", "error.log")) - if err != nil || !strings.Contains(string(errorLog), tc.wantError) { - t.Fatalf("error log = %q, %v", errorLog, err) - } - }) - } -} - -func TestRunRetainsPartialPipelineOutcomeInFailureSummary(t *testing.T) { - roots := newStateTestRoots(t) - harness := newStateTestHarness() - harness.chunkDiagnostics = []contracts.ProducerDiagnostic{stateTestDiagnostic("chunk", "partial-warning", "warning retained before failure")} - harness.extractErr = errors.New("synthetic partial pipeline failure") - - result := runStateTest(t, roots, harness.options(), true, true, "bypass") - if result.code != 1 { - t.Fatalf("code=%d stderr=%q", result.code, result.stderr) - } - bundlePath := onlyChildDir(t, roots.debug) - report := readStateTestRunReport(t, bundlePath) - if report.Succeeded || report.OutputCount != 0 || report.RejectedCount != 0 || report.WarningGroupCount != 1 || report.WarningOccurrenceCount != 1 || report.DiagnosticGroupCount != 0 || report.DiagnosticOccurrenceCount != 0 || report.DiagnosticsTruncated || report.ValidationStatus != "failed" { - t.Fatalf("partial failure report = %#v", report) - } - - var manifest artifacts.RunManifest - readStateTestSummaryJSON(t, bundlePath, "run-manifest.json", &manifest) - if manifest.RunID != report.RunID || manifest.PipelineID != "sample" || manifest.ValidationStatus != "failed" { - t.Fatalf("partial manifest = %#v", manifest) - } - var diagnostics contracts.DiagnosticCollection - readStateTestSummaryJSON(t, bundlePath, "final-diagnostics.json", &diagnostics) - if len(diagnostics.Groups) != 1 || diagnostics.Groups[0].ReasonCode != "partial-warning" { - t.Fatalf("partial diagnostics = %#v", diagnostics) - } - var events []pipeline.CheckpointEvent - readStateTestSummaryJSON(t, bundlePath, "checkpoint-events.json", &events) - if len(events) == 0 || events[0].Stage != "source" { - t.Fatalf("partial checkpoint events = %#v, want retained source decision", events) - } - var chunkPlan artifacts.ChunkPlanSummary - readStateTestSummaryJSON(t, bundlePath, "chunk-plan.json", &chunkPlan) - if chunkPlan.Mode != "bypass" || chunkPlan.ValidationStatus == "not_run" { - t.Fatalf("partial chunk plan = %#v", chunkPlan) - } -} - -func TestRunTerminalPersistenceFailuresDoNotRecurseOrHidePrimaryError(t *testing.T) { - for _, tc := range []struct { - name string - reportErr error - errorLogErr error - wantSecondary string - }{ - {name: "run report", reportErr: errors.New("injected run report failure"), wantSecondary: "injected run report failure"}, - {name: "error log", errorLogErr: errors.New("injected error log failure"), wantSecondary: "injected error log failure"}, - } { - t.Run(tc.name, func(t *testing.T) { - roots := newStateTestRoots(t) - harness := newStateTestHarness() - harness.extractErr = errors.New("primary pipeline failure") - opts := harness.options() - var terminal *recordingTerminalWriter - opts.DebugTerminalFactory = func(delegate *debugbundle.SummaryWriter) DebugTerminalWriter { - terminal = &recordingTerminalWriter{delegate: delegate, reportErr: tc.reportErr, errorLogErr: tc.errorLogErr} - return terminal - } - - result := runStateTest(t, roots, opts, true, false, "bypass") - if result.code != 1 { - t.Fatalf("code=%d stderr=%q", result.code, result.stderr) - } - if terminal == nil { - t.Fatal("terminal writer was not constructed") - } - if terminal.reportCalls != 1 || terminal.errorLogCalls != 1 { - t.Fatalf("terminal calls = report:%d error:%d", terminal.reportCalls, terminal.errorLogCalls) - } - primaryIndex := strings.Index(result.stderr, "primary pipeline failure") - secondaryIndex := strings.Index(result.stderr, tc.wantSecondary) - debugIndex := strings.Index(result.stderr, "debug=") - if primaryIndex < 0 || secondaryIndex <= primaryIndex || debugIndex <= secondaryIndex { - t.Fatalf("stderr order = %q", result.stderr) - } - }) - } -} - -func TestRunReportFailureOnSuccessIsTerminalizedWithoutRetry(t *testing.T) { - roots := newStateTestRoots(t) - opts := newStateTestHarness().options() - var terminal *recordingTerminalWriter - opts.DebugTerminalFactory = func(delegate *debugbundle.SummaryWriter) DebugTerminalWriter { - terminal = &recordingTerminalWriter{delegate: delegate, reportErr: errors.New("injected success report failure")} - return terminal - } - - result := runStateTest(t, roots, opts, true, false, "bypass") - if result.code != 1 || !strings.Contains(result.stderr, "write debug run report") || !strings.Contains(result.stderr, "injected success report failure") { - t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr) - } - if terminal == nil { - t.Fatal("terminal writer was not constructed") - } - if terminal.reportCalls != 1 || terminal.errorLogCalls != 1 { - t.Fatalf("terminal calls = report:%d error:%d", terminal.reportCalls, terminal.errorLogCalls) - } - if result.stdout != "" { - t.Fatalf("stdout=%q, want no success message", result.stdout) - } - bundlePath := onlyChildDir(t, roots.debug) - errorLog, err := os.ReadFile(filepath.Join(bundlePath, "summary", "error.log")) - if err != nil || !strings.Contains(string(errorLog), "injected success report failure") { - t.Fatalf("error log = %q, %v", errorLog, err) - } -} - -func TestRunWithoutDebugDoesNotUseTerminalSummaryWriter(t *testing.T) { - roots := newStateTestRoots(t) - harness := newStateTestHarness() - harness.extractErr = errors.New("non-debug pipeline failure") - opts := harness.options() - factoryCalls := 0 - opts.DebugTerminalFactory = func(delegate *debugbundle.SummaryWriter) DebugTerminalWriter { - factoryCalls++ - return delegate - } - - result := runStateTest(t, roots, opts, false, false, "bypass") - if result.code != 1 || !strings.Contains(result.stderr, "non-debug pipeline failure") { - t.Fatalf("code=%d stderr=%q", result.code, result.stderr) - } - if factoryCalls != 0 { - t.Fatalf("terminal summary factory calls = %d, want 0", factoryCalls) - } - assertAbsent(t, roots.debug) -} - -func TestRunRefusesExistingOutputDirectoryWithoutChangingIt(t *testing.T) { - roots := newStateTestRoots(t) - const runID = "run-1000000000-22222222222222222222222222222222" - runPath := filepath.Join(roots.output, runID) - if err := os.MkdirAll(filepath.Join(runPath, "nested"), 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(runPath, "sentinel"), []byte("existing output"), 0o644); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(runPath, "nested", "data"), []byte("preserve me"), 0o644); err != nil { - t.Fatal(err) - } - before := readTree(t, runPath) - opts := newStateTestHarness().options() - opts.RunIDGenerator = func(time.Time) (string, error) { return runID, nil } - - result := runStateTest(t, roots, opts, true, false, "bypass") - if result.code != 1 || !strings.Contains(result.stderr, "output run directory") || !strings.Contains(result.stderr, "already exists") { - t.Fatalf("code=%d stderr=%q", result.code, result.stderr) - } - if after := readTree(t, runPath); !sameFiles(after, before) { - t.Fatalf("existing output changed: before=%v after=%v", before, after) - } - bundlePath := filepath.Join(roots.debug, runID) - report := readStateTestRunReport(t, bundlePath) - if report.Succeeded || report.RunID != runID || report.OutputPath != runPath || report.DebugPath != bundlePath || report.OutputCount != 1 || report.ValidationStatus != "approved" { - t.Fatalf("output collision report = %#v", report) - } - errorLog, err := os.ReadFile(filepath.Join(bundlePath, "summary", "error.log")) - if err != nil || !strings.Contains(string(errorLog), "already exists") { - t.Fatalf("output collision error log = %q, %v", errorLog, err) - } } func TestRepeatedRunIdentityCannotOverwriteFirstOutput(t *testing.T) { - roots := newStateTestRoots(t) - const runID = "run-1000000000-33333333333333333333333333333333" - harness := newStateTestHarness() - opts := harness.options() - opts.RunIDGenerator = func(time.Time) (string, error) { return runID, nil } - - first := runStateTest(t, roots, opts, false, false, "bypass") - if first.code != 0 { - t.Fatalf("first code=%d stderr=%q", first.code, first.stderr) + fixture := newStateRunFixture(t) + fixture.options.RunIDGenerator = func(time.Time) (string, error) { return "fixed-run", nil } + first := fixture.run(t, "--cache", "bypass") + if first.status != 0 { + t.Fatalf("first=%#v", first) } - runPath := filepath.Join(roots.output, runID) - before := readTree(t, runPath) - second := runStateTest(t, roots, opts, false, false, "bypass") - if second.code != 1 || !strings.Contains(second.stderr, "already exists") { - t.Fatalf("second code=%d stderr=%q", second.code, second.stderr) + before := readTree(t, first.report.OutputDirectory) + second := fixture.run(t, "--cache", "bypass") + if second.status != 1 || !strings.Contains(second.stderr, "already exists") { + t.Fatalf("second=%#v", second) } - if after := readTree(t, runPath); !sameFiles(after, before) { - t.Fatalf("first output changed: before=%v after=%v", before, after) + if !sameFiles(before, readTree(t, first.report.OutputDirectory)) { + t.Fatal("output collision changed first run") } } -func TestRunRefusesExistingDebugBundleWithoutChangingIt(t *testing.T) { - roots := newStateTestRoots(t) - const runID = "run-1000000000-44444444444444444444444444444444" - bundlePath := filepath.Join(roots.debug, runID) - if err := os.MkdirAll(bundlePath, 0o700); err != nil { - t.Fatal(err) +func TestRunRefusesExistingOutputAndDebugDirectoriesWithoutChangingThem(t *testing.T) { + for _, debug := range []bool{false, true} { + t.Run(fmt.Sprintf("debug=%t", debug), func(t *testing.T) { + fixture := newStateRunFixture(t) + fixture.options.RunIDGenerator = func(time.Time) (string, error) { return "fixed-run", nil } + selected := filepath.Join(fixture.outputRoot, "fixed-run") + if debug { + selected = filepath.Join(fixture.debugRoot, "fixed-run") + } + if err := os.MkdirAll(filepath.Join(selected, "nested"), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(selected, "nested", "sentinel"), []byte("preserve me"), 0600); err != nil { + t.Fatal(err) + } + before := readTree(t, selected) + result := fixture.run(t, "--cache", "bypass", fmt.Sprintf("--debug=%t", debug)) + if result.status != 1 || !strings.Contains(result.stderr, selected) { + t.Fatalf("collision=%#v", result) + } + if !reflect.DeepEqual(before, readTree(t, selected)) { + t.Fatal("existing directory changed") + } + if debug { + assertAbsent(t, fixture.outputRoot) + if len(fixture.client.requestsSnapshot()) != 0 { + t.Fatal("debug allocation failure reached provider") + } + } + }) } - sentinelPath := filepath.Join(bundlePath, "sentinel") - if err := os.WriteFile(sentinelPath, []byte("existing debug"), 0o600); err != nil { - t.Fatal(err) - } - opts := newStateTestHarness().options() - opts.RunIDGenerator = func(time.Time) (string, error) { return runID, nil } - - result := runStateTest(t, roots, opts, true, false, "bypass") - if result.code != 1 || !strings.Contains(result.stderr, "debug bundle") || !strings.Contains(result.stderr, "already exists") { - t.Fatalf("code=%d stderr=%q", result.code, result.stderr) - } - if got, err := os.ReadFile(sentinelPath); err != nil || string(got) != "existing debug" { - t.Fatalf("sentinel = %q, %v", got, err) - } - assertAbsent(t, roots.output) } -func TestRunIDGenerationFailurePrecedesDebugAllocation(t *testing.T) { - roots := newStateTestRoots(t) - opts := newStateTestHarness().options() - opts.RunIDGenerator = func(time.Time) (string, error) { return "", errors.New("random source unavailable") } - - result := runStateTest(t, roots, opts, true, false, "bypass") - if result.code != 1 || !strings.Contains(result.stderr, "generate run ID: random source unavailable") { - t.Fatalf("code=%d stderr=%q", result.code, result.stderr) +func TestRunIdentityFailuresPrecedePathUse(t *testing.T) { + for _, test := range []struct { + name, id string + err error + }{ + {name: "identity generation", err: errors.New("random source unavailable")}, + {name: "unsafe identity", id: "../outside"}, + } { + t.Run(test.name, func(t *testing.T) { + fixture := newStateRunFixture(t) + fixture.options.RunIDGenerator = func(time.Time) (string, error) { return test.id, test.err } + result := fixture.run(t, "--debug") + if result.status != 1 || result.stderr == "" || len(fixture.client.requestsSnapshot()) != 0 { + t.Fatalf("identity failure=%#v", result) + } + assertAbsent(t, fixture.outputRoot) + assertAbsent(t, fixture.debugRoot) + assertAbsent(t, fixture.cacheRoot) + }) } - assertAbsent(t, roots.debug) - assertAbsent(t, roots.output) } -func TestRunRejectsUnsafeGeneratedIdentityBeforePathUse(t *testing.T) { - roots := newStateTestRoots(t) - opts := newStateTestHarness().options() - opts.RunIDGenerator = func(time.Time) (string, error) { return "../outside", nil } +type stateFailureClient struct{ cause error } - result := runStateTest(t, roots, opts, true, false, "bypass") - if result.code != 1 || !strings.Contains(result.stderr, "invalid generated run ID") || !strings.Contains(result.stderr, "one safe path component") { - t.Fatalf("code=%d stderr=%q", result.code, result.stderr) - } - assertAbsent(t, roots.debug) - assertAbsent(t, roots.output) -} - -type stateTestRoots struct{ config, input, output, plans, checkpoints, debug string } - -func newStateTestRoots(t *testing.T) stateTestRoots { - t.Helper() - base := t.TempDir() - roots := stateTestRoots{input: filepath.Join(base, "input.txt"), output: filepath.Join(base, "output"), plans: filepath.Join(base, "plans"), checkpoints: filepath.Join(base, "checkpoints"), debug: filepath.Join(base, "debug")} - if err := os.WriteFile(roots.input, []byte("application content Bearer secretvalue sk-secretvalue"), 0o600); err != nil { - t.Fatal(err) - } - roots.config = filepath.Join(base, "config.yml") - config := fmt.Sprintf("version: 5\noutput:\n directory: %q\ncache:\n chunk_plans:\n directory: %q\n mode: auto\n checkpoints:\n enabled: true\n directory: %q\ndebug:\n directory: %q\npipelines:\n sample:\n input:\n module: test/input\n chunk:\n module: test/chunk\n artifacts:\n items:\n extract:\n module: test/extract\n merge:\n module: test/merge\n normalize:\n module: test/normalize\n output:\n module: test/output\n", roots.output, roots.plans, roots.checkpoints, roots.debug) - if err := os.WriteFile(roots.config, []byte(config), 0o600); err != nil { - t.Fatal(err) - } - return roots -} - -type stateTestResult struct { - code int - stdout, stderr string -} - -func runStateTest(t *testing.T, roots stateTestRoots, opts Options, debug, resume bool, mode string) stateTestResult { - t.Helper() - args := []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", mode} - if debug { - args = append(args, "--debug") - } - if resume { - args = append(args, "--resume") - } - var stdout, stderr bytes.Buffer - return stateTestResult{RunWithOptions(args, &stdout, &stderr, opts), stdout.String(), stderr.String()} -} - -func assertStateTestOutput(t *testing.T, root string) { - t.Helper() - output := onlyChildDir(t, root) - data, err := os.ReadFile(filepath.Join(output, "result.json")) - if err != nil || string(data) != "{\"ok\":true}\n" { - t.Fatalf("output = %q, %v", data, err) - } +func (client stateFailureClient) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + return nil, client.cause } func onlyChildDir(t *testing.T, root string) string { @@ -745,18 +303,13 @@ func assertFile(t *testing.T, path string) { t.Fatalf("file %q: %v", path, err) } } + func assertAbsent(t *testing.T, path string) { t.Helper() if _, err := os.Stat(path); !os.IsNotExist(err) { t.Fatalf("%q exists or stat failed: %v", path, err) } } -func assertAnyFile(t *testing.T, root string) { - t.Helper() - if text := readAllFiles(t, root); text == "" { - t.Fatalf("no files under %q", root) - } -} func readAllFiles(t *testing.T, root string) string { t.Helper() @@ -780,46 +333,6 @@ func readAllFiles(t *testing.T, root string) string { return content.String() } -func readStateTestRunReport(t *testing.T, bundlePath string) debugbundle.RunReport { - t.Helper() - var report debugbundle.RunReport - readStateTestSummaryJSON(t, bundlePath, "run-report.json", &report) - return report -} - -func readStateTestSummaryJSON(t *testing.T, bundlePath, name string, target any) { - t.Helper() - data, err := os.ReadFile(filepath.Join(bundlePath, "summary", name)) - if err != nil { - t.Fatal(err) - } - if err := json.Unmarshal(data, target); err != nil { - t.Fatal(err) - } -} - -func assertRestrictedTree(t *testing.T, root string) { - t.Helper() - if runtime.GOOS == "windows" { - return - } - if err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - want := os.FileMode(0o600) - if info.IsDir() { - want = 0o700 - } - if info.Mode().Perm() != want { - return fmt.Errorf("%s has mode %o, want %o", path, info.Mode().Perm(), want) - } - return nil - }); err != nil { - t.Fatal(err) - } -} - func readTree(t *testing.T, root string) map[string][]byte { t.Helper() files := map[string][]byte{} @@ -845,6 +358,7 @@ func readTree(t *testing.T, root string) map[string][]byte { } return files } + func sameFiles(left, right map[string][]byte) bool { if len(left) != len(right) { return false @@ -856,197 +370,3 @@ func sameFiles(left, right map[string][]byte) bool { } return true } - -type stateTestHarness struct { - mu sync.Mutex - chunkCalls, extractCalls int - runIDCalls uint64 - extractErr error - chunkDiagnostics []contracts.ProducerDiagnostic - moduleProfiles []string - sessionIDs []string - outputDiagnostics contracts.DiagnosticCollection - includeWarnings bool - includeWarningFile bool -} - -func newStateTestHarness() *stateTestHarness { return &stateTestHarness{} } -func (h *stateTestHarness) options() Options { - registries := pipeline.Registries{Inputs: pipeline.NewInputAdapterRegistry(), Chunkers: pipeline.NewChunkerRegistry(), ArtifactCodecs: pipeline.NewArtifactCodecRegistry(), Extractors: pipeline.NewExtractorRegistry(), Mergers: pipeline.NewMergerRegistry(), Normalizers: pipeline.NewNormalizerRegistry(), Validators: pipeline.NewValidatorRegistry(), ValidatorChains: pipeline.NewValidatorChainRegistry(), Outputs: pipeline.NewOutputEncoderRegistry()} - if err := pipeline.RegisterArtifactCodec(registries.ArtifactCodecs, stateTestCodec{}); err != nil { - panic(err) - } - if err := registries.Inputs.RegisterBuilderWithSpec(pipeline.ModuleSpec{Key: "test/input", Stage: pipeline.StageInput, ExecutionClass: contracts.ExecutionClassDeterministic, Provides: []string{"source"}}, func(map[string]any) error { return nil }, func(pipeline.BuildRequest) (contracts.InputAdapter, error) { return stateTestInput{}, nil }); err != nil { - panic(err) - } - if err := registries.Chunkers.RegisterBuilderWithSpec(pipeline.ModuleSpec{Key: "test/chunk", Stage: pipeline.StageChunk, ExecutionClass: contracts.ExecutionClassLLMBacked, Requires: []string{"source"}, Provides: []string{"chunks"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "cache-reference"}}}, func(map[string]any) error { return nil }, func(pipeline.BuildRequest) (contracts.Chunker, error) { return stateTestChunker{h}, nil }); err != nil { - panic(err) - } - if err := pipeline.RegisterExtractor(registries.Extractors, pipeline.ModuleSpec{Key: "test/extract", Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: stateTestArtifactKind}, func() (contracts.Extractor[stateTestArtifact], error) { return stateTestExtractor{h}, nil }); err != nil { - panic(err) - } - if err := pipeline.RegisterMerger(registries.Mergers, pipeline.ModuleSpec{Key: "test/merge", Stage: pipeline.StageMerge, ExecutionClass: contracts.ExecutionClassLLMBacked, Requires: []string{"artifact"}, Provides: []string{"merged"}, ArtifactKind: stateTestArtifactKind}, func() (contracts.Merger[stateTestArtifact], error) { return stateTestMerger{harness: h}, nil }); err != nil { - panic(err) - } - if err := pipeline.RegisterNormalizer(registries.Normalizers, pipeline.ModuleSpec{Key: "test/normalize", Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: stateTestArtifactKind}, func() (contracts.Normalizer[stateTestArtifact], error) { return stateTestNormalizer{harness: h}, nil }); err != nil { - panic(err) - } - if err := registries.Outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "test/output", Stage: pipeline.StageOutput, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"normalized"}, Provides: []string{"output"}}, func() (contracts.OutputEncoder, error) { - return stateTestOutput{harness: h, includeDiagnostics: h.includeWarnings}, nil - }); err != nil { - panic(err) - } - return Options{Catalog: catalogFromRegistries(registries), Registries: registries, LookupEnv: emptyLookup, Now: func() time.Time { return time.Unix(1, 0) }, RunIDGenerator: func(startedAt time.Time) (string, error) { - h.mu.Lock() - defer h.mu.Unlock() - h.runIDCalls++ - return fmt.Sprintf("run-%d-%032x", startedAt.UnixNano(), h.runIDCalls), nil - }, UserCacheDir: func() (string, error) { return "", errors.New("unexpected user cache lookup") }, LLMClientFactory: func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { - return nil, nil, nil - }} -} - -type stateTestInput struct{} - -func (stateTestInput) Key() string { return "test/input" } -func (stateTestInput) Parse(_ context.Context, req contracts.ParseRequest) (*source.SourceDocument, error) { - doc := &source.SourceDocument{ID: "source", Kind: "text", Format: "text/plain", Units: []source.SourceUnit{{ID: 1, Kind: "text", Text: string(req.Raw), Ref: source.SourceRef{SourceID: "source", StartUnitID: 1, EndUnitID: 1}}}} - digest, err := source.DigestDocument(doc) - if err != nil { - return nil, err - } - doc.Digest = digest - return doc, nil -} - -type stateTestChunker struct{ harness *stateTestHarness } - -func (stateTestChunker) Key() string { return "test/chunk" } -func (stateTestChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil } -func (c stateTestChunker) Plan(_ context.Context, req contracts.ChunkRequest) (contracts.ChunkPlanResult, error) { - c.harness.mu.Lock() - c.harness.moduleProfiles = append(c.harness.moduleProfiles, req.LLMProfile) - c.harness.sessionIDs = append(c.harness.sessionIDs, req.SessionID) - c.harness.mu.Unlock() - c.harness.mu.Lock() - c.harness.chunkCalls++ - c.harness.mu.Unlock() - return contracts.ChunkPlanResult{Plan: source.ChunkPlan{SourceDigest: req.Source.Digest, Ranges: []source.ChunkRange{{StartUnitID: 1, EndUnitID: 1}}}, Diagnostics: contracts.CloneProducerDiagnostics(c.harness.chunkDiagnostics)}, nil -} - -const stateTestArtifactKind contracts.ArtifactKind = "test/artifact" - -type stateTestArtifact struct { - Value string `json:"value"` -} -type stateTestCodec struct{} - -func (stateTestCodec) Kind() contracts.ArtifactKind { return stateTestArtifactKind } -func (stateTestCodec) Schema() contracts.ArtifactSchema { - return contracts.ArtifactSchema{ID: "test.artifact", Name: "test_artifact", Version: "v1", JSONSchema: []byte(`{"type":"object"}`)} -} -func (stateTestCodec) MediaType() string { return "application/json" } -func (stateTestCodec) EncodeCandidate(v stateTestArtifact) ([]byte, error) { - return []byte(`{"value":"ok"}`), nil -} -func (stateTestCodec) Encode(v stateTestArtifact) ([]byte, error) { - return []byte(`{"value":"ok"}`), nil -} -func (stateTestCodec) Decode([]byte) (stateTestArtifact, error) { - return stateTestArtifact{Value: "ok"}, nil -} -func (codec stateTestCodec) DecodeCandidate(content []byte) (stateTestArtifact, error) { - return codec.Decode(content) -} - -type stateTestExtractor struct{ harness *stateTestHarness } - -func (stateTestExtractor) Key() string { return "test/extract" } -func (stateTestExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil } -func (e stateTestExtractor) Extract(_ context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[stateTestArtifact], error) { - e.harness.mu.Lock() - defer e.harness.mu.Unlock() - e.harness.extractCalls++ - e.harness.moduleProfiles = append(e.harness.moduleProfiles, req.LLMProfile) - e.harness.sessionIDs = append(e.harness.sessionIDs, req.SessionID) - if e.harness.extractErr != nil { - return contracts.TypedExtractionResult[stateTestArtifact]{}, e.harness.extractErr - } - return contracts.TypedExtractionResult[stateTestArtifact]{Value: stateTestArtifact{Value: "ok"}}, nil -} - -type stateTestMerger struct{ harness *stateTestHarness } - -func (stateTestMerger) Key() string { return "test/merge" } -func (m stateTestMerger) Merge(_ context.Context, req contracts.TypedMergeRequest[stateTestArtifact]) (contracts.TypedMergeResult[stateTestArtifact], error) { - m.harness.mu.Lock() - m.harness.moduleProfiles = append(m.harness.moduleProfiles, req.LLMProfile) - m.harness.sessionIDs = append(m.harness.sessionIDs, req.SessionID) - m.harness.mu.Unlock() - return contracts.TypedMergeResult[stateTestArtifact]{Value: req.ExtractOutputs[0].Value}, nil -} - -type stateTestNormalizer struct{ harness *stateTestHarness } - -func (stateTestNormalizer) Key() string { return "test/normalize" } -func (stateTestNormalizer) ReferenceSlots() []contracts.ReferenceSlot { return nil } -func (n stateTestNormalizer) Normalize(_ context.Context, req contracts.TypedNormalizeRequest[stateTestArtifact]) (contracts.TypedNormalizeResult[stateTestArtifact], error) { - n.harness.mu.Lock() - n.harness.moduleProfiles = append(n.harness.moduleProfiles, req.LLMProfile) - n.harness.sessionIDs = append(n.harness.sessionIDs, req.SessionID) - n.harness.mu.Unlock() - return contracts.TypedNormalizeResult[stateTestArtifact]{Value: req.MergeOutput.Value}, nil -} - -type stateTestOutput struct { - harness *stateTestHarness - includeDiagnostics bool -} - -func (o stateTestOutput) Key() string { return "test/output" } -func (o stateTestOutput) Encode(_ context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) { - o.harness.mu.Lock() - o.harness.outputDiagnostics = contracts.CloneDiagnosticCollection(req.Diagnostics) - o.harness.mu.Unlock() - data := []byte("{\"ok\":true}\n") - if o.includeDiagnostics && len(req.Diagnostics.Groups) > 0 { - data = []byte(fmt.Sprintf("{\"ok\":true,\"diagnostics\":%q}\n", req.Diagnostics.Groups[0].ReasonCode)) - } - files := []contracts.OutputFile{{Name: "result.json", Bytes: data}} - if o.harness.includeWarningFile { - files = append(files, contracts.OutputFile{Name: "warnings.json", Bytes: []byte("{\"warnings\":true}\n")}) - } - return contracts.OutputResult{Files: files}, nil -} - -func stateTestDiagnostic(scope, reasonCode, message string) contracts.ProducerDiagnostic { - return contracts.ProducerDiagnostic{Disposition: contracts.DiagnosticDispositionWarning, Category: contracts.DiagnosticCategoryDegradation, ReasonCode: reasonCode, OccurrenceCount: 1, Samples: []contracts.DiagnosticSample{{Scope: scope, Message: message}}} -} - -type failingDebugRecorder struct{} - -func (failingDebugRecorder) Enabled() bool { return true } -func (failingDebugRecorder) WriteJSON(string, any) error { return errors.New("trace unavailable") } -func (failingDebugRecorder) WriteBytes(string, []byte) error { return errors.New("trace unavailable") } - -type recordingTerminalWriter struct { - delegate DebugTerminalWriter - reportErr, errorLogErr error - reportCalls, errorLogCalls int -} - -func (w *recordingTerminalWriter) WriteRunReport(report debugbundle.RunReport) error { - w.reportCalls++ - if w.reportErr != nil { - return w.reportErr - } - return w.delegate.WriteRunReport(report) -} - -func (w *recordingTerminalWriter) WriteError(message string) error { - w.errorLogCalls++ - if w.errorLogErr != nil { - return w.errorLogErr - } - return w.delegate.WriteError(message) -} diff --git a/internal/cli/state_permissions_unix_test.go b/internal/cli/state_permissions_unix_test.go index 58129bb7..2296b1fe 100644 --- a/internal/cli/state_permissions_unix_test.go +++ b/internal/cli/state_permissions_unix_test.go @@ -11,28 +11,20 @@ import ( ) func TestRunCreatesReusableGroupAccessibleCacheState(t *testing.T) { - previousUmask := syscall.Umask(0o007) + previousUmask := syscall.Umask(0007) defer syscall.Umask(previousUmask) - - roots := newStateTestRoots(t) - harness := newStateTestHarness() - result := runStateTest(t, roots, harness.options(), false, false, "auto") - if result.code != 0 { - t.Fatalf("initial run code=%d stderr=%q", result.code, result.stderr) + fixture := newStateRunFixture(t) + first := fixture.run(t, "--cache", "auto") + if first.status != 0 { + t.Fatalf("initial run=%#v", first) } - assertGroupAccessibleCacheTree(t, roots.plans) - assertGroupAccessibleCacheTree(t, roots.checkpoints) - - extractCalls := harness.extractCalls - result = runStateTest(t, roots, harness.options(), false, true, "auto") - if result.code != 0 { - t.Fatalf("resume run code=%d stderr=%q", result.code, result.stderr) - } - if harness.extractCalls != extractCalls { - t.Fatalf("extract calls after reuse = %d, want %d", harness.extractCalls, extractCalls) + assertGroupAccessibleCacheTree(t, fixture.cacheRoot) + calls := len(fixture.client.requestsSnapshot()) + second := fixture.run(t, "--cache", "auto") + if second.status != 0 || len(fixture.client.requestsSnapshot()) != calls { + t.Fatalf("reuse=%#v calls=%d/%d", second, len(fixture.client.requestsSnapshot()), calls) } } - func assertGroupAccessibleCacheTree(t *testing.T, root string) { t.Helper() if err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {