package cli import ( "context" "encoding/json" "errors" "fmt" "os" "path/filepath" "reflect" "sort" "strings" "testing" "gitea.maximumdirect.net/eric/notarius/internal/core/config" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" locationoccurrenceextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/locationoccurrences" locationextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/locationregistry" locationoccurrencenormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/locationoccurrences" locationnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/locationregistry" spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells" "gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript" "go.maximumdirect.net/commandkit" "go.maximumdirect.net/promptkit" ) func TestMaintainedExamplesLoadResolveAndList(t *testing.T) { components := productionTestComponents(t) completePipelines := make(map[string]pipeline.ResolvedPipeline) for _, example := range maintainedExampleFiles(t) { t.Run(example.name, func(t *testing.T) { cfg := loadMaintainedExample(t, example.path) if example.name == "complete" && cfg.Concurrency.Workers != 4 { t.Fatalf("complete example concurrency = %#v, want four shared workers independent of backend admission", cfg.Concurrency) } raw, err := os.ReadFile(example.transcriptPath) if err != nil { t.Fatalf("read maintained transcript %q: %v", example.transcriptPath, err) } document, err := transcript.New().Parse(context.Background(), contracts.ParseRequest{ Path: example.transcriptPath, Raw: raw, }) if err != nil { t.Fatalf("parse maintained transcript %q: %v", example.transcriptPath, err) } if len(document.Units) == 0 { t.Fatalf("maintained transcript %q has no parsed units", example.transcriptPath) } for _, pipelineID := range example.pipelineIDs { effective, err := cfg.Resolve(resolveInputForMaintainedExample(components, pipelineID)) if err != nil { t.Fatalf("resolve maintained example %q: %v", pipelineID, err) } materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ ConfigPath: example.path, WorkingDir: filepath.Dir(example.path), }) if err != nil { t.Fatalf("materialize maintained example references for %q: %v", pipelineID, err) } if example.name == "complete" { if pipelineID == "dnd-session-development" { if materialized.ChunkControls.MaxAdditionalTurns != 0 || materialized.Chunk.LLMProfile != "notarius/dnd-extraction-development" { t.Fatalf("development native controls=%#v profile=%q", materialized.ChunkControls, materialized.Chunk.LLMProfile) } parent, found := completePipelines["dnd-session"] if !found { t.Fatal("complete parent pipeline was not resolved before its development variant") } if strings.Join(exampleStepLaneIDs(materialized), "|") != strings.Join(exampleStepLaneIDs(parent), "|") { t.Fatalf("development variant steps and lanes = %v, want parent composition %v", exampleStepLaneIDs(materialized), exampleStepLaneIDs(parent)) } if got, want := strings.Join(generatedReferenceBindingDescriptions(materialized), "|"), strings.Join(generatedReferenceBindingDescriptions(parent), "|"); got != want { t.Fatalf("development generated reference bindings = %q, want parent bindings %q", got, want) } } if got := exampleStepLaneIDs(materialized); strings.Join(got, "|") != "describe-session:item-registry,location-registry,npc-registry,scene-descriptions|extract-events:combat-turns,item-occurrences,location-occurrences,npc-occurrences,spells|track-enemies:enemy-events" { t.Fatalf("complete example steps and lanes = %v, want the documented D&D extractor composition", got) } completePipelines[pipelineID] = materialized locationLane := referenceContractLane(t, materialized, "location-registry") if locationLane.ArtifactKind != dnd.LocationRegistryKind || locationLane.Extract.Module != locationextract.Key || locationLane.Merge.Module != pipeline.DefaultMergeModule || locationLane.Normalize.Module != locationnormalize.Key { t.Fatalf("location lane = %#v, want typed registry composition", locationLane) } occurrenceLane := referenceContractLane(t, materialized, "location-occurrences") if occurrenceLane.ArtifactKind != dnd.LocationOccurrenceListKind || occurrenceLane.Extract.Module != locationoccurrenceextract.Key || occurrenceLane.Merge.Module != pipeline.DefaultMergeModule || occurrenceLane.Normalize.Module != locationoccurrencenormalize.Key { t.Fatalf("location occurrence lane = %#v, want typed occurrence composition", occurrenceLane) } for _, target := range []pipeline.ResolvedReferenceTarget{occurrenceLane.ExtractReferences, occurrenceLane.NormalizeReferences} { binding, found := generatedReferenceBinding(target.Bindings, "location_registry") if !found || binding.Artifact.Step != "describe-session" || binding.Artifact.Lane != "location-registry" { t.Fatalf("location occurrence %s reference = %#v, want generated location registry", target.Stage, binding) } } for _, slot := range []string{"party", "glossary"} { if len(occurrenceLane.ExtractReferences.ReferenceSet.Slots[slot].Items) != 1 { t.Fatalf("location occurrence extractor %s reference was not materialized: %#v", slot, occurrenceLane.ExtractReferences) } if _, found := occurrenceLane.NormalizeReferences.ReferenceSet.Slots[slot]; found { t.Fatalf("location occurrence normalizer unexpectedly consumes %s: %#v", slot, occurrenceLane.NormalizeReferences) } } spellLane := referenceContractLane(t, materialized, "spells") if len(spellLane.ExtractReferences.ReferenceSet.Slots["spell_catalog"].Items) != 1 || len(spellLane.NormalizeReferences.ReferenceSet.Slots["spell_catalog"].Items) != 1 { t.Fatalf("complete example spell catalog reference was not materialized: %#v", spellLane) } itemOccurrenceLane := referenceContractLane(t, materialized, "item-occurrences") for _, references := range []pipeline.ResolvedReferenceTarget{itemOccurrenceLane.ExtractReferences, itemOccurrenceLane.NormalizeReferences} { if _, found := references.ReferenceSet.Slots["npc_registry"]; found { t.Fatalf("item occurrence lane unexpectedly depends on generated NPCs: %#v", itemOccurrenceLane) } if _, found := references.ReferenceSet.Slots["scene_descriptions"]; found { t.Fatalf("item occurrence lane unexpectedly depends on generated scene descriptions: %#v", itemOccurrenceLane) } } enemyEventLane := referenceContractLane(t, materialized, "enemy-events") for slot, want := range map[string]struct{ step, lane string }{ "npc_registry": {step: "describe-session", lane: "npc-registry"}, "scene_descriptions": {step: "describe-session", lane: "scene-descriptions"}, "combat_turns": {step: "extract-events", lane: "combat-turns"}, "npc_occurrences": {step: "extract-events", lane: "npc-occurrences"}, } { binding, found := generatedReferenceBinding(enemyEventLane.ExtractReferences.Bindings, slot) if !found || binding.Artifact.Step != want.step || binding.Artifact.Lane != want.lane { t.Fatalf("enemy event %s reference = %#v, want generated %s/%s artifact", slot, binding, want.step, want.lane) } } } } var stdout, stderr strings.Builder code := RunWithOptions([]string{"pipeline", "list", "--config", example.path, "--format", "json"}, &stdout, &stderr, productionOptionsFromComponents(components)) var ids []string if err := json.Unmarshal([]byte(stdout.String()), &ids); err != nil { t.Fatal(err) } if code != 0 || strings.Join(ids, ",") != strings.Join(example.pipelineIDs, ",") || stderr.Len() != 0 { t.Fatalf("pipelines list: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } }) } } func TestMaintainedExamplesIsolateCompanionsAndInheritNativeResources(t *testing.T) { client := &productionFakeLLMClient{} loaders := &nativeLoaders{lookupEnv: emptyLookup, engineOptions: []promptkit.Option{promptkit.WithLLMClient(client)}} ctx := context.Background() completePath := repositoryPath("examples", "dnd-complete", "config.yml") complete, err := loaders.engine(ctx, nativeSelection(completePath)) if err != nil { t.Fatal(err) } base, err := complete.InspectProfile(ctx, "notarius/dnd-extraction") if err != nil { t.Fatal(err) } example, err := complete.InspectProfile(ctx, "notarius/dnd-extraction-example") if err != nil { t.Fatal(err) } development, err := complete.InspectProfile(ctx, "notarius/dnd-extraction-development") if err != nil { t.Fatal(err) } for _, profile := range []*promptkit.ProfileInspection{example, development} { if !reflect.DeepEqual(profile.Model, base.Model) || !reflect.DeepEqual(profile.TimeoutSeconds, base.TimeoutSeconds) || !reflect.DeepEqual(profile.ServiceTier, base.ServiceTier) || profile.Backend == nil || *profile.Backend != "limited-openrouter" { t.Fatalf("inherited profile=%#v, embedded parent=%#v", profile, base) } } if development.Temperature == nil || *development.Temperature != 0.2 || !reflect.DeepEqual(example.Temperature, base.Temperature) { t.Fatalf("development customization leaked into parent: %#v %#v", development, example) } bundled, err := complete.InspectBackend(ctx, "openrouter") if err != nil { t.Fatal(err) } limited, err := complete.InspectBackend(ctx, "limited-openrouter") if err != nil { t.Fatal(err) } if limited.ConcurrencyLimit == nil || *limited.ConcurrencyLimit != 2 || limited.Endpoint != bundled.Endpoint || !reflect.DeepEqual(limited.APIKeyEnv, bundled.APIKeyEnv) || !reflect.DeepEqual(limited.SmokeTests, bundled.SmokeTests) { t.Fatalf("limited backend=%#v, bundled parent=%#v", limited, bundled) } minimal, err := loaders.engine(ctx, nativeSelection(repositoryPath("examples", "dnd-minimal", "config.yml"))) if err != nil { t.Fatal(err) } if _, err := minimal.InspectProfile(ctx, "notarius/dnd-extraction-example"); !errors.Is(err, promptkit.ErrProfileNotFound) { t.Fatalf("complete example profile leaked into minimal selection: %v", err) } if _, err := minimal.InspectBackend(ctx, "limited-openrouter"); !errors.Is(err, promptkit.ErrBackendNotFound) { t.Fatalf("complete example backend leaked into minimal selection: %v", err) } if len(client.requestsSnapshot()) != 0 { t.Fatal("example inspection called provider") } } func TestMaintainedDevelopmentOverlayResolvesNativeControlsAndRelativeOutput(t *testing.T) { path := repositoryPath("examples", "dnd-complete", "config.yml") selection := nativeSelection(path) selection.Overlay = commandkit.Supplied[string]{Set: true, Value: "development"} loaders := &nativeLoaders{lookupEnv: emptyLookup} result, err := loaders.application(context.Background(), selection) if err != nil { t.Fatal(err) } loaded, err := config.FromResult(result) if err != nil { t.Fatal(err) } cfg := loaded.RuntimeConfig() if cfg.Concurrency.Workers != 1 || cfg.Output.Directory != filepath.Join(filepath.Dir(path), "notarius-output-development") { t.Fatalf("development application controls=%#v %#v", cfg.Concurrency, cfg.Output) } resolved, err := cfg.Resolve(resolveInputForMaintainedExample(productionTestComponents(t), "dnd-session")) if err != nil { t.Fatal(err) } if resolved.ResolvedPipeline.Chunk.LLMProfile != "notarius/dnd-extraction-development" || resolved.ResolvedPipeline.ChunkControls != (contracts.ProducerControls{Mode: contracts.ModeStrict, MaxAdditionalTurns: 0, Cache: contracts.CacheBypass}) { t.Fatalf("development chunk selection=%#v controls=%#v", resolved.ResolvedPipeline.Chunk, resolved.ResolvedPipeline.ChunkControls) } } func TestMaintainedExamplesValidateEffectiveProfilesOffline(t *testing.T) { t.Chdir(repositoryPath()) t.Setenv("OPENROUTER_API_KEY", "") client := &productionFakeLLMClient{} options := Options{LLMClient: client} for _, example := range maintainedExampleFiles(t) { t.Run(example.name, func(t *testing.T) { var configOut, configErr strings.Builder if code := RunWithOptions([]string{"config", "validate", "--config", example.path}, &configOut, &configErr, options); code != 0 { t.Fatalf("configuration validation: status=%d stdout=%q stderr=%q", code, configOut.String(), configErr.String()) } for _, pipelineID := range example.pipelineIDs { t.Run(pipelineID, func(t *testing.T) { var stdout, stderr strings.Builder code := RunWithOptions([]string{ "pipeline", "validate", pipelineID, "--config", example.path, }, &stdout, &stderr, options) if code != 0 || stderr.Len() != 0 || !strings.Contains(stdout.String(), pipelineID) { t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } }) } }) } var stdout, stderr strings.Builder code := RunWithOptions([]string{ "promptkit", "show", "profile", "notarius/dnd-extraction-development", "--config", repositoryPath("examples", "dnd-complete", "config.yml"), "--format", "json", }, &stdout, &stderr, options) if code != 0 || stderr.Len() != 0 || !json.Valid([]byte(stdout.String())) || !strings.Contains(stdout.String(), "dnd-extraction-development") { t.Fatalf("development profile inspection: status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } if len(client.requestsSnapshot()) != 0 { t.Fatal("offline example validation or inspection called provider") } } func exampleStepLaneIDs(resolved pipeline.ResolvedPipeline) []string { result := make([]string, 0, len(resolved.Steps)) for _, step := range resolved.Steps { laneIDs := make([]string, 0, len(step.ArtifactLanes)) for _, lane := range step.ArtifactLanes { laneIDs = append(laneIDs, lane.ID) } sort.Strings(laneIDs) result = append(result, step.ID+":"+strings.Join(laneIDs, ",")) } return result } func generatedReferenceBindingDescriptions(resolved pipeline.ResolvedPipeline) []string { var descriptions []string for _, step := range resolved.Steps { for _, lane := range step.ArtifactLanes { for _, target := range []struct { name string target pipeline.ResolvedReferenceTarget }{ {name: "extract", target: lane.ExtractReferences}, {name: "merge", target: lane.MergeReferences}, {name: "normalize", target: lane.NormalizeReferences}, } { for _, binding := range target.target.Bindings { if binding.Artifact == nil { continue } descriptions = append(descriptions, fmt.Sprintf("%s/%s/%s/%s=%s/%s", step.ID, lane.ID, target.name, binding.SlotName, binding.Artifact.Step, binding.Artifact.Lane)) } } } } sort.Strings(descriptions) return descriptions } func TestMaintainedMinimalInvocationPublishesDomainArtifacts(t *testing.T) { t.Setenv("OPENROUTER_API_KEY", "notarius-test-credential") cfg := loadMaintainedExample(t, repositoryPath("examples", "dnd-minimal", "config.yml")) resolved, err := cfg.Resolve(resolveInputForMaintainedExample(productionTestComponents(t), "dnd-session")) if err != nil { t.Fatal(err) } lane := referenceContractLane(t, resolved.ResolvedPipeline, "spells") if lane.Extract.Module != "dnd/spells" || lane.Normalize.Module != spellnormalize.Key || len(lane.ExtractReferences.Bindings) != 0 || len(lane.NormalizeReferences.Bindings) != 0 { t.Fatalf("minimal spell composition=%#v, want embedded catalog without overlay references", lane) } outputRoot := filepath.Join(t.TempDir(), "output") fake := &productionFakeLLMClient{} options := productionRunOptions(t, fake) var stdout, stderr strings.Builder code := RunWithOptions([]string{ "run", "dnd-session", "--config", repositoryPath("examples", "dnd-minimal", "config.yml"), "--input-file", repositoryPath("examples", "dnd-minimal", "transcript.json"), "--lane", "spells", "--cache", "bypass", "--output-dir", outputRoot, "--format", "json", }, &stdout, &stderr, options) if code != 0 { t.Fatalf("status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } var report runResult if err := json.Unmarshal([]byte(stdout.String()), &report); err != nil { t.Fatal(err) } runRoot := filepath.Join(outputRoot, productionRunID) if report.Pipeline != "dnd-session" || report.ArtifactCount != 1 || report.OutputDirectory != runRoot || report.FallbackCount != 0 { t.Fatalf("completion=%#v", report) } files := readTree(t, runRoot) if len(files) != 1 || files["lanes/spells.json"] == nil { t.Fatalf("published files=%v, want only the spell artifact", files) } artifact := readProductionJSON[dnd.SpellList](t, filepath.Join(runRoot, "lanes/spells.json")) if len(artifact.SpellCasts) != 1 || artifact.SpellCasts[0].Spell != "Cure Wounds" || len(artifact.SpellCasts[0].SourceRefs) != 1 || artifact.SpellCasts[0].SourceRefs[0].SourceID != "session-alpha" { t.Fatalf("artifact=%#v, want one source-linked Cure Wounds cast", artifact) } } func TestMaintainedMalformedInputOnlyRecordsDebugFailureWhenRequested(t *testing.T) { malformed := filepath.Join(t.TempDir(), "malformed.json") if err := os.WriteFile(malformed, []byte("{not valid json"), 0o600); err != nil { t.Fatal(err) } for _, debug := range []bool{false, true} { name := "without debug" if debug { name = "with debug" } t.Run(name, func(t *testing.T) { outputRoot := filepath.Join(t.TempDir(), "output") debugRoot := filepath.Join(t.TempDir(), "debug") client := &productionFakeLLMClient{} options := productionRunOptions(t, client) args := []string{ "run", "dnd-session", "--config", repositoryPath("examples", "dnd-minimal", "config.yml"), "--input-file", malformed, "--cache", "bypass", "--output-dir", outputRoot, } if debug { args = append(args, "--debug", "--debug-dir", debugRoot) } var stdout, stderr strings.Builder code := RunWithOptions(args, &stdout, &stderr, options) if code != 1 || stdout.Len() != 0 || !strings.Contains(stderr.String(), "parse input") { t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } assertAbsent(t, outputRoot) if !debug { assertAbsent(t, debugRoot) return } bundle := onlyChildDir(t, debugRoot) outcome := readProductionJSON[struct { Error string `json:"error"` Outcome struct { Pipeline string `json:"pipeline"` Published bool `json:"published"` } `json:"outcome"` }](t, filepath.Join(bundle, "outcome.json")) if !strings.Contains(outcome.Error, "parse input") || outcome.Outcome.Pipeline != "dnd-session" || outcome.Outcome.Published { t.Fatalf("failure evidence=%#v", outcome) } if len(client.requestsSnapshot()) != 0 { t.Fatal("malformed input reached provider") } }) } } func resolveInputForMaintainedExample(components productionComponents, pipelineID string) config.ResolveInput { return config.ResolveInput{PipelineID: pipelineID, Catalog: catalogFromRegistries(components.registries)} } func TestCompleteDNDSharedCLIReferencesExpandAcrossCompatibleTargets(t *testing.T) { cfg := loadMaintainedExample(t, repositoryPath("examples", "dnd-complete", "config.yml")) catalog := catalogFromRegistries(productionTestComponents(t).registries) sources := map[string]string{ "party": "/references/party.txt", "players": "/references/players.txt", "glossary": "/references/glossary.txt", "spell_catalog": "/references/spells.json", } requests := make([]cliReferenceRequest, 0, len(sources)) for _, slot := range []string{"party", "players", "glossary", "spell_catalog"} { requests = append(requests, cliReferenceRequest{ Selector: mustParseReferenceSelector(t, slot, "--reference"), Source: sources[slot], }) } overrides, unbinds, err := resolveCLIReferenceRequests(cfg, "dnd-session", nil, catalog, requests, nil) if err != nil { t.Fatalf("expand complete D&D references: %v", err) } if len(unbinds) != 0 { t.Fatalf("unbinds = %#v, want none", unbinds) } actual := make(map[string]pipeline.ReferenceBinding, len(overrides)) for _, binding := range overrides { actual[referenceContractBindingLabel(binding)] = binding } targets, err := selectedReferenceTargets(cfg, "dnd-session", nil, catalog) if err != nil { t.Fatal(err) } matched := make(map[string]int, len(sources)) for _, target := range targets { for slot, sourcePath := range sources { if _, ok := target.slots[slot]; !ok { continue } matched[slot]++ label := targetLabel(target) + "." + slot binding, ok := actual[label] if !ok || binding.Source != sourcePath || binding.BindingSource != contracts.ReferenceBindingSourceCLI { t.Fatalf("binding %q = %#v, want CLI source %q", label, binding, sourcePath) } } } for slot := range sources { if matched[slot] < 2 { t.Fatalf("reference %q matched %d target(s), want a shared D&D reference", slot, matched[slot]) } } if _, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalog, ReferenceOverrides: overrides}); err != nil { t.Fatalf("resolve complete D&D CLI references: %v", err) } }