|
|
|
|
@@ -10,9 +10,7 @@ import (
|
|
|
|
|
"strings"
|
|
|
|
|
"testing"
|
|
|
|
|
|
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle"
|
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
|
|
|
|
@@ -30,8 +28,8 @@ func TestMaintainedExamplesLoadResolveAndList(t *testing.T) {
|
|
|
|
|
for _, example := range maintainedExampleFiles(t) {
|
|
|
|
|
t.Run(example.name, func(t *testing.T) {
|
|
|
|
|
cfg := loadMaintainedExample(t, example.path)
|
|
|
|
|
if example.name == "complete" && (cfg.Concurrency.TotalLLM != 2 || cfg.Concurrency.StageWorkers["extract"] != 2) {
|
|
|
|
|
t.Fatalf("complete example concurrency = %#v, want explicit limits of 2", cfg.Concurrency)
|
|
|
|
|
if example.name == "complete" && cfg.Concurrency.Workers != 2 {
|
|
|
|
|
t.Fatalf("complete example concurrency = %#v, want an explicit shared worker limit of 2", cfg.Concurrency)
|
|
|
|
|
}
|
|
|
|
|
raw, err := os.ReadFile(example.transcriptPath)
|
|
|
|
|
if err != nil {
|
|
|
|
|
@@ -71,40 +69,17 @@ func TestMaintainedExamplesLoadResolveAndList(t *testing.T) {
|
|
|
|
|
if got, want := strings.Join(generatedReferenceBindingDescriptions(materialized), "|"), strings.Join(generatedReferenceBindingDescriptions(parent), "|"); got != want {
|
|
|
|
|
t.Fatalf("development generated reference bindings = %q, want parent bindings %q", got, want)
|
|
|
|
|
}
|
|
|
|
|
eligibleBindings := 0
|
|
|
|
|
for _, step := range materialized.Steps {
|
|
|
|
|
for _, lane := range step.ArtifactLanes {
|
|
|
|
|
for _, binding := range []struct {
|
|
|
|
|
module pipeline.ModuleBinding
|
|
|
|
|
executionClass contracts.ExecutionClass
|
|
|
|
|
}{
|
|
|
|
|
{module: lane.Extract, executionClass: lane.ExtractExecutionClass},
|
|
|
|
|
{module: lane.Normalize, executionClass: lane.NormalizeExecutionClass},
|
|
|
|
|
} {
|
|
|
|
|
if binding.executionClass != "llm_backed" {
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
eligibleBindings++
|
|
|
|
|
if got := binding.module.StructuredOutputRepairAttempts; got == nil || *got != 0 {
|
|
|
|
|
t.Fatalf("development %s repair attempts = %v, want explicit zero", binding.module.Module, got)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if eligibleBindings == 0 {
|
|
|
|
|
t.Fatal("development variant had no eligible LLM bindings for repair policy")
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if got := exampleStepLaneIDs(materialized); strings.Join(got, "|") != "describe-session:item-registry,location-registry,npc-registry,scene-descriptions|extract-events:combat-turns,item-occurrences,location-occurrences,npc-occurrences,spells|track-enemies:enemy-events" {
|
|
|
|
|
t.Fatalf("complete example steps and lanes = %v, want the documented D&D extractor composition", got)
|
|
|
|
|
}
|
|
|
|
|
completePipelines[pipelineID] = materialized
|
|
|
|
|
locationLane := referenceContractLane(t, materialized, "location-registry")
|
|
|
|
|
if locationLane.ArtifactKind != dnd.LocationRegistryKind || locationLane.Extract.Module != locationextract.Key || locationLane.Extract.Retries != 2 || locationLane.Merge.Module != pipeline.DefaultMergeModule || locationLane.Normalize.Module != locationnormalize.Key || locationLane.Normalize.Retries != 2 {
|
|
|
|
|
if locationLane.ArtifactKind != dnd.LocationRegistryKind || locationLane.Extract.Module != locationextract.Key || locationLane.Merge.Module != pipeline.DefaultMergeModule || locationLane.Normalize.Module != locationnormalize.Key {
|
|
|
|
|
t.Fatalf("location lane = %#v, want typed registry composition", locationLane)
|
|
|
|
|
}
|
|
|
|
|
occurrenceLane := referenceContractLane(t, materialized, "location-occurrences")
|
|
|
|
|
if occurrenceLane.ArtifactKind != dnd.LocationOccurrenceListKind || occurrenceLane.Extract.Module != locationoccurrenceextract.Key || occurrenceLane.Extract.Retries != 2 || occurrenceLane.Merge.Module != pipeline.DefaultMergeModule || occurrenceLane.Normalize.Module != locationoccurrencenormalize.Key {
|
|
|
|
|
if occurrenceLane.ArtifactKind != dnd.LocationOccurrenceListKind || occurrenceLane.Extract.Module != locationoccurrenceextract.Key || occurrenceLane.Merge.Module != pipeline.DefaultMergeModule || occurrenceLane.Normalize.Module != locationoccurrencenormalize.Key {
|
|
|
|
|
t.Fatalf("location occurrence lane = %#v, want typed occurrence composition", occurrenceLane)
|
|
|
|
|
}
|
|
|
|
|
for _, target := range []pipeline.ResolvedReferenceTarget{occurrenceLane.ExtractReferences, occurrenceLane.NormalizeReferences} {
|
|
|
|
|
@@ -150,8 +125,12 @@ func TestMaintainedExamplesLoadResolveAndList(t *testing.T) {
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
var stdout, stderr strings.Builder
|
|
|
|
|
code := RunWithOptions([]string{"pipelines", "list", "--config", example.path}, &stdout, &stderr, productionOptionsFromComponents(components))
|
|
|
|
|
if code != 0 || stdout.String() != strings.Join(example.pipelineIDs, "\n")+"\n" || stderr.Len() != 0 {
|
|
|
|
|
code := RunWithOptions([]string{"pipeline", "list", "--config", example.path, "--format", "json"}, &stdout, &stderr, productionOptionsFromComponents(components))
|
|
|
|
|
var ids []string
|
|
|
|
|
if err := json.Unmarshal([]byte(stdout.String()), &ids); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
if code != 0 || strings.Join(ids, ",") != strings.Join(example.pipelineIDs, ",") || stderr.Len() != 0 {
|
|
|
|
|
t.Fatalf("pipelines list: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
@@ -206,9 +185,9 @@ func TestMaintainedExamplesValidateEffectiveProfilesOffline(t *testing.T) {
|
|
|
|
|
t.Run(pipelineID, func(t *testing.T) {
|
|
|
|
|
var stdout, stderr strings.Builder
|
|
|
|
|
code := RunWithOptions([]string{
|
|
|
|
|
"config", "validate", "--config", example.path, "--pipeline", pipelineID,
|
|
|
|
|
"pipeline", "validate", pipelineID, "--config", example.path,
|
|
|
|
|
}, &stdout, &stderr, Options{})
|
|
|
|
|
if code != 0 || stderr.Len() != 0 || !strings.Contains(stdout.String(), `valid for pipeline "`+pipelineID+`"`) {
|
|
|
|
|
if code != 0 || stderr.Len() != 0 || !strings.Contains(stdout.String(), pipelineID) {
|
|
|
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
@@ -218,21 +197,11 @@ func TestMaintainedExamplesValidateEffectiveProfilesOffline(t *testing.T) {
|
|
|
|
|
|
|
|
|
|
var stdout, stderr strings.Builder
|
|
|
|
|
code := RunWithOptions([]string{
|
|
|
|
|
"config", "profile", "--config", repositoryPath("examples", "dnd-complete.config.yml"), "dnd-extraction-development",
|
|
|
|
|
"promptkit", "show", "profile", "notarius/dnd-extraction-development",
|
|
|
|
|
"--config", repositoryPath("examples", "dnd-complete.config.yml"), "--format", "json",
|
|
|
|
|
}, &stdout, &stderr, Options{})
|
|
|
|
|
var profile struct {
|
|
|
|
|
ProfileID string `json:"profile_id"`
|
|
|
|
|
Target struct {
|
|
|
|
|
BackendID string `json:"backend_id"`
|
|
|
|
|
Model string `json:"model"`
|
|
|
|
|
Temperature float64 `json:"temperature"`
|
|
|
|
|
} `json:"effective_model_params"`
|
|
|
|
|
}
|
|
|
|
|
if err := json.Unmarshal([]byte(stdout.String()), &profile); err != nil {
|
|
|
|
|
t.Fatalf("development profile JSON = %q: %v", stdout.String(), err)
|
|
|
|
|
}
|
|
|
|
|
if code != 0 || stderr.Len() != 0 || profile.ProfileID != "dnd-extraction-development" || profile.Target.BackendID != "openrouter" || profile.Target.Model != "openai/gpt-5.6-luna" || profile.Target.Temperature != 0.2 {
|
|
|
|
|
t.Fatalf("development profile inspection = code %d, %q, %q, %#v", code, stdout.String(), stderr.String(), profile)
|
|
|
|
|
if code != 0 || stderr.Len() != 0 || !json.Valid([]byte(stdout.String())) || !strings.Contains(stdout.String(), "dnd-extraction-development") {
|
|
|
|
|
t.Fatalf("development profile inspection: status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@@ -274,7 +243,16 @@ func generatedReferenceBindingDescriptions(resolved pipeline.ResolvedPipeline) [
|
|
|
|
|
return descriptions
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestMaintainedMinimalInvocationProducesJSONBundle(t *testing.T) {
|
|
|
|
|
func TestMaintainedMinimalInvocationPublishesDomainArtifacts(t *testing.T) {
|
|
|
|
|
cfg := loadMaintainedExample(t, repositoryPath("examples", "dnd-minimal.config.yml"))
|
|
|
|
|
resolved, err := cfg.Resolve(resolveInputForMaintainedExample(productionTestComponents(t), "dnd-session"))
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
lane := referenceContractLane(t, resolved.ResolvedPipeline, "spells")
|
|
|
|
|
if lane.Extract.Module != "dnd/spells" || lane.Normalize.Module != spellnormalize.Key || len(lane.ExtractReferences.Bindings) != 0 || len(lane.NormalizeReferences.Bindings) != 0 {
|
|
|
|
|
t.Fatalf("minimal spell composition=%#v, want embedded catalog without overlay references", lane)
|
|
|
|
|
}
|
|
|
|
|
outputRoot := filepath.Join(t.TempDir(), "output")
|
|
|
|
|
fake := &productionFakeLLMClient{}
|
|
|
|
|
options := productionRunOptions(t, fake)
|
|
|
|
|
@@ -282,60 +260,27 @@ func TestMaintainedMinimalInvocationProducesJSONBundle(t *testing.T) {
|
|
|
|
|
code := RunWithOptions([]string{
|
|
|
|
|
"run", "dnd-session",
|
|
|
|
|
"--config", repositoryPath("examples", "dnd-minimal.config.yml"),
|
|
|
|
|
"--input", repositoryPath("examples", "seriatim-minimal-transcript.json"),
|
|
|
|
|
"--only", "spells", "--chunk_cache", "bypass", "--output-dir", outputRoot,
|
|
|
|
|
"--input-file", repositoryPath("examples", "seriatim-minimal-transcript.json"),
|
|
|
|
|
"--lane", "spells", "--cache", "bypass", "--output-dir", outputRoot, "--format", "json",
|
|
|
|
|
}, &stdout, &stderr, options)
|
|
|
|
|
if code != 0 {
|
|
|
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
|
|
|
t.Fatalf("status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
|
|
|
}
|
|
|
|
|
if !strings.Contains(stdout.String(), `pipeline "dnd-session"`) || !strings.Contains(stdout.String(), "outputs=1 rejected=0") {
|
|
|
|
|
t.Fatalf("stdout=%q, want completed pipeline and counts", stdout.String())
|
|
|
|
|
var report runResult
|
|
|
|
|
if err := json.Unmarshal([]byte(stdout.String()), &report); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
runRoot := filepath.Join(outputRoot, productionRunID)
|
|
|
|
|
index := readProductionJSON[exampleOutputIndex](t, filepath.Join(runRoot, "index.json"))
|
|
|
|
|
if index.ManifestFile != "manifest.json" || index.RejectedFile != "rejected.json" || index.WarningsFile != "warnings.json" || len(index.OutputFiles) != 1 {
|
|
|
|
|
t.Fatalf("index = %#v, want one spells output and fixed companion files", index)
|
|
|
|
|
if report.Pipeline != "dnd-session" || report.ArtifactCount != 1 || report.OutputDirectory != runRoot || report.FallbackCount != 0 {
|
|
|
|
|
t.Fatalf("completion=%#v", report)
|
|
|
|
|
}
|
|
|
|
|
entry := index.OutputFiles[0]
|
|
|
|
|
if entry.LaneID != "spells" || entry.File != "lanes/spells.json" || entry.MediaType != "application/json" || entry.SchemaID != "notarius.dnd.spells" || entry.SchemaVersion != "v1" {
|
|
|
|
|
t.Fatalf("index output entry = %#v, want spells JSON contract", entry)
|
|
|
|
|
files := readTree(t, runRoot)
|
|
|
|
|
if len(files) != 1 || files["lanes/spells.json"] == nil {
|
|
|
|
|
t.Fatalf("published files=%v, want only the spell artifact", files)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
manifest := readProductionJSON[artifacts.RunManifest](t, filepath.Join(runRoot, "manifest.json"))
|
|
|
|
|
if manifest.PipelineID != "dnd-session" || manifest.InputModule != "seriatim" || manifest.Chunker != "generic" || manifest.OutputEncoder != "json" || manifest.ValidationStatus != "approved" || manifest.ChunkPlan == nil || manifest.ChunkPlan.Action != "bypassed" {
|
|
|
|
|
t.Fatalf("manifest = %#v, want approved minimal run", manifest)
|
|
|
|
|
}
|
|
|
|
|
if len(manifest.ArtifactLanes) != 1 {
|
|
|
|
|
t.Fatalf("manifest lanes = %#v, want exactly spells", manifest.ArtifactLanes)
|
|
|
|
|
}
|
|
|
|
|
lane := manifest.ArtifactLanes[0]
|
|
|
|
|
if lane.ID != "spells" || lane.Extractor != "dnd/spells" || lane.Merger != "appendorder" || lane.Normalizer != spellnormalize.Key {
|
|
|
|
|
t.Fatalf("manifest lane = %#v, want production spells composition", lane)
|
|
|
|
|
}
|
|
|
|
|
if len(manifest.References) != 0 {
|
|
|
|
|
t.Fatalf("base-only manifest references = %#v, want no overlay provenance", manifest.References)
|
|
|
|
|
}
|
|
|
|
|
extractorMetadata, ok := lane.Metadata["extractor"].(map[string]any)
|
|
|
|
|
if !ok || len(stringValues(extractorMetadata["catalog_overlay_ids"])) != 0 {
|
|
|
|
|
t.Fatalf("base-only extractor metadata = %#v, want no overlay IDs", lane.Metadata)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
artifact := readProductionJSON[dnd.SpellList](t, filepath.Join(runRoot, entry.File))
|
|
|
|
|
if len(artifact.SpellCasts) != 1 || artifact.SpellCasts[0].Spell != "Cure Wounds" || artifact.SpellCasts[0].SourceRefs[0].SourceID != "session-alpha" {
|
|
|
|
|
t.Fatalf("artifact = %#v, want one source-linked Cure Wounds cast", artifact)
|
|
|
|
|
}
|
|
|
|
|
rejected := readProductionJSON[struct {
|
|
|
|
|
Rejected []json.RawMessage `json:"rejected"`
|
|
|
|
|
}](t, filepath.Join(runRoot, "rejected.json"))
|
|
|
|
|
if len(rejected.Rejected) != 0 {
|
|
|
|
|
t.Fatalf("rejected = %#v, want empty rejection list", rejected.Rejected)
|
|
|
|
|
}
|
|
|
|
|
warnings := readProductionJSON[struct {
|
|
|
|
|
Groups []json.RawMessage `json:"groups"`
|
|
|
|
|
}](t, filepath.Join(runRoot, "warnings.json"))
|
|
|
|
|
if len(warnings.Groups) != 0 {
|
|
|
|
|
t.Fatalf("warnings = %#v, want empty warning list", warnings.Groups)
|
|
|
|
|
artifact := readProductionJSON[dnd.SpellList](t, filepath.Join(runRoot, "lanes/spells.json"))
|
|
|
|
|
if len(artifact.SpellCasts) != 1 || artifact.SpellCasts[0].Spell != "Cure Wounds" || len(artifact.SpellCasts[0].SourceRefs) != 1 || artifact.SpellCasts[0].SourceRefs[0].SourceID != "session-alpha" {
|
|
|
|
|
t.Fatalf("artifact=%#v, want one source-linked Cure Wounds cast", artifact)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@@ -352,11 +297,12 @@ func TestMaintainedMalformedInputOnlyRecordsDebugFailureWhenRequested(t *testing
|
|
|
|
|
t.Run(name, func(t *testing.T) {
|
|
|
|
|
outputRoot := filepath.Join(t.TempDir(), "output")
|
|
|
|
|
debugRoot := filepath.Join(t.TempDir(), "debug")
|
|
|
|
|
options := productionRunOptions(t, &productionFakeLLMClient{})
|
|
|
|
|
client := &productionFakeLLMClient{}
|
|
|
|
|
options := productionRunOptions(t, client)
|
|
|
|
|
args := []string{
|
|
|
|
|
"run", "dnd-session",
|
|
|
|
|
"--config", repositoryPath("examples", "dnd-minimal.config.yml"),
|
|
|
|
|
"--input", malformed, "--chunk_cache", "bypass", "--output-dir", outputRoot,
|
|
|
|
|
"--input-file", malformed, "--cache", "bypass", "--output-dir", outputRoot,
|
|
|
|
|
}
|
|
|
|
|
if debug {
|
|
|
|
|
args = append(args, "--debug", "--debug-dir", debugRoot)
|
|
|
|
|
@@ -372,35 +318,23 @@ func TestMaintainedMalformedInputOnlyRecordsDebugFailureWhenRequested(t *testing
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
bundle := onlyChildDir(t, debugRoot)
|
|
|
|
|
report := readProductionJSON[debugbundle.RunReport](t, filepath.Join(bundle, "summary", "run-report.json"))
|
|
|
|
|
if report.Succeeded || report.PipelineID != "dnd-session" {
|
|
|
|
|
t.Fatalf("failure report = %#v, want failed dnd-session report", report)
|
|
|
|
|
outcome := readProductionJSON[struct {
|
|
|
|
|
Error string `json:"error"`
|
|
|
|
|
Outcome struct {
|
|
|
|
|
Pipeline string `json:"pipeline"`
|
|
|
|
|
Published bool `json:"published"`
|
|
|
|
|
} `json:"outcome"`
|
|
|
|
|
}](t, filepath.Join(bundle, "outcome.json"))
|
|
|
|
|
if !strings.Contains(outcome.Error, "parse input") || outcome.Outcome.Pipeline != "dnd-session" || outcome.Outcome.Published {
|
|
|
|
|
t.Fatalf("failure evidence=%#v", outcome)
|
|
|
|
|
}
|
|
|
|
|
invocation := readProductionJSON[debugbundle.Invocation](t, filepath.Join(bundle, "summary", "invocation.json"))
|
|
|
|
|
manifest := readProductionJSON[artifacts.RunManifest](t, filepath.Join(bundle, "summary", "run-manifest.json"))
|
|
|
|
|
manifestSession, found := manifest.Metadata["session_id"]
|
|
|
|
|
if invocation.SessionID == "" || !found || manifestSession != invocation.SessionID {
|
|
|
|
|
t.Fatalf("failed run sessions: invocation=%q manifest=%#v metadata=%#v", invocation.SessionID, manifestSession, manifest.Metadata)
|
|
|
|
|
if len(client.requestsSnapshot()) != 0 {
|
|
|
|
|
t.Fatal("malformed input reached provider")
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
type exampleOutputIndex struct {
|
|
|
|
|
ManifestFile string `json:"manifest_file"`
|
|
|
|
|
OutputFiles []exampleOutputIndexEntry `json:"output_files"`
|
|
|
|
|
RejectedFile string `json:"rejected_file"`
|
|
|
|
|
WarningsFile string `json:"warnings_file"`
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
type exampleOutputIndexEntry struct {
|
|
|
|
|
LaneID string `json:"lane_id"`
|
|
|
|
|
MediaType string `json:"media_type"`
|
|
|
|
|
File string `json:"file"`
|
|
|
|
|
SchemaID string `json:"schema_id"`
|
|
|
|
|
SchemaVersion string `json:"schema_version"`
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func resolveInputForMaintainedExample(components productionComponents, pipelineID string) config.ResolveInput {
|
|
|
|
|
return config.ResolveInput{PipelineID: pipelineID, Catalog: catalogFromRegistries(components.registries)}
|
|
|
|
|
}
|
|
|
|
|
|