Migrate assembled spell and catalog contracts to native correction

This commit is contained in:
2026-11-22 05:07:00 +00:00
parent 5e68b74469
commit 21bcb4eb3b
4 changed files with 206 additions and 672 deletions

View File

@@ -1706,6 +1706,18 @@ legacy ConfigKit YAML import. Remaining CLI test migration, dependency tidy/
old-path removal, notice reconciliation, and whole-tree validation are required
before completion. This stage is not marked finished.
Assembled spell tests now exercise the real native extractor and continuation
history rather than a synthetic model-candidate/correction protocol. Retained
merged-cast ordering, canonicalization, evidence, explicit validator overrides,
and process/advisory distinctions. Native catalog correction tests cover selected
turn exhaustion and corrected overlay acceptance; strict terminal rejection
publishes no artifact. Catalog materialization retains raw-reference provenance
and semantic recognition across reordered overlay bytes, and the native CLI
publishes the grounded overlay spell. Deleted only superseded checkpoint-identity/
resume and manifest/sidecar assertions and their helpers. Focused assembled spell/
catalog tests, vet, and repeated race checks pass; remaining CLI fixtures and final
compilation are still required.
**Goal:** delete the old architecture rather than leave dormant alternatives.
**Inspect/change:** all production/test callers of the feature roadmap's removal

View File

@@ -10,19 +10,17 @@ import (
"sync"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
spells "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells"
spellcatalog "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/spells/catalog"
"go.maximumdirect.net/promptkit"
)
const assembledSpellExtractorKey = "test/dnd/spell-casts"
const assembledCorrectingSpellExtractorKey = "test/dnd/correcting-spell-casts"
const assembledDirectSpellValidatorKey = "test/dnd/direct-spell-correction"
func TestAssembledSpellPipelineNormalizesMergedCasts(t *testing.T) {
@@ -33,9 +31,8 @@ func TestAssembledSpellPipelineNormalizesMergedCasts(t *testing.T) {
}
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
Prepared: prepared,
RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"),
ChunkCacheMode: pipeline.ChunkCacheBypass,
Prepared: prepared,
RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"),
})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
@@ -46,17 +43,10 @@ func TestAssembledSpellPipelineNormalizesMergedCasts(t *testing.T) {
if !reflect.DeepEqual(chunkIndexes, []int{0, 1}) {
t.Fatalf("extractor chunk indexes = %#v, want two chunk-boundary calls", chunkIndexes)
}
if output.Manifest.ValidationStatus != "approved" || len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 1 {
t.Fatalf("run output = %#v, want approved normalized output without rejections", output)
}
if output.NormalizeOutputs[0].NormalizerKey != spellnormalize.Key {
t.Fatalf("normalized output module = %q, want %q", output.NormalizeOutputs[0].NormalizerKey, spellnormalize.Key)
}
var normalized dnd.SpellList
if err := json.Unmarshal(output.NormalizeOutputs[0].Artifact.Content, &normalized); err != nil {
t.Fatalf("decode normalized output: %v", err)
if len(output.OutputFiles) != 1 || output.FallbackCount != 0 {
t.Fatalf("output=%#v, want accepted spell artifact", output)
}
normalized := decodeAssembledOutput[dnd.SpellList](t, output.OutputFiles, "lanes/spells.json")
if len(normalized.SpellCasts) != 2 {
t.Fatalf("normalized casts = %#v, want collapsed duplicate plus distinct evidence", normalized.SpellCasts)
}
@@ -88,55 +78,60 @@ func TestAssembledSpellPipelineNormalizesMergedCasts(t *testing.T) {
t.Fatalf("duplicate diagnostic = %#v, want retained and removed merged indices", output.Diagnostics.Groups[2])
}
warningsFile := decodeAssembledOutput[struct {
Groups []contracts.DiagnosticGroup `json:"groups"`
}](t, output.OutputFiles, "warnings.json")
if len(warningsFile.Groups) != 0 {
t.Fatalf("warnings file = %#v, want no process warnings for advisory-only diagnostics", warningsFile.Groups)
}
diagnosticsFile := decodeAssembledOutput[struct {
SchemaVersion string `json:"schema_version"`
GroupCount int `json:"group_count"`
OccurrenceCount int `json:"occurrence_count"`
Truncated bool `json:"truncated"`
UnrepresentedOccurrenceCount int `json:"unrepresented_occurrence_count"`
Groups []contracts.DiagnosticGroup `json:"groups"`
}](t, output.OutputFiles, "diagnostics.json")
if diagnosticsFile.SchemaVersion != "notarius.diagnostics.v1" || diagnosticsFile.GroupCount != len(output.Diagnostics.Groups) || !reflect.DeepEqual(diagnosticsFile.Groups, output.Diagnostics.Groups) || diagnosticsFile.OccurrenceCount != diagnosticGroupOccurrences(output.Diagnostics.Groups)+output.Diagnostics.UnrepresentedOccurrenceCount || diagnosticsFile.Truncated != output.Diagnostics.Truncated || diagnosticsFile.UnrepresentedOccurrenceCount != output.Diagnostics.UnrepresentedOccurrenceCount {
t.Fatalf("diagnostics file = %#v, run diagnostics = %#v", diagnosticsFile, output.Diagnostics)
}
manifest := decodeAssembledOutput[artifacts.RunManifest](t, output.OutputFiles, "manifest.json")
if len(manifest.ArtifactLanes) != 1 || manifest.ArtifactLanes[0].Normalizer != spellnormalize.Key {
t.Fatalf("manifest lanes = %#v, want assembled spell normalizer", manifest.ArtifactLanes)
}
normalizerMetadata, ok := manifest.ArtifactLanes[0].Metadata["normalizer"].(map[string]any)
_, hasOverlayIDs := normalizerMetadata["catalog_overlay_ids"]
if !ok || normalizerMetadata["catalog_base_id"] != spellcatalog.SRD5E2014ID || !strings.HasPrefix(stringValue(normalizerMetadata["catalog_digest"]), "sha256:") || !hasOverlayIDs {
t.Fatalf("normalizer manifest metadata = %#v, want base ID, digest, and overlay IDs", manifest.ArtifactLanes[0].Metadata)
for _, group := range output.Diagnostics.Groups {
if group.Disposition == contracts.DiagnosticDispositionWarning {
t.Fatalf("advisory-only normalization emitted a process warning: %#v", group)
}
}
}
func TestAssembledSpellPipelineCorrectsRejectedDirectExtraction(t *testing.T) {
registries, resolved, extractor := assembledCorrectingSpellPipeline(t)
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
components := productionTestComponents(t)
if err := pipeline.RegisterTypedValidator[dnd.SpellList](components.registries.Validators, dnd.SpellListKind, pipeline.ValidatorSpec{Key: assembledDirectSpellValidatorKey, ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.TypedValidator[dnd.SpellList], error) { return assembledDirectSpellValidator{}, nil }); err != nil {
t.Fatal(err)
}
turns := 1
extract := pipeline.Binding(spells.Key)
extract.MaxAdditionalTurns = &turns
extract.Validators = pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{pipeline.Binding(assembledDirectSpellValidatorKey)}}
resolved, err := pipeline.ResolvePipeline(pipeline.PipelineProfile{
ID: "assembled-dnd-correcting-spells", Input: pipeline.Binding("seriatim"), Chunk: pipeline.ModuleBinding{Module: "generic", Options: map[string]any{"max_units": 2}},
Artifacts: map[string]pipeline.ArtifactLaneProfile{"spells": {Extract: extract, Normalize: pipeline.Binding(spellnormalize.Key)}}, Output: pipeline.Binding("json"),
}, pipeline.ResolveOptions{}, catalogFromRegistries(components.registries))
if err != nil {
t.Fatalf("Prepare() error = %v, want nil", err)
t.Fatal(err)
}
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
Prepared: prepared,
RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"),
ChunkCacheMode: pipeline.ChunkCacheBypass,
})
client := &assembledSpellCorrectionClient{}
prepared, err := pipeline.Prepare(resolved, components.registries, pipeline.ModuleDependencies{Engine: productionNativeEngine(t, client)})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
t.Fatal(err)
}
if len(output.Rejected) != 0 || output.Manifest.ValidationStatus != "approved" || len(output.NormalizeOutputs) != 1 {
t.Fatalf("run output = %#v, want corrected accepted spell output", output)
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{Prepared: prepared, RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json")})
if err != nil {
t.Fatal(err)
}
correction := extractor.correctionSnapshot()
if correction == nil || string(correction.AssistantResponse) != `{"spell":"Mysterious Burst"}` || !strings.Contains(correction.UserGuidance, "use a known spell name") || !strings.Contains(correction.UserGuidance, "complete corrected replacement") || strings.Contains(correction.UserGuidance, "unknown_spell") || strings.Contains(correction.UserGuidance, "spell is not in the catalog") {
t.Fatalf("extract correction = %#v, want exact rejected model response and semantic replacement guidance only", correction)
value := decodeAssembledOutput[dnd.SpellList](t, output.OutputFiles, "lanes/spells.json")
if output.FallbackCount != 0 || len(value.SpellCasts) != 1 || value.SpellCasts[0].Spell != "Cure Wounds" {
t.Fatalf("output=%#v spells=%#v", output, value)
}
client.mu.Lock()
requests := append([]promptkit.GenerateRequest(nil), client.requests...)
client.mu.Unlock()
if len(requests) != 2 {
t.Fatalf("provider calls=%d, want rejected and revised candidates", len(requests))
}
initial, revised := requests[0].Prompt.Messages, requests[1].Prompt.Messages
if len(revised) != len(initial)+2 || !reflect.DeepEqual(revised[:len(initial)], initial) {
t.Fatal("correction lost native growing history")
}
assistant, guidance := revised[len(initial)], revised[len(initial)+1]
if assistant.Role != promptkit.RoleAssistant || assistant.Content != assembledRejectedSpellResponse || guidance.Role != promptkit.RoleUser || !strings.Contains(guidance.Content, "use a known spell name") || !strings.Contains(guidance.Content, "complete replacement") {
t.Fatalf("native correction=%#v", revised[len(initial):])
}
for _, fragment := range []string{"unknown_spell", "spell is not in the catalog", assembledDirectSpellValidatorKey} {
if strings.Contains(guidance.Content, fragment) {
t.Fatalf("correction contains diagnostic prose or identifier: %s", guidance.Content)
}
}
}
@@ -159,11 +154,10 @@ func TestAssembledSpellPipelineHonorsNormalizeValidatorOverride(t *testing.T) {
t.Fatalf("Prepare() error = %v, want nil", err)
}
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
Prepared: prepared,
RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"),
ChunkCacheMode: pipeline.ChunkCacheBypass,
Prepared: prepared,
RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"),
})
if err != nil || output.Manifest.ValidationStatus != "approved" || len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 1 {
if err != nil || output.FallbackCount != 0 || len(output.OutputFiles) != 1 {
t.Fatalf("Run() error = %v output = %#v, want approved override run", err, output)
}
for _, group := range output.Diagnostics.Groups {
@@ -173,36 +167,20 @@ func TestAssembledSpellPipelineHonorsNormalizeValidatorOverride(t *testing.T) {
}
}
func TestAssembledSpellPipelinePromotesTerminalUnknownSpellDiagnostics(t *testing.T) {
func TestAssembledSpellPipelineRejectsUnknownSpellDuringNormalization(t *testing.T) {
registries, resolved, _ := assembledSpellPipeline(t, assembledSpellPipelineOptions{unknownSpell: true})
resolved.Steps[0].ArtifactLanes[0].NormalizeValidationPolicy.SemanticRejection = pipeline.SemanticRejectionRejectOutput
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
if err != nil {
t.Fatalf("Prepare() error = %v, want nil", err)
t.Fatal(err)
}
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
Prepared: prepared,
RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"),
ChunkCacheMode: pipeline.ChunkCacheBypass,
})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{Prepared: prepared, RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json")})
for _, fragment := range []string{"normalize", `lane "spells"`, "extract/dnd/spells/catalog"} {
if err == nil || !strings.Contains(err.Error(), fragment) {
t.Fatalf("error=%v, want scoped catalog rejection (%s)", err, fragment)
}
}
if output.Manifest.ValidationStatus != "rejected" || len(output.NormalizeOutputs) != 0 || len(output.Rejected) != 1 {
t.Fatalf("run output = %#v, want one rejected normalize candidate and no normalized output", output)
}
rejection := output.Rejected[0]
if rejection.Stage != string(pipeline.StageNormalize) || rejection.LaneID != "spells" || rejection.ModuleKey != spellnormalize.Key || rejection.ValidatorName != "extract/dnd/spells/catalog" || rejection.ReasonCode != "unknown_spell" {
t.Fatalf("rejection = %#v, want durable normalize catalog rejection", rejection)
}
rejectedFile := decodeAssembledOutput[struct {
Rejected []contracts.RejectedOutput `json:"rejected"`
}](t, output.OutputFiles, "rejected.json")
if !reflect.DeepEqual(rejectedFile.Rejected, output.Rejected) {
t.Fatalf("rejected file = %#v, run rejections = %#v, want durable rejection diagnostic", rejectedFile.Rejected, output.Rejected)
}
if len(output.Diagnostics.Groups) != 2 || output.Diagnostics.Groups[0].ReasonCode != spellnormalize.ReasonCodeSpellNameUnresolved || output.Diagnostics.Groups[0].Samples[0].Scope != "spell_casts[0]" || output.Diagnostics.Groups[1].ReasonCode != "spell_not_near_source" {
t.Fatalf("diagnostics = %#v, want complete terminal normalize validation diagnostics", output.Diagnostics)
if len(output.OutputFiles) != 0 || output.FallbackCount != 0 {
t.Fatalf("strict normalization failure published output: %#v", output)
}
}
@@ -213,37 +191,24 @@ func TestAssembledSpellPipelinePromotesUnknownSpellAdvisoryWhenOverrideAccepts(t
t.Fatalf("Prepare() error = %v, want nil", err)
}
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
Prepared: prepared,
RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"),
ChunkCacheMode: pipeline.ChunkCacheBypass,
Prepared: prepared,
RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"),
})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if output.Manifest.ValidationStatus != "approved" || len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 1 {
t.Fatalf("run output = %#v, want accepted unknown spell with explicit validator override", output)
}
var normalized dnd.SpellList
if err := json.Unmarshal(output.NormalizeOutputs[0].Artifact.Content, &normalized); err != nil {
t.Fatalf("decode normalized output: %v", err)
if output.FallbackCount != 0 || len(output.OutputFiles) != 1 {
t.Fatalf("output=%#v, want accepted explicit validator override", output)
}
normalized := decodeAssembledOutput[dnd.SpellList](t, output.OutputFiles, "lanes/spells.json")
if len(normalized.SpellCasts) != 1 || normalized.SpellCasts[0].Spell != "Mysterious Burst" {
t.Fatalf("normalized casts = %#v, want unresolved name preserved", normalized.SpellCasts)
}
if len(output.Diagnostics.Groups) != 1 || output.Diagnostics.Groups[0].ReasonCode != spellnormalize.ReasonCodeSpellNameUnresolved || output.Diagnostics.Groups[0].Samples[0].Scope != "spell_casts[0]" {
t.Fatalf("diagnostics = %#v, want promoted scoped unresolved-name diagnostic", output.Diagnostics)
}
warningsFile := decodeAssembledOutput[struct {
Groups []contracts.DiagnosticGroup `json:"groups"`
}](t, output.OutputFiles, "warnings.json")
if len(warningsFile.Groups) != 0 {
t.Fatalf("warnings file = %#v, want no process warnings for an advisory diagnostic", warningsFile.Groups)
}
diagnosticsFile := decodeAssembledOutput[struct {
Groups []contracts.DiagnosticGroup `json:"groups"`
}](t, output.OutputFiles, "diagnostics.json")
if !reflect.DeepEqual(diagnosticsFile.Groups, output.Diagnostics.Groups) {
t.Fatalf("diagnostics file = %#v, run diagnostics = %#v", diagnosticsFile.Groups, output.Diagnostics)
if output.Diagnostics.Groups[0].Disposition == contracts.DiagnosticDispositionWarning {
t.Fatal("unknown-spell advisory became a process warning")
}
}
@@ -252,47 +217,6 @@ type assembledSpellPipelineOptions struct {
unknownSpell bool
}
func assembledCorrectingSpellPipeline(t *testing.T) (pipeline.Registries, pipeline.ResolvedPipeline, *assembledCorrectingSpellExtractor) {
t.Helper()
components := productionTestComponents(t)
extractor := &assembledCorrectingSpellExtractor{}
if err := pipeline.RegisterExtractor[dnd.SpellList](components.registries.Extractors, pipeline.ModuleSpec{
Key: assembledCorrectingSpellExtractorKey,
Stage: pipeline.StageExtract,
ExecutionClass: contracts.ExecutionClassLLMBacked,
CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1,
Requires: []string{"chunks", "source.transcript"},
Provides: []string{"dnd.spell_casts"},
ArtifactKind: dnd.SpellListKind,
}, func() (contracts.Extractor[dnd.SpellList], error) {
return extractor, nil
}); err != nil {
t.Fatalf("register correcting extractor: %v", err)
}
if err := pipeline.RegisterTypedValidator[dnd.SpellList](components.registries.Validators, dnd.SpellListKind, pipeline.ValidatorSpec{Key: assembledDirectSpellValidatorKey, ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.TypedValidator[dnd.SpellList], error) {
return assembledDirectSpellValidator{}, nil
}); err != nil {
t.Fatalf("register direct spell validator: %v", err)
}
extract := pipeline.Binding(assembledCorrectingSpellExtractorKey)
extract.Retries = 1
extract.Validators = pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{{Module: assembledDirectSpellValidatorKey}}}
resolved, err := pipeline.ResolvePipeline(pipeline.PipelineProfile{
ID: "assembled-dnd-correcting-spells",
Input: pipeline.Binding("seriatim"),
Chunk: pipeline.ModuleBinding{Module: "generic", Options: map[string]any{"max_units": 1}},
Artifacts: map[string]pipeline.ArtifactLaneProfile{
"spells": {Extract: extract, Normalize: pipeline.Binding(spellnormalize.Key)},
},
Output: pipeline.Binding("json"),
}, pipeline.ResolveOptions{}, catalogFromRegistries(components.registries))
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
return components.registries, resolved, extractor
}
func assembledSpellPipeline(t *testing.T, options assembledSpellPipelineOptions) (pipeline.Registries, pipeline.ResolvedPipeline, *assembledSpellExtractor) {
t.Helper()
components := productionTestComponents(t)
@@ -338,50 +262,28 @@ type assembledSpellExtractor struct {
unknownSpell bool
}
type assembledCorrectingSpellExtractor struct {
mu sync.Mutex
correction *contracts.SemanticCorrection
const assembledRejectedSpellResponse = `{"spell_casts":[{"caster":"Aria","spell":"Mysterious Burst","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`
type assembledSpellCorrectionClient struct {
mu sync.Mutex
requests []promptkit.GenerateRequest
}
func (*assembledCorrectingSpellExtractor) Key() string { return assembledCorrectingSpellExtractorKey }
func (*assembledCorrectingSpellExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (e *assembledCorrectingSpellExtractor) Extract(_ context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.SpellList], error) {
if req.Source == nil || req.Chunk == nil {
return contracts.TypedExtractionResult[dnd.SpellList]{}, fmt.Errorf("correcting assembled extractor requires source and chunk")
func (client *assembledSpellCorrectionClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
response := `{"spell":"accepted"}`
value := dnd.SpellList{SpellCasts: []dnd.SpellCast{}}
if req.Chunk.Index == 0 && req.Correction == nil {
response = `{"spell":"Mysterious Burst"}`
value.SpellCasts = []dnd.SpellCast{{Caster: "Aria", Spell: "Mysterious Burst", SourceRefs: []source.SourceRef{{SourceID: req.Source.ID, StartUnitID: 1, EndUnitID: 1}}}}
client.mu.Lock()
defer client.mu.Unlock()
if !productionSchemaHasProperty(request, "spell_casts") {
return nil, fmt.Errorf("unexpected native response contract")
}
if req.Chunk.Index == 0 && req.Correction != nil {
correction, err := contracts.CloneSemanticCorrection(req.Correction)
if err != nil {
return contracts.TypedExtractionResult[dnd.SpellList]{}, err
}
e.mu.Lock()
e.correction = correction
e.mu.Unlock()
value.SpellCasts = []dnd.SpellCast{{Caster: "Aria", Spell: "Cure Wounds", SourceRefs: []source.SourceRef{{SourceID: req.Source.ID, StartUnitID: 1, EndUnitID: 1}}}}
client.requests = append(client.requests, request)
content := assembledRejectedSpellResponse
if len(client.requests) > 1 {
content = `{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`
}
candidate, err := contracts.NewModelCandidate([]byte(response), contracts.CorrectionProtocolSingleResponseV1)
if err != nil {
return contracts.TypedExtractionResult[dnd.SpellList]{}, err
}
return contracts.TypedExtractionResult[dnd.SpellList]{Value: value, ModelCandidate: candidate}, nil
}
func (e *assembledCorrectingSpellExtractor) correctionSnapshot() *contracts.SemanticCorrection {
e.mu.Lock()
defer e.mu.Unlock()
correction, err := contracts.CloneSemanticCorrection(e.correction)
if err != nil {
return nil
}
return correction
return &promptkit.GenerateResponse{Content: content}, nil
}
type assembledDirectSpellValidator struct{}
@@ -405,7 +307,7 @@ func (e *assembledSpellExtractor) Key() string { return assembledSpellExtractorK
func (*assembledSpellExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (e *assembledSpellExtractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.SpellList], error) {
func (e *assembledSpellExtractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest[dnd.SpellList]) (contracts.TypedExtractionResult[dnd.SpellList], error) {
if err := ctx.Err(); err != nil {
return contracts.TypedExtractionResult[dnd.SpellList]{}, err
}
@@ -446,14 +348,6 @@ func (e *assembledSpellExtractor) chunkIndexesSnapshot() []int {
return append([]int(nil), e.chunkIndexes...)
}
func diagnosticGroupOccurrences(groups []contracts.DiagnosticGroup) int {
count := 0
for _, group := range groups {
count += group.OccurrenceCount
}
return count
}
func decodeAssembledOutput[T any](t *testing.T, files []contracts.OutputFile, name string) T {
t.Helper()
for _, file := range files {

View File

@@ -5,19 +5,19 @@ import (
"encoding/json"
"fmt"
"path/filepath"
"strings"
"sync"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
"go.maximumdirect.net/promptkit"
)
func TestProductionSpellCatalogValidationRetries(t *testing.T) {
const retries = 2
func TestProductionSpellCatalogValidationCoordinatesCorrections(t *testing.T) {
const additionalTurns = 2
tests := []struct {
name string
responses []string
@@ -33,11 +33,11 @@ func TestProductionSpellCatalogValidationRetries(t *testing.T) {
productionSpellResponse("Unknown Spell"),
productionSpellResponse("Unknown Spell"),
},
wantCalls: retries + 1,
wantCalls: additionalTurns + 1,
wantRejected: true,
},
{
name: "overlay spell becomes valid on retry",
name: "overlay spell becomes valid after correction",
responses: []string{
productionSpellResponse("Unknown Spell"),
productionSpellResponse("Aegis of Emberfall"),
@@ -53,6 +53,12 @@ func TestProductionSpellCatalogValidationRetries(t *testing.T) {
components := productionTestComponents(t)
configPath := writeProductionSpellCatalogContractConfig(t)
cfg := loadMaintainedExample(t, configPath)
profile := cfg.Pipelines["dnd-session"]
lane := profile.Artifacts["spells"]
turns := additionalTurns
lane.Extract.MaxAdditionalTurns = &turns
profile.Artifacts["spells"] = lane
cfg.Pipelines["dnd-session"] = profile
effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalogFromRegistries(components.registries)})
if err != nil {
t.Fatalf("resolve production configuration: %v", err)
@@ -64,56 +70,36 @@ func TestProductionSpellCatalogValidationRetries(t *testing.T) {
if err != nil {
t.Fatalf("materialize production references: %v", err)
}
materialized.Steps[0].ArtifactLanes[0].Extract.Retries = retries
materialized.Steps[0].ArtifactLanes[0].ExtractValidationPolicy.SemanticRejection = pipeline.SemanticRejectionRejectOutput
llmClient := &catalogRetryLLMClient{responses: tt.responses}
prepared, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: llmClient})
llmClient := &catalogCorrectionLLMClient{responses: tt.responses}
prepared, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{Engine: productionNativeEngine(t, llmClient)})
if err != nil {
t.Fatalf("prepare production pipeline: %v", err)
}
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
Prepared: prepared,
RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"),
ChunkCacheMode: pipeline.ChunkCacheBypass,
Prepared: prepared,
RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"),
})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if calls := llmClient.CallCount(); calls > retries+1 || calls != tt.wantCalls {
t.Fatalf("LLM calls = %d, want %d and no more than %d", calls, tt.wantCalls, retries+1)
if calls := llmClient.CallCount(); calls > additionalTurns+1 || calls != tt.wantCalls {
t.Fatalf("LLM calls = %d, want %d and no more than %d", calls, tt.wantCalls, additionalTurns+1)
}
if tt.wantRejected {
if len(output.Rejected) != 1 || len(output.NormalizeOutputs) != 0 {
t.Fatalf("rejected = %#v normalized = %#v, want one nonfatal rejection and no merge output", output.Rejected, output.NormalizeOutputs)
}
rejection := output.Rejected[0]
if rejection.ReasonCode != "unknown_spell" || rejection.AttemptCount != retries+1 {
t.Fatalf("rejection = %#v, want exhausted unknown-spell rejection", rejection)
}
if len(output.Diagnostics.Groups) != 1 || output.Diagnostics.Groups[0].ReasonCode != "spell_not_near_source" {
t.Fatalf("diagnostics = %#v, want complete terminal validation diagnostic", output.Diagnostics)
if err == nil || !strings.Contains(err.Error(), "extract/dnd/spells/catalog") || len(output.OutputFiles) != 0 || output.FallbackCount != 0 {
t.Fatalf("output=%#v error=%v, want strict terminal catalog rejection", output, err)
}
return
}
if len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 1 {
t.Fatalf("rejected = %#v normalized = %#v, want only accepted output", output.Rejected, output.NormalizeOutputs)
}
var value dnd.SpellList
if err := json.Unmarshal(output.NormalizeOutputs[0].Artifact.Content, &value); err != nil {
t.Fatalf("decode normalized spell list: %v", err)
if err != nil || output.FallbackCount != 0 || len(output.OutputFiles) != 1 {
t.Fatalf("output=%#v error=%v", output, err)
}
value := decodeAssembledOutput[dnd.SpellList](t, output.OutputFiles, "lanes/spells.json")
if len(value.SpellCasts) != 1 || value.SpellCasts[0].Spell != tt.wantSpell {
t.Fatalf("normalized spell list = %#v, want accepted overlay spell", value)
}
if len(output.Diagnostics.Groups) != 2 || output.Diagnostics.Groups[0].ReasonCode != tt.wantAdvisoryCode {
t.Fatalf("diagnostics = %#v, want terminal extract and normalize diagnostics", output.Diagnostics)
}
if len(output.Diagnostics.Groups) != 2 {
t.Fatalf("diagnostics = %#v, want extract and normalize data-quality advisories", output.Diagnostics)
}
for _, diagnostic := range output.Diagnostics.Groups {
if diagnostic.Disposition != contracts.DiagnosticDispositionAdvisory || diagnostic.ReasonCode != tt.wantAdvisoryCode {
t.Fatalf("diagnostics = %#v, want only data-quality advisories", output.Diagnostics)
@@ -123,47 +109,43 @@ func TestProductionSpellCatalogValidationRetries(t *testing.T) {
}
}
type catalogRetryLLMClient struct {
type catalogCorrectionLLMClient struct {
mu sync.Mutex
responses []string
calls int
}
func (client *catalogRetryLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
func (client *catalogCorrectionLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
if err := ctx.Err(); err != nil {
return contracts.StructuredCompletionResponse{}, err
return nil, err
}
if req.PromptID != spells.PromptID {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected prompt %q", req.PromptID)
if !productionSchemaHasProperty(request, "spell_casts") {
return nil, fmt.Errorf("unexpected native spell response contract")
}
client.mu.Lock()
index := client.calls
client.calls++
client.mu.Unlock()
if index >= len(client.responses) {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("missing fake response %d", index)
return nil, fmt.Errorf("missing fake response %d", index)
}
content := []byte(client.responses[index])
if err := json.Unmarshal(content, out); err != nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate fake structured target: %w", err)
}
return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "deterministic", ProfileID: req.ProfileID}, nil
return &promptkit.GenerateResponse{Content: client.responses[index]}, nil
}
func (client *catalogRetryLLMClient) CallCount() int {
func (client *catalogCorrectionLLMClient) CallCount() int {
client.mu.Lock()
defer client.mu.Unlock()
return client.calls
}
func productionSpellResponse(name string) string {
content, err := json.Marshal(dnd.SpellList{SpellCasts: []dnd.SpellCast{{
Caster: "Aria",
Spell: name,
SourceRefs: []source.SourceRef{{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 1}},
content, err := json.Marshal(map[string]any{"spell_casts": []any{map[string]any{
"caster": "Aria", "spell": name, "source_refs": []any{map[string]int{"start_unit_id": 1, "end_unit_id": 1}},
}}})
if err != nil {
panic(err)
}
return string(content)
}
var _ promptkit.LLMClient = (*catalogCorrectionLLMClient)(nil)

View File

@@ -1,8 +1,6 @@
package cli
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"os"
"path/filepath"
@@ -10,100 +8,69 @@ import (
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/checkpoint"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells"
spellcatalog "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/spells/catalog"
)
func TestSpellCatalogBytesAffectCheckpointIdentityButNotSemanticDigest(t *testing.T) {
func TestMaterializedSpellCatalogOrderingPreservesSemanticContents(t *testing.T) {
components := productionTestComponents(t)
configPath := writeProductionSpellCatalogContractConfig(t)
effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
catalog := catalogFromRegistries(components.registries)
effective, err := loadMaintainedExample(t, configPath).Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalog})
if err != nil {
t.Fatalf("resolve production configuration: %v", err)
t.Fatal(err)
}
overlayPath := filepath.Join(t.TempDir(), "catalog.json")
resolved := effective.ResolvedPipeline
bindings := resolved.Steps[0].ArtifactLanes[0].ExtractReferences.Bindings
catalogBindingIndex := -1
for index, binding := range bindings {
if binding.SlotName == spellcatalog.SpellCatalogReferenceSlot {
catalogBindingIndex = index
break
overlayPath := filepath.Join(t.TempDir(), "catalog.json")
for _, target := range []*pipeline.ResolvedReferenceTarget{&resolved.Steps[0].ArtifactLanes[0].ExtractReferences, &resolved.Steps[0].ArtifactLanes[0].NormalizeReferences} {
matched := false
for index := range target.Bindings {
if target.Bindings[index].SlotName == spellcatalog.SpellCatalogReferenceSlot {
target.Bindings[index].Source = overlayPath
matched = true
}
}
if !matched {
t.Fatal("missing configured spell catalog binding")
}
}
if catalogBindingIndex < 0 {
t.Fatalf("spell catalog bindings = %#v, want catalog binding", bindings)
}
resolved.Steps[0].ArtifactLanes[0].ExtractReferences.Bindings[catalogBindingIndex].Source = overlayPath
normalizeBindings := resolved.Steps[0].ArtifactLanes[0].NormalizeReferences.Bindings
normalizeCatalogBindingIndex := -1
for index, binding := range normalizeBindings {
if binding.SlotName == spellcatalog.SpellCatalogReferenceSlot {
normalizeCatalogBindingIndex = index
break
materialize := func(contents string) pipeline.ResolvedPipeline {
t.Helper()
if err := os.WriteFile(overlayPath, []byte(contents), 0600); err != nil {
t.Fatal(err)
}
value, _, err := pipeline.MaterializeReferences(resolved, catalog, pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)})
if err != nil {
t.Fatal(err)
}
return value
}
if normalizeCatalogBindingIndex < 0 {
t.Fatalf("normalize spell catalog bindings = %#v, want catalog binding", normalizeBindings)
}
resolved.Steps[0].ArtifactLanes[0].NormalizeReferences.Bindings[normalizeCatalogBindingIndex].Source = overlayPath
if err := os.WriteFile(overlayPath, []byte(reorderedOverlayA), 0o600); err != nil {
t.Fatal(err)
}
materializedA, _, err := pipeline.MaterializeReferences(resolved, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)})
if err != nil {
t.Fatalf("materialize first catalog: %v", err)
}
identityA := catalogCheckpointIdentity(t, materializedA)
metadataA := catalogExtractorMetadata(t, materializedA)
normalizerMetadataA := catalogNormalizerMetadata(t, materializedA)
referenceA := catalogReference(t, materializedA)
if err := os.WriteFile(overlayPath, []byte(reorderedOverlayB), 0o600); err != nil {
t.Fatal(err)
}
materializedB, _, err := pipeline.MaterializeReferences(resolved, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)})
if err != nil {
t.Fatalf("materialize reordered catalog: %v", err)
}
identityB := catalogCheckpointIdentity(t, materializedB)
metadataB := catalogExtractorMetadata(t, materializedB)
normalizerMetadataB := catalogNormalizerMetadata(t, materializedB)
referenceB := catalogReference(t, materializedB)
if identityA.Digest == identityB.Digest {
t.Fatalf("checkpoint identity digest = %q for both raw catalog files, want invalidation", identityA.Digest)
}
if referenceA.Digest == referenceB.Digest || referenceA.OriginURI != referenceB.OriginURI {
t.Fatalf("catalog reference provenance changed from %#v to %#v, want same origin and different raw digest", referenceA, referenceB)
}
digestA, ok := metadataA["catalog_digest"].(string)
if !ok {
t.Fatalf("first extractor catalog metadata = %#v, want digest", metadataA)
}
digestB, ok := metadataB["catalog_digest"].(string)
if !ok || digestA != digestB {
t.Fatalf("extractor catalog digests = %q and %q, want same semantic digest", digestA, digestB)
}
if got, want := metadataA["catalog_overlay_ids"], []string{"campaign.a", "campaign.b"}; !reflect.DeepEqual(got, want) || !reflect.DeepEqual(metadataB["catalog_overlay_ids"], want) {
t.Fatalf("extractor overlay IDs = %#v and %#v, want %#v", got, metadataB["catalog_overlay_ids"], want)
}
normalizerDigestA, ok := normalizerMetadataA["catalog_digest"].(string)
normalizerDigestB, okB := normalizerMetadataB["catalog_digest"].(string)
if !ok || !okB || normalizerDigestA != digestA || normalizerDigestB != digestB {
t.Fatalf("normalizer catalog digests = %#v and %#v, want extractor semantic digests %q and %q", normalizerMetadataA["catalog_digest"], normalizerMetadataB["catalog_digest"], digestA, digestB)
}
if got, want := normalizerMetadataA["catalog_overlay_ids"], []string{"campaign.a", "campaign.b"}; !reflect.DeepEqual(got, want) || !reflect.DeepEqual(normalizerMetadataB["catalog_overlay_ids"], want) {
t.Fatalf("normalizer overlay IDs = %#v and %#v, want %#v", got, normalizerMetadataB["catalog_overlay_ids"], want)
first, second := materialize(reorderedOverlayA), materialize(reorderedOverlayB)
laneA, laneB := first.Steps[0].ArtifactLanes[0], second.Steps[0].ArtifactLanes[0]
for _, pair := range [][2]pipeline.ResolvedReferenceTarget{
{laneA.ExtractReferences, laneB.ExtractReferences},
{laneA.NormalizeReferences, laneB.NormalizeReferences},
} {
itemsA, itemsB := pair[0].ReferenceSet.Slots[spellcatalog.SpellCatalogReferenceSlot].Items, pair[1].ReferenceSet.Slots[spellcatalog.SpellCatalogReferenceSlot].Items
if len(itemsA) != 1 || len(itemsB) != 1 {
t.Fatal("materialized catalog missing")
}
if itemsA[0].Digest == itemsB[0].Digest || itemsA[0].Origin != itemsB[0].Origin {
t.Fatalf("raw catalog provenance=%#v/%#v", itemsA, itemsB)
}
catalogA, err := spellcatalog.ResolveEffectiveCatalog(pair[0].ReferenceSet)
if err != nil {
t.Fatal(err)
}
catalogB, err := spellcatalog.ResolveEffectiveCatalog(pair[1].ReferenceSet)
if err != nil {
t.Fatal(err)
}
if catalogA.Digest() != catalogB.Digest() || !reflect.DeepEqual(catalogA.OverlayIDs(), []string{"campaign.a", "campaign.b"}) || !reflect.DeepEqual(catalogB.OverlayIDs(), catalogA.OverlayIDs()) || !reflect.DeepEqual(catalogA.PromptSpells(), catalogB.PromptSpells()) {
t.Fatal("catalog ordering changed semantic recognition")
}
}
}
@@ -123,11 +90,11 @@ func TestConfiguredSpellCatalogBindingChangesResolvedPipelineIdentity(t *testing
t.Fatal(err)
}
components := productionTestComponents(t)
first, err := loadMaintainedExample(t, firstPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
first, err := loadMaintainedExample(t, firstPath).Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalogFromRegistries(components.registries)})
if err != nil {
t.Fatalf("resolve first configuration: %v", err)
}
second, err := loadMaintainedExample(t, secondPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
second, err := loadMaintainedExample(t, secondPath).Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalogFromRegistries(components.registries)})
if err != nil {
t.Fatalf("resolve changed configuration: %v", err)
}
@@ -136,360 +103,39 @@ func TestConfiguredSpellCatalogBindingChangesResolvedPipelineIdentity(t *testing
}
}
func TestSemanticSpellCatalogFingerprintChangesCheckpointIdentityWithoutReferenceChange(t *testing.T) {
components := productionTestComponents(t)
configPath := writeProductionSpellCatalogContractConfig(t)
effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
if err != nil {
t.Fatal(err)
}
materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)})
if err != nil {
t.Fatal(err)
}
prepared, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}})
if err != nil {
t.Fatal(err)
}
fingerprints := prepared.CheckpointFingerprints()
wantNames := map[string]struct{}{
"extract:spells:" + spells.Key + ":effective_catalog": {},
"extract:spells:" + spells.Key + ":validator:2:extract/dnd/spells/catalog:effective_catalog": {},
"normalize:spells:" + spellnormalize.Key + ":effective_catalog": {},
"normalize:spells:" + spellnormalize.Key + ":validator:2:extract/dnd/spells/catalog:effective_catalog": {},
}
seen := make(map[string]string, len(fingerprints))
for _, fingerprint := range fingerprints {
if _, ok := wantNames[fingerprint.Name]; ok {
seen[fingerprint.Name] = fingerprint.Value
}
}
if len(seen) != len(wantNames) {
t.Fatalf("prepared fingerprints = %#v, want scoped extractor and normalize catalog identities", fingerprints)
}
var catalogDigest string
for name, value := range seen {
if catalogDigest == "" {
catalogDigest = value
} else if value != catalogDigest {
t.Fatalf("prepared fingerprint %q = %q, want shared semantic catalog digest %q", name, value, catalogDigest)
}
}
identityFor := func(values []pipeline.CheckpointFingerprint) checkpoint.Identity {
identity, identityErr := checkpoint.NewIdentity(checkpoint.IdentityInput{
Pipeline: materialized,
InputKey: materialized.Input.Module,
RawInputDigest: "sha256:unchanged-input",
References: pipeline.ReferenceProvenance(materialized),
ProvenanceFingerprints: checkpointIdentityFingerprints(values),
})
if identityErr != nil {
t.Fatal(identityErr)
}
return identity
}
first := identityFor(fingerprints)
changed := replaceCheckpointFingerprintValue(t, fingerprints, normalizeSpellCatalogFingerprintName(), "sha256:changed-effective-catalog")
assertOnlyCheckpointFingerprintChanged(t, fingerprints, changed, normalizeSpellCatalogFingerprintName())
second := identityFor(changed)
if first.Digest == second.Digest || reflect.DeepEqual(first.ReferenceDigests, nil) || !reflect.DeepEqual(first.ReferenceDigests, second.ReferenceDigests) {
t.Fatalf("identities = %#v / %#v, want semantic invalidation with unchanged reference provenance", first, second)
}
}
func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t *testing.T) {
components := productionTestComponents(t)
configPath := writeProductionSpellCatalogContractConfig(t)
effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
if err != nil {
t.Fatal(err)
}
materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)})
if err != nil {
t.Fatal(err)
}
prepared, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}})
if err != nil {
t.Fatal(err)
}
fingerprints := prepared.CheckpointFingerprints()
llmFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_configuration", Value: "sha256:configuration-one"}}
settings := config.CheckpointCacheConfig{Enabled: true, Directory: t.TempDir()}
recorder, _, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, false)
if err != nil {
t.Fatal(err)
}
doc := source.SourceDocument{ID: "source", Kind: "transcript", Format: "application/json"}
doc.Units = []source.SourceUnit{{ID: 1, Kind: "turn", Text: "Aria casts Cure Wounds.", Ref: source.SourceRef{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 1}}}
doc.Digest, err = source.DigestDocument(&doc)
if err != nil {
t.Fatal(err)
}
if err := recorder.SourceSucceeded(materialized.Input.Module, &doc); err != nil {
t.Fatal(err)
}
normalizeDependencies := []pipeline.CheckpointFingerprint{{Name: "artifact[0]", Value: "sha256:merged-artifact"}}
normalizeSchema := contracts.ArtifactSchema{ID: "notarius.dnd.spells", Name: "notarius_dnd_spells", Version: "v1"}
normalizeArtifact := pipeline.CheckpointArtifact{
LaneID: "spells", ModuleKey: spellnormalize.Key, SourceID: doc.ID,
SchemaDigest: contracts.DigestArtifactSchema(normalizeSchema),
Artifact: contracts.SerializedArtifact{
Kind: dnd.SpellListKind, Schema: normalizeSchema, MediaType: "application/json", Content: []byte(`{"spell_casts":[]}`),
},
}
if err := recorder.NormalizeSucceeded("spells", spellnormalize.Key, normalizeDependencies, normalizeArtifact); err != nil {
t.Fatal(err)
}
_, sameLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true)
if err != nil {
t.Fatal(err)
}
if _, decision := sameLoader.Source(materialized.Input.Module); !decision.Reused {
t.Fatalf("same fingerprint decision = %#v, want reuse", decision)
}
if restored, decision := sameLoader.Normalize("spells", spellnormalize.Key, normalizeDependencies); !decision.Reused || string(restored.Output.Artifact.Content) != `{"spell_casts":[]}` {
t.Fatalf("same normalize checkpoint = %#v, decision=%#v, want reuse", restored, decision)
}
changed := replaceCheckpointFingerprintValue(t, fingerprints, normalizeSpellCatalogFingerprintName(), "sha256:changed-effective-catalog")
assertOnlyCheckpointFingerprintChanged(t, fingerprints, changed, normalizeSpellCatalogFingerprintName())
_, changedLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changed, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true)
if err != nil {
t.Fatal(err)
}
if _, decision := changedLoader.Source(materialized.Input.Module); decision.Reused {
t.Fatalf("changed fingerprint decision = %#v, want cold miss", decision)
}
if _, decision := changedLoader.Normalize("spells", spellnormalize.Key, normalizeDependencies); decision.Reused {
t.Fatalf("changed normalize fingerprint decision = %#v, want normalize checkpoint cold miss", decision)
}
changedMapping := replaceCheckpointFingerprintValue(t, fingerprints, extractSpellMappingFingerprintName(), "dnd.spells.extract_mapping.changed")
assertOnlyCheckpointFingerprintChanged(t, fingerprints, changedMapping, extractSpellMappingFingerprintName())
_, mappingLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changedMapping, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true)
if err != nil {
t.Fatal(err)
}
if _, decision := mappingLoader.Source(materialized.Input.Module); decision.Reused {
t.Fatalf("changed mapping policy decision = %#v, want cold miss", decision)
}
changedLLMFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_configuration", Value: "sha256:configuration-two"}}
_, profileLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, changedLLMFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true)
if err != nil {
t.Fatal(err)
}
if _, decision := profileLoader.Source(materialized.Input.Module); decision.Reused {
t.Fatalf("changed PromptKit configuration decision = %#v, want cold miss", decision)
}
if _, decision := profileLoader.Normalize("spells", spellnormalize.Key, normalizeDependencies); decision.Reused {
t.Fatalf("changed PromptKit configuration normalize decision = %#v, want cold miss", decision)
}
}
func normalizeSpellCatalogFingerprintName() string {
return "normalize:spells:" + spellnormalize.Key + ":effective_catalog"
}
func extractSpellMappingFingerprintName() string {
return "extract:spells:" + spells.Key + ":mapping_policy"
}
func replaceCheckpointFingerprintValue(t *testing.T, fingerprints []pipeline.CheckpointFingerprint, name, value string) []pipeline.CheckpointFingerprint {
t.Helper()
changed := append([]pipeline.CheckpointFingerprint(nil), fingerprints...)
matches := 0
for index := range changed {
if changed[index].Name == name {
changed[index].Value = value
matches++
}
}
if matches != 1 {
t.Fatalf("checkpoint fingerprints = %#v, want exactly one fingerprint named %q", fingerprints, name)
}
return changed
}
func assertOnlyCheckpointFingerprintChanged(t *testing.T, before, after []pipeline.CheckpointFingerprint, changedName string) {
t.Helper()
if len(before) != len(after) {
t.Fatalf("fingerprint lengths = %d and %d, want equal", len(before), len(after))
}
changes := 0
for index := range before {
if before[index].Name != after[index].Name {
t.Fatalf("fingerprint[%d] name changed from %q to %q", index, before[index].Name, after[index].Name)
}
if before[index].Value == after[index].Value {
continue
}
changes++
if before[index].Name != changedName {
t.Fatalf("fingerprint %q changed unexpectedly", before[index].Name)
}
}
if changes != 1 {
t.Fatalf("fingerprints changed %d values, want exactly %q", changes, changedName)
}
}
func TestMaintainedProductionOverlayRunAlignsGroundingValidationAndProvenance(t *testing.T) {
func TestProductionOverlayRunAlignsGroundingAndValidation(t *testing.T) {
outputRoot := filepath.Join(t.TempDir(), "output")
fake := &productionFakeLLMClient{spellResponse: productionSpellResponse("Aegis of Emberfall")}
options := productionRunOptions(t, fake)
client := &productionFakeLLMClient{spellResponse: productionSpellResponse("Aegis of Emberfall")}
configPath := writeProductionContractConfig(t, productionSpellCatalogContractConfig(t)+productionProviderConfiguration)
var stdout, stderr strings.Builder
code := RunWithOptions([]string{
"run", "dnd-session",
"--config", writeProductionSpellCatalogContractConfig(t),
"--input", repositoryPath("examples", "seriatim-minimal-transcript.json"),
"--only", "spells", "--chunk_cache", "bypass", "--output-dir", outputRoot,
}, &stdout, &stderr, options)
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input-file", repositoryPath("examples", "seriatim-minimal-transcript.json"), "--lane", "spells", "--cache", "bypass", "--output-dir", outputRoot, "--format", "json"}, &stdout, &stderr, productionRunOptions(t, client))
if code != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
t.Fatalf("status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
runRoot := filepath.Join(outputRoot, productionRunID)
manifest := readProductionJSON[artifacts.RunManifest](t, filepath.Join(runRoot, "manifest.json"))
if manifest.ValidationStatus != "approved" || len(manifest.References) == 0 || len(manifest.ArtifactLanes) != 1 {
t.Fatalf("manifest = %#v, want approved overlay run with one lane and references", manifest)
}
lane := manifest.ArtifactLanes[0]
extractorMetadata, ok := lane.Metadata["extractor"].(map[string]any)
if !ok {
t.Fatalf("lane metadata = %#v, want extractor metadata", lane.Metadata)
}
if extractorMetadata["catalog_base_id"] != spellcatalog.SRD5E2014ID || !strings.HasPrefix(stringValue(extractorMetadata["catalog_digest"]), "sha256:") {
t.Fatalf("extractor catalog metadata = %#v, want base ID and semantic digest", extractorMetadata)
}
if got := stringValues(extractorMetadata["catalog_overlay_ids"]); !reflect.DeepEqual(got, []string{"notarius.example-campaign"}) {
t.Fatalf("catalog overlay IDs = %#v, want maintained overlay", got)
}
normalizerMetadata, ok := lane.Metadata["normalizer"].(map[string]any)
if !ok {
t.Fatalf("lane metadata = %#v, want normalizer metadata", lane.Metadata)
}
if normalizerMetadata["catalog_base_id"] != spellcatalog.SRD5E2014ID || !strings.HasPrefix(stringValue(normalizerMetadata["catalog_digest"]), "sha256:") || !reflect.DeepEqual(stringValues(normalizerMetadata["catalog_overlay_ids"]), []string{"notarius.example-campaign"}) {
t.Fatalf("normalizer catalog metadata = %#v, want base ID, semantic digest, and overlay IDs", normalizerMetadata)
}
if normalizerMetadata["catalog_digest"] != extractorMetadata["catalog_digest"] || !reflect.DeepEqual(stringValues(normalizerMetadata["catalog_overlay_ids"]), stringValues(extractorMetadata["catalog_overlay_ids"])) {
t.Fatalf("extractor metadata = %#v, normalizer metadata = %#v, want shared catalog identity", extractorMetadata, normalizerMetadata)
}
var catalogProvenances []artifacts.ReferenceProvenance
for index := range manifest.References {
reference := &manifest.References[index]
if reference.SlotName == spellcatalog.SpellCatalogReferenceSlot {
catalogProvenances = append(catalogProvenances, *reference)
}
}
if len(catalogProvenances) != 2 {
t.Fatalf("manifest references = %#v, want independently materialized extract and normalize catalog provenance", manifest.References)
}
overlayBytes := readRepositoryFile(t, "examples", "dnd-spell-catalog.json")
for _, catalogProvenance := range catalogProvenances {
if catalogProvenance.Stage != "extract" && catalogProvenance.Stage != "normalize" {
t.Fatalf("catalog provenance = %#v, want extract or normalize scope", catalogProvenance)
}
if catalogProvenance.LaneID != "spells" || catalogProvenance.OriginType != "file" || catalogProvenance.MediaType != "application/json" || catalogProvenance.SizeBytes != int64(len(overlayBytes)) || catalogProvenance.Digest != digestBytes(overlayBytes) || !strings.Contains(catalogProvenance.OriginURI, "dnd-spell-catalog.json") {
t.Fatalf("catalog provenance = %#v, want raw overlay provenance in both scopes", catalogProvenance)
}
}
manifestBytes, err := json.Marshal(manifest)
if err != nil {
var report runResult
if err := json.Unmarshal([]byte(stdout.String()), &report); err != nil {
t.Fatal(err)
}
for _, leaked := range []string{"Aegis of Emberfall", "Emberfall Aegis", "Notarius example campaign spell names"} {
if strings.Contains(string(manifestBytes), leaked) {
t.Fatalf("manifest leaked overlay content %q", leaked)
}
if report.ArtifactCount != 1 || report.FallbackCount != 0 || report.OutputDirectory != filepath.Join(outputRoot, productionRunID) {
t.Fatalf("completion=%#v", report)
}
requests := fake.requestsFor(spells.PromptID)
requests := client.requestsWithProperty("spell_casts")
if len(requests) != 1 {
t.Fatalf("spell requests = %d, want one", len(requests))
t.Fatalf("native spell requests=%d", len(requests))
}
catalogInput, ok := requests[0].Inputs[spellcatalog.SpellCatalogReferenceSlot]
if !ok || !strings.Contains(string(catalogInput.Content), `"canonical_name":"Aegis of Emberfall"`) || !strings.Contains(string(catalogInput.Content), `"aliases":["Emberfall Aegis"]`) {
t.Fatalf("spell catalog prompt input = %#v, want canonical overlay name and recognition alias", catalogInput)
var prompt strings.Builder
for _, message := range requests[0].Prompt.Messages {
prompt.WriteString(message.Content)
}
artifact := readProductionJSON[dnd.SpellList](t, filepath.Join(runRoot, "lanes", "spells.json"))
if len(artifact.SpellCasts) != 1 || artifact.SpellCasts[0].Spell != "Aegis of Emberfall" {
t.Fatalf("artifact = %#v, want accepted overlay-only canonical spell", artifact)
}
rejected := readProductionJSON[struct {
Rejected []json.RawMessage `json:"rejected"`
}](t, filepath.Join(runRoot, "rejected.json"))
if len(rejected.Rejected) != 0 {
t.Fatalf("rejected = %#v, want no rejected output", rejected.Rejected)
}
}
func catalogCheckpointIdentity(t *testing.T, resolved pipeline.ResolvedPipeline) checkpoint.Identity {
t.Helper()
identity, err := checkpoint.NewIdentity(checkpoint.IdentityInput{
Pipeline: resolved,
InputKey: resolved.Input.Module,
RawInputDigest: "sha256:catalog-test-input",
References: pipeline.ReferenceProvenance(resolved),
})
if err != nil {
t.Fatalf("create checkpoint identity: %v", err)
}
return identity
}
func catalogExtractorMetadata(t *testing.T, resolved pipeline.ResolvedPipeline) map[string]any {
t.Helper()
lane := resolved.Steps[0].ArtifactLanes[0]
extractor, err := spells.New(&productionFakeLLMClient{}, spells.Options{}, lane.ExtractReferences.ReferenceSet)
if err != nil {
t.Fatalf("construct extractor: %v", err)
}
return extractor.ManifestMetadata()
}
func catalogNormalizerMetadata(t *testing.T, resolved pipeline.ResolvedPipeline) map[string]any {
t.Helper()
lane := resolved.Steps[0].ArtifactLanes[0]
normalizer, err := spellnormalize.New(spellnormalize.Options{}, lane.NormalizeReferences.ReferenceSet)
if err != nil {
t.Fatalf("construct normalizer: %v", err)
}
return normalizer.ManifestMetadata()
}
func catalogReference(t *testing.T, resolved pipeline.ResolvedPipeline) artifacts.ReferenceProvenance {
t.Helper()
for _, reference := range pipeline.ReferenceProvenance(resolved) {
if reference.SlotName == spellcatalog.SpellCatalogReferenceSlot && reference.Stage == "extract" && reference.LaneID == "spells" {
return reference
for _, fragment := range []string{`"canonical_name":"Aegis of Emberfall"`, `"aliases":["Emberfall Aegis"]`} {
if !strings.Contains(prompt.String(), fragment) {
t.Fatalf("catalog prompt omitted overlay recognition %q", fragment)
}
}
t.Fatalf("resolved references = %#v, want spell catalog provenance", pipeline.ReferenceProvenance(resolved))
return artifacts.ReferenceProvenance{}
}
func stringValue(value any) string {
result, _ := value.(string)
return result
}
func stringValues(value any) []string {
raw, err := json.Marshal(value)
if err != nil {
return nil
artifact := readProductionJSON[dnd.SpellList](t, filepath.Join(report.OutputDirectory, "lanes", "spells.json"))
if len(artifact.SpellCasts) != 1 || artifact.SpellCasts[0].Spell != "Aegis of Emberfall" {
t.Fatalf("spell artifact=%#v", artifact)
}
var values []string
if err := json.Unmarshal(raw, &values); err != nil {
return nil
}
return values
}
func digestBytes(value []byte) string {
sum := sha256.Sum256(value)
return "sha256:" + hex.EncodeToString(sum[:])
}
const reorderedOverlayA = `{